orchestra/internal/orchestrator/orchestrator.go

// Package orchestrator connects router lease events to an opaque herdr
// session. It is deliberately small: scheduling remains in router and the
// adapter remains the only component that knows how to drive a harness.
package orchestrator

import (
	"context"
	"encoding/json"
	"fmt"
	"orchestra/internal/authz"
	"orchestra/internal/continuity"
	"orchestra/internal/domain"
	"orchestra/internal/herdr"
	"orchestra/internal/store"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type Worktrees interface {
	Create(context.Context, domain.Task) (string, error)
}
type WorktreeSpec interface {
	Spec(domain.Task) (string, string, bool)
}
type WorktreeCleaner interface {
	Remove(context.Context, domain.Task, string) error
}
type Adapters interface {
	Adapter(string) (herdr.Adapter, error)
}

// GitWorktrees creates one isolated checkout per task. The root is expected
// to be a clone containing the project's remote; callers may set a separate
// root per deployment.
type GitWorktrees struct {
	Root        string
	Repo        string
	TaskFileSHA string
}

func (w GitWorktrees) Spec(domain.Task) (string, string, bool) {
	return w.Repo, w.Root, w.Repo != "" && w.Root != ""
}

func (w GitWorktrees) Create(ctx context.Context, t domain.Task) (string, error) {
	if w.Root == "" || w.Repo == "" {
		return "", fmt.Errorf("worktree: root and repo required")
	}
	if err := os.MkdirAll(w.Root, 0755); err != nil {
		return "", err
	}
	p := filepath.Join(w.Root, t.ID)
	if _, err := os.Stat(p); err == nil {
		if w.TaskFileSHA != "" {
			if err := continuity.VerifyTaskFile(p, w.TaskFileSHA); err != nil {
				return "", err
			}
		}
		return p, nil
	}
	branch := "orchestra/" + t.ID
	cmd := exec.CommandContext(ctx, "git", "-C", w.Repo, "worktree", "add", "-b", branch, p, "HEAD")
	if out, err := cmd.CombinedOutput(); err != nil {
		return "", fmt.Errorf("%s: %w", string(out), err)
	}
	if err := writeTaskFile(ctx, p, t); err != nil {
		return "", err
	}
	if w.TaskFileSHA != "" {
		if err := continuity.VerifyTaskFile(p, w.TaskFileSHA); err != nil {
			return "", err
		}
	}
	return p, nil
}

// writeTaskFile commits the §6.2 immutable TASK.md into a freshly created
// worktree. It must be committed, not left dirty, so ScratchCommit's
// "TASK.md is immutable" check (which inspects `git status`) sees it as
// clean, and so its hash survives independent of any later scratch commits.
func writeTaskFile(ctx context.Context, worktree string, t domain.Task) error {
	path := filepath.Join(worktree, "TASK.md")
	if _, err := os.Stat(path); err == nil {
		return nil
	}
	if err := os.WriteFile(path, continuity.RenderTaskFile(t), 0644); err != nil {
		return err
	}
	for _, args := range [][]string{{"add", "TASK.md"}, {"commit", "-m", "orchestra: TASK.md"}} {
		cmd := exec.CommandContext(ctx, "git", append([]string{"-C", worktree}, args...)...)
		if out, err := cmd.CombinedOutput(); err != nil {
			return fmt.Errorf("%s: %w", string(out), err)
		}
	}
	return nil
}

func (w GitWorktrees) Remove(ctx context.Context, _ domain.Task, path string) error {
	if path == "" {
		return fmt.Errorf("worktree: path required")
	}
	cmd := exec.CommandContext(ctx, "git", "-C", w.Repo, "worktree", "remove", "--force", path)
	if out, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("%s: %w", string(out), err)
	}
	return nil
}

// ProjectRepo is the minimal shape PerProjectGitWorktrees needs from a
// project's registry entry — kept local (not importing internal/registry)
// so orchestrator does not depend on registry's config-loading concerns.
type ProjectRepo struct {
	Repo         string
	WorktreeRoot string
}

// PerProjectGitWorktrees resolves a task's repo/root by its project (spec
// §2.2: each project is first-class and may have its own checkout), falling
// back to Default for any project not present in Projects — this keeps
// single-repo deployments working unchanged.
type PerProjectGitWorktrees struct {
	Projects    map[string]ProjectRepo
	Default     GitWorktrees
	TaskFileSHA string
}

func (w PerProjectGitWorktrees) Spec(t domain.Task) (string, string, bool) {
	g := w.Default
	if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" {
		g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo}
	}
	return g.Spec(t)
}

func (w PerProjectGitWorktrees) Create(ctx context.Context, t domain.Task) (string, error) {
	g := w.Default
	if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" {
		g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo, TaskFileSHA: w.TaskFileSHA}
	}
	if g.TaskFileSHA == "" {
		g.TaskFileSHA = w.TaskFileSHA
	}
	return g.Create(ctx, t)
}

func (w PerProjectGitWorktrees) Remove(ctx context.Context, t domain.Task, path string) error {
	g := w.Default
	if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" {
		g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo}
	}
	return g.Remove(ctx, t, path)
}

type AdapterFactory struct{ Herdrs map[string]herdr.Adapter }

func (f AdapterFactory) Adapter(id string) (herdr.Adapter, error) {
	a, ok := f.Herdrs[id]
	if !ok {
		return nil, fmt.Errorf("adapter %q not registered", id)
	}
	return a, nil
}

type Coordinator struct {
	Store     *store.Store
	Worktrees Worktrees
	Adapters  Adapters
	StatePath string
	mu        sync.Mutex
	sessions  map[string]herdr.Session
	loaded    bool
	healthMu  sync.RWMutex
	health    MonitorHealth
}

type MonitorHealth struct {
	Running   bool      `json:"running"`
	LastRun   time.Time `json:"last_run"`
	LastError string    `json:"last_error,omitempty"`
	Expired   int       `json:"expired"`
	// TurnBoundaryDegraded counts rotation ticks where Face B (spec §5.2,
	// §5.3 — "the Stop-hook/Face-B decides rotation, not the router") could
	// not be consulted, so occupancy-only thresholding is standing in. This
	// must stay observable rather than a silent fallback: an operator (or
	// the brief) can see when a deployment's rotation safety is degraded.
	TurnBoundaryDegraded int                      `json:"turn_boundary_degraded"`
	Sessions             map[string]SessionHealth `json:"sessions,omitempty"`
}

type SessionHealth struct {
	Status             string    `json:"status,omitempty"`
	WaitingForApproval bool      `json:"waiting_for_approval"`
	Blocker            string    `json:"blocker,omitempty"`
	UpdatedAt          time.Time `json:"updated_at"`
	LastError          string    `json:"last_error,omitempty"`
	// Occupancy and OccupancyError make the number rotation actually decides
	// on observable (spec §5.2.1: verify this against a live session before
	// trusting it). A resolution/read failure is recorded here rather than
	// silently treated as "not time to rotate yet" by a bare continue.
	Occupancy      float64 `json:"occupancy,omitempty"`
	OccupancyError string  `json:"occupancy_error,omitempty"`
}

// adapterFor resolves the herdr adapter for a session. Session.HerdrID (the
// registered herdr instance id, e.g. "homesrv-claude") is authoritative;
// Session.Harness (the harness kind, e.g. "claude") is only a fallback for
// sessions persisted before HerdrID was tracked. Adapters are keyed by
// instance id, so falling back to the lease's harness id (recorded on the
// task) rather than the kind keeps this resolvable even then.
func (c *Coordinator) adapterFor(taskID string, session herdr.Session) (herdr.Adapter, error) {
	id := session.HerdrID
	if id == "" {
		if task, ok := c.Store.Task(taskID); ok && task.Lease != nil {
			id = task.Lease.HarnessID
		}
	}
	if id == "" {
		id = session.Harness
	}
	return c.Adapters.Adapter(id)
}

func (c *Coordinator) MonitorHealth() MonitorHealth {
	c.healthMu.RLock()
	defer c.healthMu.RUnlock()
	return c.health
}
func (c *Coordinator) setMonitorHealth(err error, expired int) {
	c.healthMu.Lock()
	defer c.healthMu.Unlock()
	c.health.Running = err == nil
	c.health.LastRun = time.Now().UTC()
	c.health.Expired += expired
	if err != nil {
		c.health.LastError = err.Error()
	} else {
		c.health.LastError = ""
	}
}

func (c *Coordinator) recordTurnBoundaryDegraded() {
	c.healthMu.Lock()
	defer c.healthMu.Unlock()
	c.health.TurnBoundaryDegraded++
}

func waitingForApproval(status string) bool {
	s := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(status, "-", "_"), " ", "_"))
	return s == "waiting_for_approval" || s == "awaiting_approval" || s == "approval_required"
}

func (c *Coordinator) refreshSessionHealth(ctx context.Context) {
	c.loadSessions()
	c.mu.Lock()
	sessions := make(map[string]herdr.Session, len(c.sessions))
	for id, s := range c.sessions {
		sessions[id] = s
	}
	c.mu.Unlock()
	c.healthMu.Lock()
	if c.health.Sessions == nil {
		c.health.Sessions = map[string]SessionHealth{}
	}
	c.healthMu.Unlock()
	for taskID, session := range sessions {
		a, err := c.adapterFor(taskID, session)
		if err != nil {
			continue
		}
		var h SessionHealth
		h.UpdatedAt = time.Now().UTC()
		if occ, occErr := a.Occupancy(session); occErr != nil {
			h.OccupancyError = occErr.Error()
		} else {
			h.Occupancy = occ
		}
		p, ok := a.(herdr.AgentStatus)
		if !ok {
			c.healthMu.Lock()
			c.health.Sessions[taskID] = h
			c.healthMu.Unlock()
			continue
		}
		status, err := p.AgentStatus(ctx, session)
		h.Status = status
		h.WaitingForApproval = waitingForApproval(status)
		if err != nil {
			h.LastError = err.Error()
		} else if blocker, ok := a.(herdr.AgentBlocker); ok && strings.EqualFold(status, "blocked") {
			h.Blocker, _ = blocker.AgentBlocker(ctx, session)
		}
		c.healthMu.Lock()
		c.health.Sessions[taskID] = h
		c.healthMu.Unlock()
	}
}

func (c *Coordinator) loadSessions() {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.loaded {
		return
	}
	c.loaded = true
	c.sessions = map[string]herdr.Session{}
	if c.StatePath == "" {
		return
	}
	b, err := os.ReadFile(c.StatePath)
	if err != nil {
		return
	}
	_ = json.Unmarshal(b, &c.sessions)
}

func (c *Coordinator) saveSessionsLocked() error {
	if c.StatePath == "" {
		return nil
	}
	b, err := json.Marshal(c.sessions)
	if err != nil {
		return err
	}
	tmp := c.StatePath + ".tmp"
	if err = os.WriteFile(tmp, b, 0600); err != nil {
		return err
	}
	return os.Rename(tmp, c.StatePath)
}

// Reconcile drops mappings whose task lease did not survive restart and kills
// their recoverable herdr sessions so an orphan cannot keep consuming a slot.
func (c *Coordinator) Reconcile(ctx context.Context) error {
	c.loadSessions()
	c.mu.Lock()
	for taskID, session := range c.sessions {
		t, ok := c.Store.Task(taskID)
		if ok && t.State == domain.StateLeased {
			continue
		}
		if a, err := c.adapterFor(taskID, session); err == nil {
			_ = a.Kill(ctx, session)
		}
		delete(c.sessions, taskID)
	}
	err := c.saveSessionsLocked()
	c.mu.Unlock()
	if err != nil {
		return err
	}
	// Session health is derived state. Rebuild it immediately from the
	// durable session mappings so a restart does not hide an outstanding
	// approval until the first periodic monitor tick.
	c.refreshSessionHealth(ctx)
	return nil
}

// Monitor performs conservative hard-threshold rotation. The adapter owns
// the handoff creation; the coordinator only publishes its content address
// and frees the lease for pickup by the router.
func (c *Coordinator) Monitor(ctx context.Context, hard float64, interval time.Duration) error {
	if err := c.Reconcile(ctx); err != nil {
		c.setMonitorHealth(err, 0)
		return err
	}
	if interval <= 0 {
		interval = 30 * time.Second
	}
	t := time.NewTicker(interval)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			c.healthMu.Lock()
			c.health.Running = false
			c.healthMu.Unlock()
			return ctx.Err()
		case <-t.C:
			c.refreshSessionHealth(ctx)
			c.cleanupCompleted(ctx)
			expired, err := c.expire(ctx)
			c.setMonitorHealth(err, len(expired))
			if err != nil {
				continue
			}
			c.rotate(ctx, hard)
		}
	}
}

func (c *Coordinator) cleanupCompleted(ctx context.Context) {
	cleaner, ok := c.Worktrees.(WorktreeCleaner)
	if !ok {
		return
	}
	c.loadSessions()
	c.mu.Lock()
	defer c.mu.Unlock()
	changed := false
	for taskID, session := range c.sessions {
		t, exists := c.Store.Task(taskID)
		if !exists || t.State != domain.StateCompleted {
			continue
		}
		if err := cleaner.Remove(ctx, t, session.Worktree); err != nil {
			c.healthMu.Lock()
			c.health.LastError = "worktree cleanup: " + err.Error()
			c.healthMu.Unlock()
			continue
		}
		delete(c.sessions, taskID)
		changed = true
	}
	if changed {
		_ = c.saveSessionsLocked()
	}
}

func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
	// pane.exited is the low-latency path; lease expiry below remains the
	// authoritative backstop when herdr misses an exit notification.
	c.loadSessions()
	c.mu.Lock()
	for taskID, s := range c.sessions {
		if t, ok := c.Store.Task(taskID); ok && t.State == domain.StateLeased {
			if a, ae := c.adapterFor(taskID, s); ae == nil {
				if p, ok := a.(herdr.PaneExit); ok {
					if exited, ee := p.PaneExited(ctx, s); ee == nil && exited {
						b, _ := json.Marshal(map[string]string{"reason": "pane_exited", "harness_id": s.Harness})
						_ = c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
					}
				}
			}
		}
	}
	c.mu.Unlock()
	events, err := c.Store.ExpireLeases(time.Now())
	if err != nil {
		return events, err
	}
	for _, e := range events {
		c.loadSessions()
		c.mu.Lock()
		s, ok := c.sessions[e.TaskID]
		delete(c.sessions, e.TaskID)
		if ok {
			if a, ae := c.adapterFor(e.TaskID, s); ae == nil {
				_ = a.Kill(ctx, s)
			}
		}
		_ = c.saveSessionsLocked()
		c.mu.Unlock()
	}
	return events, nil
}
func (c *Coordinator) rotate(ctx context.Context, hard float64) {
	c.loadSessions()
	c.mu.Lock()
	sessions := make(map[string]herdr.Session, len(c.sessions))
	for id, s := range c.sessions {
		sessions[id] = s
	}
	c.mu.Unlock()
	for taskID, session := range sessions {
		task, ok := c.Store.Task(taskID)
		if !ok || task.State != domain.StateLeased {
			continue
		}
		a, err := c.adapterFor(taskID, session)
		if err != nil {
			continue
		}
		reason := "threshold"
		occupancy, err := a.Occupancy(session)
		if err != nil || occupancy < hard {
			continue
		}
		// Face B is treated as required, not best-effort (spec §5.2/§5.3):
		// an adapter that supports the turn-boundary probe but fails to
		// answer it blocks this tick's release rather than silently
		// proceeding as if mid-turn interruption were safe. Only an
		// adapter that genuinely does not implement TurnBoundary at all
		// falls back to occupancy-only thresholding, and that fallback is
		// recorded so it is observable (MonitorHealth.TurnBoundaryDegraded)
		// instead of invisible.
		if boundary, ok := a.(herdr.TurnBoundary); ok {
			atBoundary, boundaryErr := boundary.AtTurnBoundary(ctx, session)
			if boundaryErr != nil {
				c.recordTurnBoundaryDegraded()
				continue
			}
			if !atBoundary {
				continue
			}
		} else {
			c.recordTurnBoundaryDegraded()
		}
		if requester, ok := a.(herdr.HandoffRequester); ok {
			if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffFile)); statErr != nil {
				if !session.HandoffRequested {
					if reqErr := requester.RequestHandoff(ctx, session); reqErr == nil {
						session.HandoffRequested = true
						c.mu.Lock()
						c.sessions[taskID] = session
						_ = c.saveSessionsLocked()
						c.mu.Unlock()
					}
				}
				continue
			}
		}
		ref, err := a.Release(ctx, session)
		if err != nil {
			continue
		}
		if ref == "" {
			continue
		}
		anchorSHA, err := herdr.HeadSHA(session.Worktree)
		if err != nil {
			// Cannot certify the anchor: do not release with an invalid
			// TaskReleased payload (it would fail validation and strand
			// the lease/session). Leave the lease intact for the next
			// tick or TTL expiry to reclaim.
			continue
		}
		b, _ := json.Marshal(map[string]string{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA})
		e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b, Surface: string(authz.System)}
		if c.Store.Append(e) == nil {
			c.mu.Lock()
			delete(c.sessions, taskID)
			_ = c.saveSessionsLocked()
			c.mu.Unlock()
		}
	}
}

func (c *Coordinator) Start(ctx context.Context, e domain.Event) error {
	if e.Type != "TaskLeased" {
		return nil
	}
	if c.Store == nil || c.Worktrees == nil || c.Adapters == nil {
		return fmt.Errorf("orchestrator: dependencies required")
	}
	c.loadSessions()
	t, ok := c.Store.Task(e.TaskID)
	if !ok {
		return domain.ErrNotFound
	}
	var p struct {
		HarnessID  string `json:"harness_id"`
		HandoffRef string `json:"handoff_ref"`
	}
	if err := json.Unmarshal(e.Payload, &p); err != nil || p.HarnessID == "" {
		return fmt.Errorf("orchestrator: invalid lease")
	}
	a, err := c.Adapters.Adapter(p.HarnessID)
	if err != nil {
		return c.block(t, "adapter: "+err.Error())
	}
	var w string
	if creator, ok := a.(herdr.WorktreeCreator); ok {
		planner, planned := c.Worktrees.(WorktreeSpec)
		if !planned {
			return c.block(t, "worktree: repository specification unavailable")
		}
		repo, root, valid := planner.Spec(t)
		if !valid {
			return c.block(t, "worktree: repository and root required")
		}
		w, err = creator.CreateWorktree(ctx, repo, root, t.ID)
	} else {
		w, err = c.Worktrees.Create(ctx, t)
	}
	if err != nil {
		return c.block(t, "worktree: "+err.Error())
	}
	// Best-effort: TASK.md only exists for worktrees this process can read
	// locally (the GitWorktrees path). A herdr-hosted worktree on a remote
	// machine (WorktreeCreator path) is the same cross-host gap named in
	// AUDIT.md's federation-fork section — not solved here.
	taskFileSHA, _ := continuity.TaskFileHash(w)
	s, err := a.Lease(ctx, t.ID, w)
	if err != nil {
		return c.block(t, "lease: "+err.Error())
	}
	if p.HandoffRef != "" {
		// §6.2 pickup validation: never bootstrap a successor onto a handoff
		// whose anchor/dirty-file/TASK.md hashes don't match what's actually
		// in the worktree. A failure here blocks the task rather than
		// silently trusting an unvalidated ref (this is the gap AUDIT.md's
		// B6 named as unreached from the live path).
		h, err := continuity.Load(p.HandoffRef, c.Store)
		if err != nil {
			_ = a.Kill(ctx, s)
			return c.block(t, "handoff: "+err.Error())
		}
		if err := continuity.ValidatePickup(w, h, taskFileSHA); err != nil {
			_ = a.Kill(ctx, s)
			return c.block(t, "pickup: "+err.Error())
		}
		if err = a.Bootstrap(ctx, s, p.HandoffRef); err != nil {
			_ = a.Kill(ctx, s)
			return c.block(t, "bootstrap: "+err.Error())
		}
	}
	s.HerdrID = p.HarnessID
	s.TaskFileSHA = taskFileSHA
	c.mu.Lock()
	if c.sessions == nil {
		c.sessions = map[string]herdr.Session{}
	}
	c.sessions[t.ID] = s
	err = c.saveSessionsLocked()
	c.mu.Unlock()
	c.healthMu.Lock()
	if c.health.Sessions == nil {
		c.health.Sessions = map[string]SessionHealth{}
	}
	c.health.Sessions[t.ID] = SessionHealth{Status: "running", UpdatedAt: time.Now().UTC()}
	c.healthMu.Unlock()
	return err
}

func (c *Coordinator) block(t domain.Task, reason string) error {
	b, _ := json.Marshal(map[string]string{"blocker": reason})
	return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskBlocked", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)})
}

func (c *Coordinator) Session(taskID string) (herdr.Session, bool) {
	c.loadSessions()
	c.mu.Lock()
	defer c.mu.Unlock()
	s, ok := c.sessions[taskID]
	return s, ok
}

func (c *Coordinator) Capture(ctx context.Context, taskID, source string) (string, error) {
	s, ok := c.Session(taskID)
	if !ok {
		return "", domain.ErrNotFound
	}
	id := s.HerdrID
	if id == "" {
		if t, ok := c.Store.Task(taskID); ok && t.Lease != nil {
			id = t.Lease.HarnessID
		}
	}
	a, err := c.Adapters.Adapter(id)
	if err != nil {
		return "", err
	}
	p, ok := a.(herdr.PaneCapture)
	if !ok {
		return "", fmt.Errorf("pane capture unsupported")
	}
	return p.PaneCapture(ctx, s, source)
}