// Package orchestrator connects router lease events to an opaque herdr // session. It is deliberately small: scheduling remains in router and the // adapter remains the only component that knows how to drive a harness. package orchestrator import ( "context" "encoding/json" "errors" "fmt" "orchestra/internal/authz" "orchestra/internal/continuity" "orchestra/internal/domain" "orchestra/internal/herdr" "orchestra/internal/store" "os" "os/exec" "path/filepath" "strings" "sync" "time" ) type Worktrees interface { Create(context.Context, domain.Task) (string, error) } type WorktreeSpec interface { Spec(domain.Task) (string, string, bool) } type WorktreeCleaner interface { Remove(context.Context, domain.Task, string) error } type Adapters interface { Adapter(string) (herdr.Adapter, error) } // GitWorktrees creates one isolated checkout per task. The root is expected // to be a clone containing the project's remote; callers may set a separate // root per deployment. type GitWorktrees struct { Root string Repo string TaskFileSHA string } func (w GitWorktrees) Spec(domain.Task) (string, string, bool) { return w.Repo, w.Root, w.Repo != "" && w.Root != "" } func (w GitWorktrees) Create(ctx context.Context, t domain.Task) (string, error) { if w.Root == "" || w.Repo == "" { return "", fmt.Errorf("worktree: root and repo required") } if err := os.MkdirAll(w.Root, 0755); err != nil { return "", err } p := filepath.Join(w.Root, t.ID) if _, err := os.Stat(p); err == nil { if w.TaskFileSHA != "" { if err := continuity.VerifyTaskFile(p, w.TaskFileSHA); err != nil { return "", err } } return p, nil } branch := "orchestra/" + t.ID cmd := exec.CommandContext(ctx, "git", "-C", w.Repo, "worktree", "add", "-b", branch, p, "HEAD") if out, err := cmd.CombinedOutput(); err != nil { return "", fmt.Errorf("%s: %w", string(out), err) } if err := writeTaskFile(ctx, p, t); err != nil { return "", err } if w.TaskFileSHA != "" { if err := continuity.VerifyTaskFile(p, w.TaskFileSHA); err != nil { return "", err } } return p, nil } // writeTaskFile commits the §6.2 immutable TASK.md into a freshly created // worktree. It must be committed, not left dirty, so ScratchCommit's // "TASK.md is immutable" check (which inspects `git status`) sees it as // clean, and so its hash survives independent of any later scratch commits. func writeTaskFile(ctx context.Context, worktree string, t domain.Task) error { path := filepath.Join(worktree, "TASK.md") if _, err := os.Stat(path); err == nil { return nil } if err := os.WriteFile(path, continuity.RenderTaskFile(t), 0644); err != nil { return err } for _, args := range [][]string{{"add", "TASK.md"}, {"commit", "-m", "orchestra: TASK.md"}} { cmd := exec.CommandContext(ctx, "git", append([]string{"-C", worktree}, args...)...) if out, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("%s: %w", string(out), err) } } return nil } func (w GitWorktrees) Remove(ctx context.Context, _ domain.Task, path string) error { if path == "" { return fmt.Errorf("worktree: path required") } cmd := exec.CommandContext(ctx, "git", "-C", w.Repo, "worktree", "remove", "--force", path) if out, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("%s: %w", string(out), err) } return nil } // ProjectRepo is the minimal shape PerProjectGitWorktrees needs from a // project's registry entry — kept local (not importing internal/registry) // so orchestrator does not depend on registry's config-loading concerns. type ProjectRepo struct { Repo string WorktreeRoot string } // PerProjectGitWorktrees resolves a task's repo/root by its project (spec // §2.2: each project is first-class and may have its own checkout), falling // back to Default for any project not present in Projects — this keeps // single-repo deployments working unchanged. type PerProjectGitWorktrees struct { Projects map[string]ProjectRepo Default GitWorktrees TaskFileSHA string } func (w PerProjectGitWorktrees) Spec(t domain.Task) (string, string, bool) { g := w.Default if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" { g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo} } return g.Spec(t) } func (w PerProjectGitWorktrees) Create(ctx context.Context, t domain.Task) (string, error) { g := w.Default if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" { g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo, TaskFileSHA: w.TaskFileSHA} } if g.TaskFileSHA == "" { g.TaskFileSHA = w.TaskFileSHA } return g.Create(ctx, t) } func (w PerProjectGitWorktrees) Remove(ctx context.Context, t domain.Task, path string) error { g := w.Default if p, ok := w.Projects[t.Project]; ok && p.Repo != "" && p.WorktreeRoot != "" { g = GitWorktrees{Root: p.WorktreeRoot, Repo: p.Repo} } return g.Remove(ctx, t, path) } type AdapterFactory struct{ Herdrs map[string]herdr.Adapter } func (f AdapterFactory) Adapter(id string) (herdr.Adapter, error) { a, ok := f.Herdrs[id] if !ok { return nil, fmt.Errorf("adapter %q not registered", id) } return a, nil } type Coordinator struct { Store *store.Store Worktrees Worktrees Adapters Adapters StatePath string // LocalHerdr, when set, is the coordinator's machine-ownership boundary. // A coordinator must never operate a pane or checkout owned by another // machine; federation workers own those operations locally. LocalHerdr func(string) bool mu sync.Mutex sessions map[string]herdr.Session loaded bool healthMu sync.RWMutex health MonitorHealth // Hard is the occupancy threshold Monitor's periodic rotate() runs // against, mirrored here so TurnDecision (the synchronous, per-turn // counterpart driven by the Face-B stop hook) evaluates the same // threshold rather than needing its own copy passed in by the caller. Hard float64 // Soft is the advisory occupancy threshold (spec §5.3: "soft ~55% // threshold") at which TurnDecision starts asking the agent to prepare a // handoff — non-blocking, doesn't require a turn boundary — well before // Hard forces one. Zero means "use the package default" (see // defaultSoft), so existing callers that never set this field keep // working unchanged. Soft float64 // Thrash tunes DetectThrash's three circuit breakers (§5.3). Zero-value // fields fall back to herdr's own defaults, so leaving this unset works. Thrash herdr.ThrashConfig } // defaultSoft is used whenever Coordinator.Soft is unset (zero value). const defaultSoft = 0.55 func (c *Coordinator) soft() float64 { if c.Soft > 0 { return c.Soft } return defaultSoft } // checkActivityTriggers is S11's milestone/thrash pair: given an adapter that // implements herdr.ActivityReader, read its tool-call history and evaluate // both detectors. thrash takes priority (a circuit breaker overrides a // coherent-looking commit), same as the caller would want either way since // only one handoff request happens per tick. Returns the reason to request // ("thrash"/"milestone") and its dead ends, or "" if neither fired or the // adapter has no activity source at all — the latter is not degraded-and- // recorded the way TurnBoundary's absence is, since these two triggers are // additive on top of threshold/manual rotation, not a required safety gate. func checkActivityTriggers(ctx context.Context, a herdr.Adapter, session herdr.Session, cfg herdr.ThrashConfig) (reason string, deadEnds []continuity.DeadEnd) { reader, ok := a.(herdr.ActivityReader) if !ok { return "", nil } calls, err := reader.Activity(ctx, session) if err != nil { return "", nil } if thrash, de := herdr.DetectThrash(calls, cfg); thrash { return "thrash", de } if herdr.DetectMilestone(calls) { return "milestone", nil } return "", nil } // requestReasonedHandoff is the shared "ask once, remember we asked" wiring // checkActivityTriggers' two callers (rotate, TurnDecision) both need — same // HandoffRequested guard the occupancy-driven HandoffRequester path already // uses, so a repeated thrash/milestone detection on later ticks doesn't // reprompt every time before the agent has finished writing the file. func (c *Coordinator) requestReasonedHandoff(ctx context.Context, taskID string, session herdr.Session, a herdr.Adapter, reason string, deadEnds []continuity.DeadEnd) { if session.HandoffRequested { return } if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffReportFile)); statErr == nil { return } requester, ok := a.(herdr.ReasonedHandoffRequester) if !ok { return } if err := requester.RequestHandoffReason(ctx, session, reason, deadEnds); err != nil { return } session.HandoffRequested = true session.HandoffReason = reason c.mu.Lock() c.sessions[taskID] = session _ = c.saveSessionsLocked() c.mu.Unlock() } type MonitorHealth struct { Running bool `json:"running"` LastRun time.Time `json:"last_run"` LastError string `json:"last_error,omitempty"` Expired int `json:"expired"` // TurnBoundaryDegraded counts rotation ticks where Face B (spec §5.2, // §5.3 — "the Stop-hook/Face-B decides rotation, not the router") could // not be consulted, so occupancy-only thresholding is standing in. This // must stay observable rather than a silent fallback: an operator (or // the brief) can see when a deployment's rotation safety is degraded. TurnBoundaryDegraded int `json:"turn_boundary_degraded"` Sessions map[string]SessionHealth `json:"sessions,omitempty"` } type SessionHealth struct { Status string `json:"status,omitempty"` WaitingForApproval bool `json:"waiting_for_approval"` Blocker string `json:"blocker,omitempty"` UpdatedAt time.Time `json:"updated_at"` LastError string `json:"last_error,omitempty"` // Occupancy and OccupancyError make the number rotation actually decides // on observable (spec §5.2.1: verify this against a live session before // trusting it). A resolution/read failure is recorded here rather than // silently treated as "not time to rotate yet" by a bare continue. Occupancy float64 `json:"occupancy,omitempty"` OccupancyError string `json:"occupancy_error,omitempty"` } // adapterFor resolves the herdr adapter for a session. Session.HerdrID (the // registered herdr instance id, e.g. "homesrv-claude") is authoritative; // Session.Harness (the harness kind, e.g. "claude") is only a fallback for // sessions persisted before HerdrID was tracked. Adapters are keyed by // instance id, so falling back to the lease's harness id (recorded on the // task) rather than the kind keeps this resolvable even then. func (c *Coordinator) adapterFor(taskID string, session herdr.Session) (herdr.Adapter, error) { id := session.HerdrID if id == "" { if task, ok := c.Store.Task(taskID); ok && task.Lease != nil { id = task.Lease.HarnessID } } if id == "" { id = session.Harness } if c.LocalHerdr != nil && !c.LocalHerdr(id) { return nil, fmt.Errorf("session %s is owned by non-local herdr %s", taskID, id) } return c.Adapters.Adapter(id) } func (c *Coordinator) MonitorHealth() MonitorHealth { c.healthMu.RLock() defer c.healthMu.RUnlock() return c.health } func (c *Coordinator) setMonitorHealth(err error, expired int) { c.healthMu.Lock() defer c.healthMu.Unlock() c.health.Running = err == nil c.health.LastRun = time.Now().UTC() c.health.Expired += expired if err != nil { c.health.LastError = err.Error() } else { c.health.LastError = "" } } func (c *Coordinator) recordTurnBoundaryDegraded() { c.healthMu.Lock() defer c.healthMu.Unlock() c.health.TurnBoundaryDegraded++ } func waitingForApproval(status string) bool { s := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(status, "-", "_"), " ", "_")) return s == "waiting_for_approval" || s == "awaiting_approval" || s == "approval_required" } func (c *Coordinator) refreshSessionHealth(ctx context.Context) { c.loadSessions() c.mu.Lock() sessions := make(map[string]herdr.Session, len(c.sessions)) for id, s := range c.sessions { sessions[id] = s } c.mu.Unlock() c.healthMu.Lock() if c.health.Sessions == nil { c.health.Sessions = map[string]SessionHealth{} } c.healthMu.Unlock() for taskID, session := range sessions { a, err := c.adapterFor(taskID, session) if err != nil { continue } var h SessionHealth h.UpdatedAt = time.Now().UTC() if occ, occErr := a.Occupancy(session); occErr != nil { h.OccupancyError = occErr.Error() } else { h.Occupancy = occ } p, ok := a.(herdr.AgentStatus) if !ok { c.healthMu.Lock() c.health.Sessions[taskID] = h c.healthMu.Unlock() continue } status, err := p.AgentStatus(ctx, session) h.Status = status h.WaitingForApproval = waitingForApproval(status) if err != nil { h.LastError = err.Error() } else if blocker, ok := a.(herdr.AgentBlocker); ok && strings.EqualFold(status, "blocked") { h.Blocker, _ = blocker.AgentBlocker(ctx, session) } c.healthMu.Lock() c.health.Sessions[taskID] = h c.healthMu.Unlock() } } func (c *Coordinator) loadSessions() { c.mu.Lock() defer c.mu.Unlock() if c.loaded { return } c.loaded = true c.sessions = map[string]herdr.Session{} if c.StatePath == "" { return } b, err := os.ReadFile(c.StatePath) if err != nil { return } _ = json.Unmarshal(b, &c.sessions) } func (c *Coordinator) saveSessionsLocked() error { if c.StatePath == "" { return nil } b, err := json.Marshal(c.sessions) if err != nil { return err } tmp := c.StatePath + ".tmp" f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) if err != nil { return err } if _, err = f.Write(b); err == nil { err = f.Sync() } if closeErr := f.Close(); err == nil { err = closeErr } if err != nil { _ = os.Remove(tmp) return err } if err = os.Rename(tmp, c.StatePath); err != nil { return err } dir, err := os.Open(filepath.Dir(c.StatePath)) if err != nil { return err } defer dir.Close() return dir.Sync() } // Reconcile drops mappings whose task lease did not survive restart and kills // their recoverable herdr sessions so an orphan cannot keep consuming a slot. func (c *Coordinator) Reconcile(ctx context.Context) error { c.loadSessions() c.mu.Lock() for taskID, session := range c.sessions { t, ok := c.Store.Task(taskID) if ok && (t.State == domain.StateLeased || t.State == domain.StateNeedsAttention) { continue } if a, err := c.adapterFor(taskID, session); err == nil { _ = a.Kill(ctx, session) } delete(c.sessions, taskID) } err := c.saveSessionsLocked() c.mu.Unlock() if err != nil { return err } // Session health is derived state. Rebuild it immediately from the // durable session mappings so a restart does not hide an outstanding // approval until the first periodic monitor tick. c.refreshSessionHealth(ctx) return nil } // Monitor performs conservative hard-threshold rotation. The adapter owns // the handoff creation; the coordinator only publishes its content address // and frees the lease for pickup by the router. func (c *Coordinator) Monitor(ctx context.Context, hard float64, interval time.Duration) error { if err := c.Reconcile(ctx); err != nil { c.setMonitorHealth(err, 0) return err } c.Hard = hard if interval <= 0 { interval = 30 * time.Second } t := time.NewTicker(interval) defer t.Stop() for { select { case <-ctx.Done(): c.healthMu.Lock() c.health.Running = false c.healthMu.Unlock() return ctx.Err() case <-t.C: c.refreshSessionHealth(ctx) c.cleanupCompleted(ctx) c.checkConventions(ctx) expired, err := c.expire(ctx) c.setMonitorHealth(err, len(expired)) if err != nil { continue } c.rotate(ctx, hard) } } } func (c *Coordinator) cleanupCompleted(ctx context.Context) { cleaner, ok := c.Worktrees.(WorktreeCleaner) if !ok { return } c.loadSessions() c.mu.Lock() defer c.mu.Unlock() changed := false for taskID, session := range c.sessions { t, exists := c.Store.Task(taskID) if !exists || t.State != domain.StateCompleted { continue } if err := cleaner.Remove(ctx, t, session.Worktree); err != nil { c.healthMu.Lock() c.health.LastError = "worktree cleanup: " + err.Error() c.healthMu.Unlock() continue } delete(c.sessions, taskID) changed = true } if changed { _ = c.saveSessionsLocked() } } // checkConventions is §6.3: "on update, the orchestra injects a notice to // agents whose current task is adjacent" — adjacency here is "same project's // base repo," and staleness is tracked by comparing each session's own // last-known continuity.ConventionsHash against the base repo's current one, // never by trusting the agent to notice on its own. func (c *Coordinator) checkConventions(ctx context.Context) { spec, ok := c.Worktrees.(WorktreeSpec) if !ok { return } c.loadSessions() c.mu.Lock() sessions := make(map[string]herdr.Session, len(c.sessions)) for id, s := range c.sessions { sessions[id] = s } c.mu.Unlock() changed := false for taskID, session := range sessions { t, ok := c.Store.Task(taskID) if !ok || t.State != domain.StateLeased { continue } repo, _, valid := spec.Spec(t) if !valid { continue } hash, err := continuity.ConventionsHash(repo) if err != nil || hash == session.ConventionsHash { continue } a, err := c.adapterFor(taskID, session) if err != nil { continue } notifier, ok := a.(herdr.ConventionsNotifier) if !ok { continue } if err := notifier.NotifyConventionsChanged(ctx, session); err != nil { continue } session.ConventionsHash = hash c.mu.Lock() c.sessions[taskID] = session c.mu.Unlock() changed = true } if changed { c.mu.Lock() _ = c.saveSessionsLocked() c.mu.Unlock() } } func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) { // pane.exited is the low-latency path; lease expiry below remains the // authoritative backstop when herdr misses an exit notification. c.loadSessions() c.mu.Lock() for taskID, s := range c.sessions { if t, ok := c.Store.Task(taskID); ok && (t.State == domain.StateLeased || t.State == domain.StateNeedsAttention) { if a, ae := c.adapterFor(taskID, s); ae == nil { if p, ok := a.(herdr.PaneExit); ok { if exited, ee := p.PaneExited(ctx, s); ee == nil && exited { b, _ := json.Marshal(map[string]any{"reason": "pane_exited", "harness_id": s.Harness, "lease_epoch": t.Lease.Epoch, "expected_version": t.Version}) _ = c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)}) } } } } } c.mu.Unlock() var events []domain.Event var firstErr error for _, task := range c.Store.Tasks() { if (task.State != domain.StateLeased && task.State != domain.StateNeedsAttention) || task.Lease == nil || task.Lease.Until.After(time.Now()) { continue } // Stop a local predecessor before making its lease eligible for a // successor. If this cannot be done, keep both the mapping and the // lease: safety beats reclaim speed. c.mu.Lock() s, local := c.sessions[task.ID] c.mu.Unlock() if local { a, adapterErr := c.adapterFor(task.ID, s) if adapterErr != nil { if firstErr == nil { firstErr = fmt.Errorf("expire %s: resolve old pane: %w", task.ID, adapterErr) } continue } if killErr := a.Kill(ctx, s); killErr != nil { if firstErr == nil { firstErr = fmt.Errorf("expire %s: quarantine old pane: %w", task.ID, killErr) } continue } c.mu.Lock() delete(c.sessions, task.ID) _ = c.saveSessionsLocked() c.mu.Unlock() } e, expireErr := c.Store.ExpireLease(task.ID, time.Now()) if expireErr != nil { if !errors.Is(expireErr, domain.ErrConflict) && firstErr == nil { firstErr = expireErr } continue } events = append(events, e) } return events, firstErr } // handoffReason reads HandoffFile from the worktree, if present, and returns // its meta.reason ("threshold|milestone|thrash|manual" per §6.1). An unread­ // able or invalid file returns "" — callers treat that as "no signal yet", // never as "manual". func handoffReason(worktree string) string { b, err := os.ReadFile(filepath.Join(worktree, herdr.HandoffFile)) if err != nil { return "" } h, err := continuity.Decode(b) if err != nil { return "" } return h.Meta.Reason } func (c *Coordinator) rotate(ctx context.Context, hard float64) { c.loadSessions() c.mu.Lock() sessions := make(map[string]herdr.Session, len(c.sessions)) for id, s := range c.sessions { sessions[id] = s } c.mu.Unlock() for taskID, session := range sessions { task, ok := c.Store.Task(taskID) if !ok || task.State != domain.StateLeased { continue } a, err := c.adapterFor(taskID, session) if err != nil { continue } reason := "threshold" // Agent-initiated ROTATE, and the two orchestrator-detected triggers // (§5.3: manual / milestone / thrash) all short-circuit the same way // once a handoff carrying that reason already exists: the boundary // question has already been answered, so skip occupancy and the // turn-boundary probe and go straight to release. existingReason := handoffReason(session.Worktree) bypass := existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash" if bypass { reason = existingReason } else { if trigger, deadEnds := checkActivityTriggers(ctx, a, session, c.Thrash); trigger != "" { c.requestReasonedHandoff(ctx, taskID, session, a, trigger, deadEnds) continue } occupancy, err := a.Occupancy(session) if err != nil || occupancy < c.soft() { continue } if occupancy < hard { // Soft threshold (§5.3): request a handoff early, advisory // only — no release, no turn-boundary requirement. if requester, ok := a.(herdr.HandoffRequester); ok { if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffReportFile)); statErr != nil && !session.HandoffRequested { if reqErr := requester.RequestHandoff(ctx, session); reqErr == nil { session.HandoffRequested = true session.HandoffReason = "threshold" c.mu.Lock() c.sessions[taskID] = session _ = c.saveSessionsLocked() c.mu.Unlock() } } } continue } // Face B is treated as required, not best-effort (spec // §5.2/§5.3): an adapter that supports the turn-boundary probe // but fails to answer it blocks this tick's release rather than // silently proceeding as if mid-turn interruption were safe. // Only an adapter that genuinely does not implement // TurnBoundary at all falls back to occupancy-only // thresholding, and that fallback is recorded so it is // observable (MonitorHealth.TurnBoundaryDegraded) instead of // invisible. if boundary, ok := a.(herdr.TurnBoundary); ok { atBoundary, boundaryErr := boundary.AtTurnBoundary(ctx, session) if boundaryErr != nil { c.recordTurnBoundaryDegraded() continue } if !atBoundary { continue } } else { c.recordTurnBoundaryDegraded() } } if requester, ok := a.(herdr.HandoffRequester); ok { if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffReportFile)); statErr != nil { if !session.HandoffRequested { if reqErr := requester.RequestHandoff(ctx, session); reqErr == nil { session.HandoffRequested = true session.HandoffReason = reason c.mu.Lock() c.sessions[taskID] = session _ = c.saveSessionsLocked() c.mu.Unlock() } } continue } } ref, err := a.Release(ctx, session) if err != nil { // A release failure is operational state, not a silent retry. Keep // the fenced lease and pane for recovery while durably exposing the // failed phase to the worker and operator. _ = c.block(task, "rotation release: "+err.Error()) continue } if ref == "" { _ = c.block(task, "rotation release: empty handoff reference") continue } anchorSHA, err := herdr.HeadSHA(session.Worktree) if err != nil { // Cannot certify the anchor. Record the fault while retaining the // owner; an unseen bare continue used to leave this state opaque. _ = c.block(task, "rotation anchor: "+err.Error()) continue } b, _ := json.Marshal(map[string]any{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA, "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version}) e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b, Surface: string(authz.System)} if c.Store.Append(e) == nil { // A release only transfers the lease; this local coordinator owns // the predecessor pane until it has actually stopped it. if err := a.Kill(ctx, session); err == nil { c.mu.Lock() delete(c.sessions, taskID) _ = c.saveSessionsLocked() c.mu.Unlock() } } } } // Turn decision verdicts (spec §5.3, AUDIT.md Phase 2 items 1-2). These are // the only valid results of TurnDecision and the only values the // POST /v1/harness/turn endpoint may return. const ( TurnContinue = "continue" TurnPrepareHandoff = "prepare_handoff" TurnRotateNow = "rotate_now" TurnRefuse = "refuse" ) // TurnDecision evaluates a single leased task's rotation state synchronously, // at a harness-reported turn boundary, and acts on the result. It mirrors // rotate()'s per-task logic (occupancy → turn-boundary → handoff-file → // release) but is invoked once per turn from the Face-B stop hook instead of // on Monitor's ticker, so an agent that's about to stop gets an authoritative // answer instead of waiting for the next tick. `refuse` covers every case // where continuing to let the harness stop would be unsafe: the turn // boundary can't be verified, or release/anchor certification failed. func (c *Coordinator) TurnDecision(ctx context.Context, taskID string) (string, error) { c.loadSessions() c.mu.Lock() session, ok := c.sessions[taskID] c.mu.Unlock() if !ok { return "", fmt.Errorf("orchestrator: no session for task %q", taskID) } task, ok := c.Store.Task(taskID) if !ok || task.State != domain.StateLeased { return "", fmt.Errorf("orchestrator: task %q not leased", taskID) } a, err := c.adapterFor(taskID, session) if err != nil { return "", fmt.Errorf("orchestrator: adapter: %w", err) } // Agent-initiated ROTATE, and the two orchestrator-detected triggers // (§5.3: manual / milestone / thrash): a handoff already written with one // of these reasons is itself the boundary signal — skip occupancy and the // turn-boundary probe and release immediately. if existingReason := handoffReason(session.Worktree); existingReason == "manual" || existingReason == "milestone" || existingReason == "thrash" { return c.finishRelease(ctx, taskID, task, session, a, existingReason) } d := (RotationStateMachine{Soft: c.soft(), Hard: c.Hard, Thrash: c.Thrash}).Evaluate(ctx, a, session) if d.Degraded != nil { return "", fmt.Errorf("orchestrator: rotation: %w", d.Degraded) } if d.Action == TurnContinue { return TurnContinue, nil } if d.Action == TurnRefuse { return TurnRefuse, nil } if d.Reason == "milestone" || d.Reason == "thrash" { c.requestReasonedHandoff(ctx, taskID, session, a, d.Reason, d.DeadEnds) return TurnPrepareHandoff, nil } if d.Action == TurnPrepareHandoff { // Soft threshold (§5.3): advisory only. Ask the agent to start // preparing a handoff well before Hard forces one, but don't block // the turn on a boundary check — the agent is free to keep working. if requester, ok := a.(herdr.HandoffRequester); ok { if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffReportFile)); statErr != nil { if !session.HandoffRequested { if reqErr := requester.RequestHandoff(ctx, session); reqErr == nil { session.HandoffRequested = true session.HandoffReason = "threshold" c.mu.Lock() c.sessions[taskID] = session _ = c.saveSessionsLocked() c.mu.Unlock() } } } } return TurnPrepareHandoff, nil } if requester, ok := a.(herdr.HandoffRequester); ok { if _, statErr := os.Stat(filepath.Join(session.Worktree, herdr.HandoffReportFile)); statErr != nil { if !session.HandoffRequested { if reqErr := requester.RequestHandoff(ctx, session); reqErr == nil { session.HandoffRequested = true session.HandoffReason = "threshold" c.mu.Lock() c.sessions[taskID] = session _ = c.saveSessionsLocked() c.mu.Unlock() } } return TurnPrepareHandoff, nil } } return c.finishRelease(ctx, taskID, task, session, a, "threshold") } // finishRelease runs the common release tail shared by TurnDecision's // threshold path and its agent-initiated-ROTATE (reason=manual) shortcut: // call Adapter.Release, certify the anchor against the real worktree HEAD, // and emit TaskReleased. Any failure refuses rather than emitting a // TaskReleased payload that would fail validation and strand the session. func (c *Coordinator) finishRelease(ctx context.Context, taskID string, task domain.Task, session herdr.Session, a herdr.Adapter, reason string) (string, error) { ref, err := a.Release(ctx, session) if err != nil || ref == "" { return TurnRefuse, nil } anchorSHA, err := herdr.HeadSHA(session.Worktree) if err != nil { // Cannot certify the anchor: refuse rather than release with an // invalid TaskReleased payload, same as rotate()'s bare continue. return TurnRefuse, nil } b, _ := json.Marshal(map[string]any{"handoff_ref": ref, "reason": reason, "anchor_sha": anchorSHA, "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version}) e := domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: task.Version + 1, Payload: b, Surface: string(authz.System)} if err := c.Store.Append(e); err != nil { return TurnRefuse, nil } if err := a.Kill(ctx, session); err != nil { return TurnRefuse, nil } c.mu.Lock() delete(c.sessions, taskID) _ = c.saveSessionsLocked() c.mu.Unlock() return TurnRotateNow, nil } func (c *Coordinator) Start(ctx context.Context, e domain.Event) error { if e.Type != "TaskLeased" { return nil } if c.Store == nil || c.Worktrees == nil || c.Adapters == nil { return fmt.Errorf("orchestrator: dependencies required") } c.loadSessions() t, ok := c.Store.Task(e.TaskID) if !ok { return domain.ErrNotFound } var p struct { HarnessID string `json:"harness_id"` HandoffRef string `json:"handoff_ref"` } if err := json.Unmarshal(e.Payload, &p); err != nil || p.HarnessID == "" { return fmt.Errorf("orchestrator: invalid lease") } if c.LocalHerdr != nil && !c.LocalHerdr(p.HarnessID) { return c.block(t, "remote herdr must be operated by its federation worker") } a, err := c.Adapters.Adapter(p.HarnessID) if err != nil { return c.block(t, "adapter: "+err.Error()) } // Worktrees, including immutable TASK.md, are coordinator-local state. // A remote herdr must be driven by its federation worker instead of being // asked to create an opaque checkout that this coordinator cannot validate. w, err := c.Worktrees.Create(ctx, t) if err != nil { return c.block(t, "worktree: "+err.Error()) } taskFileSHA, _ := continuity.TaskFileHash(w) prompt := taskLaunchPrompt(t) var s herdr.Session if promptLeaser, ok := a.(herdr.PromptLeaser); ok { s, err = promptLeaser.LeasePrompt(ctx, t.ID, w, prompt) } else { s, err = a.Lease(ctx, t.ID, w) } if err != nil { // A UI-changing prompt can time out after herdr accepted it. Keep the // live pane mapped before recording TaskNeedsAttention so a later completion // can reconcile the lifecycle instead of becoming an orphan (B15). if s.PaneID != "" { s.HerdrID = p.HarnessID s.TaskFileSHA = taskFileSHA s.ConventionsHash, _ = continuity.ConventionsHash(w) _ = c.rememberSession(t.ID, s) } return c.block(t, "lease: "+err.Error()) } if p.HandoffRef != "" { // §6.2 pickup validation: never bootstrap a successor onto a handoff // whose anchor/dirty-file/TASK.md hashes don't match what's actually // in the worktree. A failure here blocks the task rather than // silently trusting an unvalidated ref (this is the gap AUDIT.md's // B6 named as unreached from the live path). h, err := continuity.Load(p.HandoffRef, c.Store) if err != nil { _ = a.Kill(ctx, s) return c.block(t, "handoff: "+err.Error()) } if err := continuity.ValidatePickup(w, h, taskFileSHA); err != nil { _ = a.Kill(ctx, s) return c.block(t, "pickup: "+err.Error()) } if err = a.Bootstrap(ctx, s, p.HandoffRef); err != nil { _ = a.Kill(ctx, s) return c.block(t, "bootstrap: "+err.Error()) } } s.HerdrID = p.HarnessID s.TaskFileSHA = taskFileSHA // Best-effort, same caveat as taskFileSHA above: only meaningful for a // worktree this process can read locally. Snapshots the shared-docs // state this session starts trusting; checkConventions notices drift // from here, not from whatever the agent's own cached view is (§6.3). s.ConventionsHash, _ = continuity.ConventionsHash(w) err = c.rememberSession(t.ID, s) c.healthMu.Lock() if c.health.Sessions == nil { c.health.Sessions = map[string]SessionHealth{} } c.health.Sessions[t.ID] = SessionHealth{Status: "running", UpdatedAt: time.Now().UTC()} c.healthMu.Unlock() return err } func taskLaunchPrompt(t domain.Task) string { var b strings.Builder fmt.Fprintf(&b, "Begin Orchestra task %s.\n", t.ID) if t.Title != "" { fmt.Fprintf(&b, "Title: %s\n", t.Title) } if t.Description != "" { fmt.Fprintf(&b, "Instructions:\n%s\n", t.Description) } else { b.WriteString("Inspect the repository, understand the task context, and proceed with the requested work.\n") } b.WriteString("This is the authoritative task instruction. Work only within this task's worktree. Do not edit TASK.md if it exists.") return b.String() } func (c *Coordinator) rememberSession(taskID string, s herdr.Session) error { c.mu.Lock() defer c.mu.Unlock() if c.sessions == nil { c.sessions = map[string]herdr.Session{} } c.sessions[taskID] = s return c.saveSessionsLocked() } func (c *Coordinator) block(t domain.Task, reason string) error { p := map[string]any{"blocker": reason, "block_reason": string(domain.InferBlockReason(reason)), "lifecycle_phase": "needs_attention", "last_error": reason, "pane_state": "unknown", "session_evidence": domain.SessionEvidence{PaneState: "unknown", Source: "coordinator", CheckedAt: time.Now().UTC()}} if s, ok := c.Session(t.ID); ok { p["pane_id"] = s.PaneID p["harness_id"] = s.HerdrID p["pane_state"] = "open" p["session_evidence"] = domain.SessionEvidence{PaneID: s.PaneID, HarnessID: s.HerdrID, PaneState: "open", Source: "coordinator", CheckedAt: time.Now().UTC()} } b, _ := json.Marshal(p) if t.Lease != nil { p["harness_id"] = t.Lease.HarnessID p["lease_epoch"] = t.Lease.Epoch p["expected_version"] = t.Version b, _ = json.Marshal(p) } return c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskNeedsAttention", TaskID: t.ID, Version: t.Version + 1, Payload: b, Surface: string(authz.System)}) } func (c *Coordinator) Session(taskID string) (herdr.Session, bool) { c.loadSessions() c.mu.Lock() defer c.mu.Unlock() s, ok := c.sessions[taskID] return s, ok } // RequestHandoff asks the live harness to prepare its agent-authored handoff. // It deliberately does not release the pane: a later validated handoff is the // only evidence that can make a rotation safe. func (c *Coordinator) RequestHandoff(ctx context.Context, taskID string) error { c.loadSessions() c.mu.Lock() s, ok := c.sessions[taskID] c.mu.Unlock() if !ok { return fmt.Errorf("session not found for task %s", taskID) } if s.HandoffRequested { return nil } a, err := c.adapterFor(taskID, s) if err != nil { return err } req, ok := a.(herdr.HandoffRequester) if !ok { return fmt.Errorf("harness does not support handoff requests") } if err := req.RequestHandoff(ctx, s); err != nil { return err } s.HandoffRequested = true c.mu.Lock() c.sessions[taskID] = s err = c.saveSessionsLocked() c.mu.Unlock() return err } // RespondApproval is the local implementation of the same guarded command // contract used by federation workers. It rechecks the displayed capture at // the owning herdr immediately before input is sent. func (c *Coordinator) RespondApproval(ctx context.Context, taskID string, grant bool, expectedCapture string) error { c.loadSessions() c.mu.Lock() s, ok := c.sessions[taskID] c.mu.Unlock() if !ok { return fmt.Errorf("session not found for task %s", taskID) } a, err := c.adapterFor(taskID, s) if err != nil { return err } responder, ok := a.(herdr.ApprovalResponder) if !ok { return fmt.Errorf("harness does not support approval responses") } return responder.RespondApproval(ctx, s, grant, expectedCapture) } func (c *Coordinator) Capture(ctx context.Context, taskID, source string) (string, error) { s, ok := c.Session(taskID) if !ok { return "", domain.ErrNotFound } id := s.HerdrID if id == "" { if t, ok := c.Store.Task(taskID); ok && t.Lease != nil { id = t.Lease.HarnessID } } a, err := c.Adapters.Adapter(id) if err != nil { return "", err } p, ok := a.(herdr.PaneCapture) if !ok { return "", fmt.Errorf("pane capture unsupported") } return p.PaneCapture(ctx, s, source) }