// orchestra-worker consumes router-issued leases for one or more local // execution backends. Each declared harness is a separate federation identity // with its own token, cursor, backend, and state file, because the coordinator // authorizes a lease call by comparing the URL's worker id against the lease's // harness id. Homesrv remains the scheduler and CAS authority. This process // owns only local Git and pane operations. package main import ( "context" "encoding/json" "errors" "fmt" "log" "net/http" "orchestra/internal/agentctx" "orchestra/internal/buildinfo" "orchestra/internal/continuity" "orchestra/internal/domain" "orchestra/internal/federation" "orchestra/internal/herdr" "orchestra/internal/orchestrator" "orchestra/internal/review" "orchestra/internal/workphase" "os" "os/exec" "os/signal" "path/filepath" "reflect" "sort" "strconv" "strings" "syscall" "time" ) type worker struct { api federation.Client backend herdr.Backend // herdr is retained as a test/backward-compatibility alias. Production // workers set backend; executionBackend keeps older state-machine tests // from needing protocol-irrelevant rewrites. herdr *herdr.Client harnessID, harness, repo, root, remote string projects map[string]projectConfig cursor uint64 tasks map[string]domain.Task sessions map[string]herdr.Session leases map[string]lease releases map[string]releaseTransaction quarantined map[string]bool statePath string hard float64 registration federation.Worker lastError string lastErrorAt time.Time soft float64 window int64 } func (w *worker) executionBackend() herdr.Backend { if w.backend != nil { return w.backend } if w.herdr != nil { return w.herdr } return nil } func (w *worker) recordError(err error) { if err == nil { return } w.lastError = err.Error() w.lastErrorAt = time.Now().UTC() } func (w *worker) health(ctx context.Context) federation.WorkerHealth { h := federation.WorkerHealth{HerdrStatus: "unknown"} if backend := w.executionBackend(); backend != nil { h.Backend = backend.Kind() } for taskID, session := range w.sessions { // Workers currently advertise capacity one. Pick deterministically so a // recovered legacy state with more sessions remains intelligible. if h.ActiveTask == "" || taskID < h.ActiveTask { h.ActiveTask, h.ActivePane = taskID, session.PaneID } } if backend := w.executionBackend(); backend != nil { checkCtx, cancel := context.WithTimeout(ctx, 3*time.Second) err := backend.Check(checkCtx) cancel() h.CheckedAt = time.Now().UTC() if err == nil { h.HerdrStatus = "reachable" } else { h.HerdrStatus = "unreachable" w.recordError(fmt.Errorf("local %s backend: %w", backend.Kind(), err)) } } h.LastError, h.ErrorAt = w.lastError, w.lastErrorAt return h } type lease struct { Epoch string `json:"epoch"` HandoffRef string `json:"handoff_ref,omitempty"` TransactionID string `json:"transaction_id,omitempty"` AnchorSHA string `json:"anchor_sha,omitempty"` PickupAcknowledged bool `json:"pickup_acknowledged,omitempty"` Version int `json:"version"` Until time.Time `json:"until"` UsageBaseline float64 `json:"usage_baseline,omitempty"` // ProgressSHA hashes the pane capture taken at the last renewal. Renewal // requires the pane to have changed since then, or the agent to be busy. ProgressSHA string `json:"progress_sha,omitempty"` } type releaseTransaction struct { ID string `json:"id"` LeaseVersion int `json:"lease_version"` Ref string `json:"handoff_ref,omitempty"` AnchorSHA string `json:"anchor_sha,omitempty"` Phase string `json:"phase"` // prepared, anchor_pushed, event_committed, pickup_validated, predecessor_retired AgentReleased bool `json:"agent_released,omitempty"` LastError string `json:"last_error,omitempty"` UpdatedAt time.Time `json:"updated_at"` } type projectConfig struct { Repo string `json:"repo"` Root string `json:"worktree_root"` Remote string `json:"remote"` QualityGate string `json:"quality_gate,omitempty"` SafeOperations []string `json:"safe_operations,omitempty"` } type completionEvidence struct { TaskID string `json:"task_id"` Project string `json:"project"` Worker string `json:"worker"` Harness string `json:"harness"` PaneID string `json:"pane_id"` BaseSHA string `json:"base_sha"` ResultSHA string `json:"result_sha"` Branch string `json:"branch"` Remote string `json:"remote"` QualityGate string `json:"quality_gate,omitempty"` GateExit int `json:"gate_exit"` GateOutput string `json:"gate_output,omitempty"` CompletedAt time.Time `json:"completed_at"` } // tail keeps the end of a gate log, which is where the failure is. func tail(s string, max int) string { if len(s) <= max { return s } return s[len(s)-max:] } type workerState struct { Cursor uint64 `json:"cursor"` Sessions map[string]herdr.Session `json:"sessions"` Tasks map[string]domain.Task `json:"tasks"` Leases map[string]lease `json:"leases"` Releases map[string]releaseTransaction `json:"releases"` Quarantined map[string]bool `json:"quarantined,omitempty"` } func (w *worker) load() error { b, e := os.ReadFile(w.statePath) if e == nil { var s workerState if err := json.Unmarshal(b, &s); err != nil { return fmt.Errorf("corrupt worker state %s: %w", w.statePath, err) } w.cursor = s.Cursor w.sessions = s.Sessions w.tasks = s.Tasks w.leases = s.Leases w.releases = s.Releases w.quarantined = s.Quarantined } else if !errors.Is(e, os.ErrNotExist) { return fmt.Errorf("read worker state %s: %w", w.statePath, e) } if w.sessions == nil { w.sessions = map[string]herdr.Session{} } if w.tasks == nil { w.tasks = map[string]domain.Task{} } if w.leases == nil { w.leases = map[string]lease{} } if w.releases == nil { w.releases = map[string]releaseTransaction{} } if w.quarantined == nil { w.quarantined = map[string]bool{} } return nil } func (w *worker) save() error { b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases, Releases: w.releases, Quarantined: w.quarantined}) if e != nil { return e } if e := os.MkdirAll(filepath.Dir(w.statePath), 0700); e != nil { return e } tmp := w.statePath + ".tmp" f, e := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) if e != nil { return e } if _, e = f.Write(b); e == nil { e = f.Sync() } if closeErr := f.Close(); e == nil { e = closeErr } if e != nil { _ = os.Remove(tmp) return e } if e = os.Rename(tmp, w.statePath); e != nil { return e } dir, e := os.Open(filepath.Dir(w.statePath)) if e != nil { return e } defer dir.Close() return dir.Sync() } // quarantine stops a pane before its lost lease mapping can be forgotten. // A failed close remains durable and is retried; it is never treated as a // harmless cleanup error while the old harness could still be working. func (w *worker) quarantine(ctx context.Context, taskID string, s herdr.Session) { if w.executionBackend() == nil { w.quarantined[taskID] = true return } if err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).Kill(ctx, s); err != nil { w.quarantined[taskID] = true w.recordError(fmt.Errorf("quarantine %s: %w", taskID, err)) return } delete(w.sessions, taskID) delete(w.quarantined, taskID) } func (w *worker) retryQuarantines(ctx context.Context) { for taskID := range w.quarantined { if s, ok := w.sessions[taskID]; ok { w.quarantine(ctx, taskID, s) } } } type artifactCAS struct{ api federation.Client } func (c artifactCAS) PutArtifact(b []byte) (string, error) { return c.api.PutArtifact(context.Background(), b) } func (c artifactCAS) Artifact(ref string) ([]byte, error) { return c.api.Artifact(context.Background(), ref) } func created(e domain.Event) (domain.Task, bool) { if e.Type != "TaskCreated" { return domain.Task{}, false } var p struct { Source string `json:"source"` ExternalID string `json:"external_id"` Project string `json:"project"` Capability []string `json:"capability"` Title string `json:"title"` Description string `json:"description"` Acceptance []string `json:"acceptance"` QualityGate string `json:"quality_gate"` } if json.Unmarshal(e.Payload, &p) != nil || p.Source == "" || p.ExternalID == "" || p.Project == "" { return domain.Task{}, false } return domain.Task{ID: e.TaskID, Source: p.Source, ExternalID: p.ExternalID, Project: p.Project, Capability: p.Capability, Title: p.Title, Description: p.Description, Acceptance: p.Acceptance, QualityGate: p.QualityGate}, true } func (w *worker) project(t domain.Task) (projectConfig, error) { if w.projects != nil { if p, ok := w.projects[t.Project]; ok && p.Repo != "" && p.Root != "" && p.Remote != "" { return p, nil } return projectConfig{}, fmt.Errorf("project %q is not configured on worker", t.Project) } return projectConfig{Repo: w.repo, Root: w.root, Remote: w.remote}, nil } func (w *worker) syncBase(ctx context.Context, p projectConfig) error { if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil { return fmt.Errorf("fetch base checkout: %s: %w", out, err) } branch, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "symbolic-ref", "--quiet", "--short", "HEAD").Output() if err != nil { return fmt.Errorf("identify base branch: %w", err) } branchName := strings.TrimSpace(string(branch)) if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "merge", "--ff-only", p.Remote+"/"+branchName).CombinedOutput(); err != nil { return fmt.Errorf("fast-forward base checkout: %s: %w", out, err) } return nil } func (w *worker) start(ctx context.Context, t domain.Task, ref string) error { var wt string var h continuity.Handoff var err error p, err := w.project(t) if err != nil { return err } // Synchronize the local base before any worktree operation. A worker never // treats a coordinator-side path as truth; the Git remote is the only // cross-machine transport. if err := w.syncBase(ctx, p); err != nil { return err } if ref != "" { b, err := w.api.Artifact(ctx, ref) if err != nil { return err } h, err = continuity.Decode(b) if err != nil { return err } if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil { return fmt.Errorf("fetch pickup anchor: %s: %w", out, err) } wt = filepath.Join(p.Root, t.ID) if _, err := os.Stat(wt); os.IsNotExist(err) { if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "worktree", "add", "-b", "orchestra/"+t.ID, wt, h.Anchor.GitSHA).CombinedOutput(); err != nil { return fmt.Errorf("create pickup worktree: %s: %w", out, err) } } if err = continuity.ValidatePickup(wt, h, taskHash(t)); err != nil { return err } } else { wt, err = (orchestrator.GitWorktrees{Repo: p.Repo, Root: p.Root}).Create(ctx, t) if err != nil { return err } } backend := w.executionBackend() if backend == nil { return fmt.Errorf("execution backend is not configured") } if _, err = backend.Worktree(ctx, p.Repo, wt, "orchestra/"+t.ID); err != nil { return err } s, err := backend.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID) if err != nil { return err } // The phase this session was launched to run. A later phase change makes // this session's context the wrong one, which is what rotates it (F22). s.Phase = string(currentPhase(t)) s.TaskFileSHA = taskHash(t) if w.harness == "claude" { s.ContextHandoffSHA, _ = fileSHA256(filepath.Join(wt, "HANDOFF.md")) } // One renderer, on both machines. The worker fetches the reduced // authority as data and renders it with agentctx, so a decision the human // recorded before this session existed is visible from its first turn. intent, err := w.api.Intent(ctx, t.ID) if err != nil { return fmt.Errorf("effective intent: %w", err) } in := agentctx.Input{ Task: t, Intent: intent, Phase: t.WorkPhase, DecisionRequest: t.DecisionRequest, Git: agentctx.GitState{Worktree: wt, Branch: "orchestra/" + t.ID}, RepoRules: agentctx.DiscoverRepoRules(wt), } if sha, shaErr := herdr.HeadSHA(wt); shaErr == nil { in.Git.HeadSHA = sha } if ref != "" { in.Handoff = &h } if len(p.SafeOperations) > 0 { in.Policy = []string{ "Permitted without an operator grant, inside this worktree only: " + strings.Join(p.SafeOperations, ", ") + ".", "Network access, secrets, destructive actions, and paths outside this worktree require an explicit operator approval.", } } if t.ResearchRef != "" { b, artErr := w.api.Artifact(ctx, t.ResearchRef) if artErr != nil { return fmt.Errorf("research artifact: %w", artErr) } r, decErr := workphase.DecodeStoredResearch(b) if decErr != nil { return fmt.Errorf("research artifact: %w", decErr) } in.Research = &r } if t.PlanRef != "" { b, artErr := w.api.Artifact(ctx, t.PlanRef) if artErr != nil { return fmt.Errorf("plan artifact: %w", artErr) } pl, decErr := workphase.DecodeStoredPlan(b) if decErr != nil { return fmt.Errorf("plan artifact: %w", decErr) } in.Plan = &pl } if t.Review != nil { b, artErr := w.api.Artifact(ctx, t.Review.ArtifactRef) if artErr != nil { return fmt.Errorf("review artifact: %w", artErr) } r, decErr := review.Decode(b) if decErr != nil { return fmt.Errorf("review artifact: %w", decErr) } in.Review = &r } built, err := agentctx.Build(in) if err != nil { return fmt.Errorf("build context: %w", err) } prompt := built.System + "\n\n" + built.Task // The transport decides what is submitted, never what the agent receives: // the file holds the exact bytes agentctx rendered either way. submitted, transport := prompt, herdr.LaunchInline if lt, ok := backend.(herdr.LaunchTransporter); ok { transport = lt.LaunchTransport(w.harness) } if transport == herdr.LaunchFileRef { submitted = herdr.LaunchReference } if writeErr := herdr.WriteLaunchContext(s.Worktree, prompt); writeErr != nil { // Under LaunchFileRef the file is the instruction, so a failed write // is a failed launch rather than lost evidence. if transport == herdr.LaunchFileRef { return fmt.Errorf("launch context %s: %w", t.ID, writeErr) } w.recordError(fmt.Errorf("launch context %s: %w", t.ID, writeErr)) } // The launch instruction carried these, so the first turn boundary must // not re-announce them as news. for _, d := range intent.Decisions { s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID) } w.sessions[t.ID] = s if err := w.save(); err != nil { return err } // A prompt response can be lost after the backend accepted it. Persist the // session first so the worker can reconcile/release it after restart. if err := backend.Prompt(ctx, s.PaneID, submitted, 0); err != nil { return err } // Acknowledging a launch means the harness accepted the instruction, not // that the adapter call returned nil. Without this the worker reported a // started agent while the prompt sat unsubmitted in the input editor. if c, ok := backend.(herdr.InputConfirmer); ok { evidence, confirmErr := c.ConfirmInput(ctx, s, submitted) if confirmErr != nil { // An unsubmitted prompt leaves a live pane that nothing owns, and // a retained session would make the retry skip this task // entirely. Reclaim both so the released lease can be re-leased. if killErr := backend.Kill(ctx, s); killErr != nil { w.recordError(fmt.Errorf("kill unlaunched pane %s: %w", t.ID, killErr)) } delete(w.sessions, t.ID) if saveErr := w.save(); saveErr != nil { w.recordError(fmt.Errorf("save after failed launch %s: %w", t.ID, saveErr)) } return fmt.Errorf("launch %s: %w", t.ID, confirmErr) } log.Printf("launch %s confirmed: %s", t.ID, evidence) } // Baseline the progress check at launch, not at the first renewal. The // renewal gate exempts a lease with no baseline, which handed a pane that // opened and never started a full free renewal period (F34) — the exact // case the gate exists to catch. Capturing here costs one pane read and // makes the first renewal a real comparison. if l, ok := w.leases[t.ID]; ok && l.ProgressSHA == "" { if text, progressErr := w.paneProgress(ctx, herdr.CLIAdapter{Backend: backend, Harness: w.harness}, s); progressErr == nil { l.ProgressSHA = domain.Hash([]byte(text)) w.leases[t.ID] = l } else { w.recordError(fmt.Errorf("baseline launch progress %s: %w", t.ID, progressErr)) } } if l, ok := w.leases[t.ID]; ok { if err := w.api.Start(ctx, t.ID, l.Epoch, l.Version, w.sessionEvidence(ctx, t.ID, s)); err != nil { return fmt.Errorf("ack start: %w", err) } l.Version++ w.leases[t.ID] = l if err := w.save(); err != nil { return err } } if ref != "" { return w.ackPickup(ctx, t.ID, s) } return nil } func classifyLaunchError(err error, sessionStarted bool) string { // Positive evidence that the harness never accepted the prompt is not // uncertainty. Release the lease so the existing retry path can take it. if errors.Is(err, herdr.ErrPromptNotSubmitted) { return "prompt_not_submitted" } if sessionStarted { // A prompt response can be lost after herdr accepted it. Never reclaim // that pane just because its acknowledgement was uncertain. return "launch_uncertain" } text := strings.ToLower(err.Error()) if strings.Contains(text, "handoff") || strings.Contains(text, "pickup") || strings.Contains(text, "task.md") { return "invalid_handoff" } return "launch_transient" } func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) } func fileSHA256(path string) (string, error) { b, err := os.ReadFile(path) if err != nil { return "", err } return domain.Hash(b), nil } func (w *worker) releaseReady(ctx context.Context) { for id, s := range w.sessions { if w.quarantined[id] { continue } if l := w.leases[id]; l.HandoffRef != "" && !l.PickupAcknowledged { if err := w.ackPickup(ctx, id, s); err != nil { w.recordError(err) continue } } if _, err := os.Stat(filepath.Join(s.Worktree, ".orchestra", "done")); err == nil { // A done marker is an intent, not enough on its own: do not race a // still-running native harness into committing half-written work. status, statusErr := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).AgentStatus(ctx, s) if statusErr != nil { w.recordError(fmt.Errorf("completion identity %s: %w", id, statusErr)) continue } if herdr.IsBusy(status) { w.recordError(fmt.Errorf("completion %s deferred: agent status %s", id, status)) continue } evidence, err := w.finalize(ctx, id, s) if err != nil { w.recordError(fmt.Errorf("complete %s: %w", id, err)) log.Printf("complete %s: %v", id, err) continue } report, _ := json.Marshal(evidence) ref, err := w.api.PutArtifact(ctx, report) if err != nil { w.recordError(fmt.Errorf("upload completion %s: %w", id, err)) log.Printf("upload completion %s: %v", id, err) continue } outcome, err := w.submit(ctx, id, s, evidence) if errors.Is(err, errReviewArtifact) { // Only the reviewing agent can fix this, and it is still alive // to be told. Dropping the done marker stops the five-second // retry and makes the corrected file the thing that finishes. w.recordError(fmt.Errorf("submit %s: %w", id, err)) w.answerRefusedReview(ctx, id, s, err) continue } if err != nil { w.recordError(fmt.Errorf("submit %s: %w", id, err)) log.Printf("submit %s: %v", id, err) continue } if outcome == federation.SubmitChangesRequested { // The sealed review sent the work back. The task is in // implement again, so the done marker is stale and the phase // change rotates this session on the next tick. _ = os.Remove(filepath.Join(s.Worktree, ".orchestra", "done")) log.Printf("review returned %s to implementation", id) continue } if outcome == federation.SubmitNoPublisher { if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Epoch, w.leases[id].Version, w.usageReceipt(s, w.leases[id]), w.sessionEvidence(ctx, id, s)); err != nil { w.recordError(fmt.Errorf("complete %s: %w", id, err)) log.Printf("complete %s: %v", id, err) continue } } // Completion is durable before closing the exact pane. If close // fails, retain the session mapping for a later explicit cleanup. a := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness} if err := a.Kill(ctx, s); err != nil { w.recordError(fmt.Errorf("close completed pane %s: %w", id, err)) log.Printf("close completed pane %s: %v", id, err) continue } _ = os.Remove(filepath.Join(s.Worktree, ".orchestra", "done")) _ = os.Remove(filepath.Join(s.Worktree, ".orchestra")) delete(w.sessions, id) delete(w.leases, id) _ = w.save() continue } if _, err := os.Stat(filepath.Join(s.Worktree, herdr.HandoffReportFile)); err == nil || w.releases[id].ID != "" { w.advanceRelease(ctx, id, s) continue } w.rotationTick(ctx, id, s) } } func (w *worker) adapter(s herdr.Session, remote string) herdr.CLIAdapter { a := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness, Window: w.window, CAS: artifactCAS{w.api}, Remote: remote} switch w.harness { case "claude": a.Usage = herdr.ClaudeUsage case "codex": a.Usage = herdr.CodexUsage case "opencode": a.Usage = herdr.OpenCodeUsage } return a } // rotationTick is the checkout-owner state machine. Occupancy, tool activity, // and pane status are all read from the persisted harness session identity; // any unknown source is recorded and never treated as zero usage. func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) { // Claude Code owns its context threshold through the installed // context-handoff hook. A changed HANDOFF.md means that hook has landed a // durable local continuation. Resume in the same process with Claude's // native context reset instead of manufacturing Orchestra's cross-worker // release artifact. Codex and OpenCode continue through the existing // occupancy/release state machine below. if w.harness == "claude" { if err := w.advanceClaudeContextReset(ctx, id, s); err != nil { w.recordError(fmt.Errorf("Claude context reset %s: %w", id, err)) } // A turn boundary is not a rotation. Claude owns its context rollover // through the installed hook, which is why the occupancy state machine // below is skipped, but phase requests and human decisions are carried // at the boundary and returning here left both unreachable on this // harness. Every decision recorded against a live Claude session went // undelivered, and no phase request could ever be read. t, ok := w.tasks[id] if !ok { w.recordError(fmt.Errorf("turn boundary %s: task cache missing", id)) return } p, err := w.project(t) if err != nil { w.recordError(err) return } w.federatedTurn(ctx, id, w.adapter(s, p.Remote), orchestrator.TurnContinue) return } t, ok := w.tasks[id] if !ok { w.recordError(fmt.Errorf("rotation %s: task cache missing", id)) return } p, err := w.project(t) if err != nil { w.recordError(err) return } a := w.adapter(s, p.Remote) resolved, err := a.ResolveSessionIdentity(s) if err != nil { w.recordError(fmt.Errorf("rotation %s occupancy degraded: %w", id, err)) return } // DeepEqual, not !=: Session carries a slice since decisions are tracked // per session, so it is no longer comparable with ==. if !reflect.DeepEqual(resolved, s) { w.sessions[id] = resolved s = resolved _ = w.save() } d := (orchestrator.RotationStateMachine{Soft: w.soft, Hard: w.hard}).Evaluate(ctx, a, s) if d.ActivityDegraded != nil { w.recordError(fmt.Errorf("rotation %s activity degraded: %w", id, d.ActivityDegraded)) } if d.Degraded != nil { w.recordError(fmt.Errorf("rotation %s degraded: %w", id, d.Degraded)) if d.Action == orchestrator.TurnContinue || d.Action == "" { return } } if d.Action == orchestrator.TurnContinue || d.Action == "" { w.federatedTurn(ctx, id, a, orchestrator.TurnContinue) return } if d.Action == orchestrator.TurnRefuse || s.HandoffRequested { return } if d.Reason == "milestone" || d.Reason == "thrash" { if err := a.RequestHandoffReason(ctx, s, d.Reason, d.DeadEnds); err != nil { w.recordError(fmt.Errorf("rotation %s %s prompt: %w", id, d.Reason, err)) return } } else if err := a.RequestHandoff(ctx, s); err != nil { w.recordError(fmt.Errorf("rotation %s threshold prompt: %w", id, err)) return } s.HandoffRequested, s.HandoffReason = true, d.Reason w.sessions[id] = s _ = w.save() } func (w *worker) sendLine(ctx context.Context, s herdr.Session, line string) error { backend := w.executionBackend() if backend == nil { return fmt.Errorf("execution backend is not configured") } if err := backend.SendText(ctx, s, line); err != nil { return fmt.Errorf("send %q: %w", line, err) } if err := backend.SendKeys(ctx, s, []string{"ENTER"}); err != nil { return fmt.Errorf("submit %q: %w", line, err) } // A lost Enter here leaves the session mid-rollover with /clear sitting in // the editor, which is worse than a lost launch: nothing retries it. if err := w.confirmInput(ctx, s, line); err != nil { return fmt.Errorf("submit %q: %w", line, err) } return nil } func (w *worker) advanceClaudeContextReset(ctx context.Context, id string, s herdr.Session) error { backend := w.executionBackend() if backend == nil { return fmt.Errorf("execution backend is not configured") } if s.ContextResetSHA == "" { sha, err := fileSHA256(filepath.Join(s.Worktree, "HANDOFF.md")) if os.IsNotExist(err) { return nil } if err != nil { return fmt.Errorf("read HANDOFF.md: %w", err) } if sha == s.ContextHandoffSHA { return nil } status, err := backend.AgentStatus(ctx, s) if err != nil { return fmt.Errorf("confirm Claude stopped after handoff: %w", err) } if status != "idle" { return nil } s.ContextResetSHA = sha s.ContextResetPhase = "clear" w.sessions[id] = s if err := w.save(); err != nil { return err } } switch s.ContextResetPhase { case "clear": if err := w.sendLine(ctx, s, "/clear"); err != nil { return err } s.ContextResetPhase = "handoff" w.sessions[id] = s if err := w.save(); err != nil { return err } // /clear redraws Claude's input UI asynchronously. Give it a small, // bounded interval before submitting the new-session file mention. select { case <-ctx.Done(): return ctx.Err() case <-time.After(500 * time.Millisecond): } fallthrough case "handoff": if err := w.sendLine(ctx, s, "@HANDOFF.md"); err != nil { return err } s.ContextHandoffSHA = s.ContextResetSHA s.ContextResetSHA = "" s.ContextResetPhase = "" // Claude normally opens a fresh transcript for /clear. Force the next // observation to discover it instead of retaining the exhausted path. s.SessionFile = "" w.sessions[id] = s return w.save() default: return fmt.Errorf("unknown persisted context-reset phase %q", s.ContextResetPhase) } } func (w *worker) advanceRelease(ctx context.Context, id string, s herdr.Session) { t, ok := w.tasks[id] if !ok { w.recordError(fmt.Errorf("release %s: task cache missing", id)) return } p, err := w.project(t) if err != nil { w.recordError(fmt.Errorf("release %s: %w", id, err)) return } if w.releases == nil { w.releases = map[string]releaseTransaction{} } tx := w.releases[id] if tx.ID == "" { l, ok := w.leases[id] if !ok { w.recordError(fmt.Errorf("release %s: lease missing", id)) return } tx = releaseTransaction{ID: domain.NewID(), LeaseVersion: l.Version, Phase: "prepared", UpdatedAt: time.Now().UTC()} w.releases[id] = tx _ = w.save() } a := w.adapter(s, p.Remote) if tx.Phase == "prepared" { prepared, err := a.PrepareRelease(ctx, s) if err != nil { tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC() w.releases[id] = tx _ = w.save() w.recordError(fmt.Errorf("release %s prepare: %w", id, err)) return } tx.Ref, tx.AnchorSHA, tx.Phase, tx.LastError, tx.UpdatedAt = prepared.Ref, prepared.AnchorSHA, "anchor_pushed", "", time.Now().UTC() w.releases[id] = tx _ = w.save() } if tx.Phase == "anchor_pushed" { l := w.leases[id] if err := w.api.Release(ctx, id, tx.Ref, tx.AnchorSHA, tx.ID, l.Epoch, tx.LeaseVersion, w.sessionEvidence(ctx, id, s)); err != nil { tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC() w.releases[id] = tx _ = w.save() w.recordError(fmt.Errorf("release %s commit: %w", id, err)) return } tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC() w.releases[id] = tx _ = w.save() } if tx.Phase == "event_committed" && !tx.AgentReleased { if err := a.ReleaseAgent(ctx, s); err != nil { tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC() w.releases[id] = tx _ = w.save() w.recordError(fmt.Errorf("release %s release agent: %w", id, err)) return } tx.AgentReleased, tx.LastError, tx.UpdatedAt = true, "", time.Now().UTC() w.releases[id] = tx _ = w.save() } if tx.Phase == "pickup_validated" { if err := a.Kill(ctx, s); err != nil { tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC() w.releases[id] = tx _ = w.save() w.recordError(fmt.Errorf("release %s retire predecessor: %w", id, err)) return } _ = os.Remove(filepath.Join(s.Worktree, herdr.HandoffReportFile)) tx.Phase, tx.UpdatedAt = "predecessor_retired", time.Now().UTC() w.releases[id] = tx _ = w.save() delete(w.sessions, id) delete(w.releases, id) _ = w.save() } } // ackPickup retries the successor acknowledgement from persisted lease state. // It is safe after a lost response: the coordinator recognizes the exact // transaction/lease epoch as an idempotent pickup. func (w *worker) ackPickup(ctx context.Context, id string, s herdr.Session) error { l, ok := w.leases[id] if !ok || l.HandoffRef == "" || l.TransactionID == "" || l.AnchorSHA == "" { return fmt.Errorf("pickup %s: lease is missing its release transaction", id) } if l.PickupAcknowledged { return nil } if err := w.api.Pickup(ctx, id, l.HandoffRef, l.AnchorSHA, l.TransactionID, l.Epoch, l.Version, w.sessionEvidence(ctx, id, s)); err != nil { return fmt.Errorf("pickup %s acknowledgement: %w", id, err) } l.PickupAcknowledged = true l.Version++ // TaskPickupValidated increments the task version. w.leases[id] = l return w.save() } func (w *worker) sessionEvidence(ctx context.Context, taskID string, s herdr.Session) domain.SessionEvidence { e := domain.SessionEvidence{PaneID: s.PaneID, HarnessID: w.harnessID, PaneState: "open", Source: "worker", CheckedAt: time.Now().UTC()} text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, s, "recent") if err != nil { e.PaneState = "unreachable" return e } if capture, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: taskID, PaneID: s.PaneID, Text: text}); err == nil { e.CapturedAt = capture.At } return e } func (w *worker) usageReceipt(s herdr.Session, l lease) map[string]any { if s.SessionFile == "" && !(w.harness == "opencode" && s.SessionID != "") { return map[string]any{"harness_id": w.harnessID, "consumed": 0, "known": false, "error": "native usage identity missing"} } var usage herdr.Usage var err error switch w.harness { case "claude": usage, err = herdr.ClaudeUsage(s.SessionFile) case "codex": usage, err = herdr.CodexUsage(s.SessionFile) case "opencode": usage, err = herdr.OpenCodeSessionUsage(s.SessionID) } if err != nil { return map[string]any{"harness_id": w.harnessID, "consumed": 0, "known": false, "error": err.Error()} } delta := float64(usage.Numerator()) - l.UsageBaseline if delta < 0 { delta = 0 } return map[string]any{"harness_id": w.harnessID, "input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead, "cache_write_tokens": usage.CacheWrite, "consumed": delta, "lease_usage_delta": delta, "known": true} } func git(ctx context.Context, dir string, args ...string) ([]byte, error) { return exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...).CombinedOutput() } // finalize performs only mechanical delivery work. It never asks the harness // to narrate Git state, gates, or a report; those are generated from the // worker-owned checkout and then verified against the configured remote. func (w *worker) finalize(ctx context.Context, id string, s herdr.Session) (completionEvidence, error) { t, ok := w.tasks[id] if !ok { return completionEvidence{}, fmt.Errorf("task cache missing") } p, err := w.project(t) if err != nil { return completionEvidence{}, err } if s.TaskFileSHA != "" { if err := continuity.VerifyTaskFile(s.Worktree, s.TaskFileSHA); err != nil { return completionEvidence{}, fmt.Errorf("verify immutable TASK.md: %w", err) } } base, err := git(ctx, s.Worktree, "rev-parse", "HEAD") if err != nil { return completionEvidence{}, fmt.Errorf("base sha: %s: %w", base, err) } e := completionEvidence{TaskID: id, Project: t.Project, Worker: w.harnessID, Harness: w.harness, PaneID: s.PaneID, BaseSHA: strings.TrimSpace(string(base)), Remote: p.Remote, QualityGate: t.QualityGate, CompletedAt: time.Now().UTC()} if _, err := git(ctx, s.Worktree, "diff", "--quiet", "--", "TASK.md"); err != nil { return completionEvidence{}, errors.New("TASK.md was modified") } if out, err := git(ctx, s.Worktree, "add", "-A", "--", ".", stageExclude); err != nil { return completionEvidence{}, fmt.Errorf("stage result: %s: %w", out, err) } if _, err := git(ctx, s.Worktree, "diff", "--cached", "--quiet"); err != nil { if out, err := git(ctx, s.Worktree, "commit", "-m", "orchestra: complete "+id); err != nil { return completionEvidence{}, fmt.Errorf("commit result: %s: %w", out, err) } } sha, err := git(ctx, s.Worktree, "rev-parse", "HEAD") if err != nil { return completionEvidence{}, fmt.Errorf("result sha: %s: %w", sha, err) } e.ResultSHA = strings.TrimSpace(string(sha)) // The gate runs after the result commit, against a clean tree that is // byte-for-byte the commit being submitted. Running it first bound the // evidence to the pre-commit HEAD, which CheckSubmission rejects because // gate sha, review sha and head sha must be one commit. gateCommand := t.QualityGate if gateCommand == "" { gateCommand = p.QualityGate } e.QualityGate = gateCommand if gateCommand != "" { gate := exec.CommandContext(ctx, "sh", "-c", gateCommand) gate.Dir = s.Worktree out, gateErr := gate.CombinedOutput() e.GateOutput = tail(string(out), review.MaxGateOutputBytes) if gateErr != nil { e.GateExit = 1 return completionEvidence{}, fmt.Errorf("quality gate %q: %s: %w", gateCommand, out, gateErr) } } // The submission branch is derived, not the worktree's local name: the // coordinator computes "orchestra/" independently, and a pull // request can only be opened for a branch both halves name the same way. e.Branch = "orchestra/" + id if out, err := git(ctx, s.Worktree, "push", p.Remote, "HEAD:refs/heads/"+e.Branch); err != nil { return completionEvidence{}, fmt.Errorf("push result: %s: %w", out, err) } remote, err := git(ctx, s.Worktree, "ls-remote", p.Remote, "refs/heads/"+e.Branch) if err != nil || !strings.HasPrefix(string(remote), e.ResultSHA+"\t") { return completionEvidence{}, fmt.Errorf("verify pushed sha: got %q: %w", strings.TrimSpace(string(remote)), err) } return e, nil } // reviewFile is where the reviewing session leaves its findings. The reviewer // supplies findings and nothing else: the commit they are bound to is the // result commit the worker just made, which the agent cannot know and must not // assert. const reviewFile = "review.json" // errReviewArtifact is a refusal only the reviewing agent can fix. It is // separated from every other submission failure because recording it in worker // health alone would leave a live session being retried every five seconds // with nothing telling it what is wrong — the silent-loop shape this codebase // keeps producing (F39, F42). var errReviewArtifact = errors.New("review artifact") // answerRefusedReview tells the reviewing session why its finish was refused // and clears the done marker so a corrected artifact is what finishes the // phase. The refusal survives a failed send: the marker is dropped either way, // and the agent writes it again when it has written the file. func (w *worker) answerRefusedReview(ctx context.Context, id string, s herdr.Session, cause error) { text := "Orchestra refused your completion: " + cause.Error() + "\n\nWrite .orchestra/" + reviewFile + " as {\"findings\": [...]}, an empty list if you found nothing, then write .orchestra/done again. Do not set a commit sha." if err := w.sendPrompt(ctx, s, text); err != nil { w.recordError(fmt.Errorf("deliver review refusal %s: %w", id, err)) } if err := os.Remove(filepath.Join(s.Worktree, ".orchestra", "done")); err != nil { w.recordError(fmt.Errorf("clear done marker %s: %w", id, err)) } log.Printf("completion %s refused: %v", id, cause) } // submit seals the review and hands the result to the human through a pull // request. It is the only path from a reviewed change to TaskSubmitted; the // direct completion call remains for a project with no forge configured. func (w *worker) submit(ctx context.Context, id string, s herdr.Session, e completionEvidence) (string, error) { var result review.Result b, err := os.ReadFile(filepath.Join(s.Worktree, ".orchestra", reviewFile)) if err != nil { // An absent review file is not a pass. Submission needs a sealed // review, and inventing an empty one would launder "the reviewer wrote // nothing" into "the reviewer found nothing". return "", fmt.Errorf("%w: the review phase sealed no .orchestra/%s: %v", errReviewArtifact, reviewFile, err) } if err := json.Unmarshal(b, &result); err != nil { return "", fmt.Errorf("%w: .orchestra/%s is not valid JSON: %v", errReviewArtifact, reviewFile, err) } result.ResultSHA = e.ResultSHA if err := result.Validate(); err != nil { return "", fmt.Errorf("%w: .orchestra/%s: %v", errReviewArtifact, reviewFile, err) } l := w.leases[id] // The gate result is bound to the commit the gate ran against, which // finalize guarantees is the commit being submitted. A project with no // quality gate still produces a bound result: Passed() needs the sha. gate := domain.GateResult{Command: e.QualityGate, ExitCode: e.GateExit, SHA: e.ResultSHA, Output: e.GateOutput} return w.api.Submit(ctx, id, l.Epoch, l.Version, e.ResultSHA, e.Remote, result, gate) } // stageExclude keeps Orchestra's own control files out of the result commit. // It names the directory, not the marker inside it: .orchestra carries a // .gitignore of "*" (herdr/adapter.go), and naming an ignored file in a // pathspec makes git refuse the whole add with "The following paths are // ignored by one of your .gitignore files". F42, live on run 5: the review // agent wrote .orchestra/done, and every completion attempt failed on that // refusal, once every five seconds, with the task stuck in review. Excluding // the directory also holds for a worktree that has no inner .gitignore. const stageExclude = ":!.orchestra" func (w *worker) renewLeases(ctx context.Context) { if w.executionBackend() == nil { return } now := time.Now() for taskID, l := range w.leases { s, ok := w.sessions[taskID] if !ok || l.Version == 0 || l.Until.After(now.Add(domain.LeaseRenewAt)) { continue } adapter := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness} // Input lines are excluded: keystrokes arriving at a pane, from // Orchestra or from anyone else, are not the agent doing work. text, err := w.paneProgress(ctx, adapter, s) if err != nil { w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err)) continue } // A live pane is not progress. Renewing on pane existence alone let a // pane that opened and never started hold its lease forever, which is // what orphaned the July task once the launch itself had failed. status, err := adapter.AgentStatus(ctx, s) if err != nil { w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err)) continue } progress := domain.Hash([]byte(text)) switch { case herdr.IsBusy(status): case progress != l.ProgressSHA && l.ProgressSHA != "": case l.ProgressSHA == "": // First renewal has no baseline to compare against. Record one and // allow this renewal; the next one must show real movement. default: w.recordError(fmt.Errorf("lease %s not renewed: agent status %s and pane unchanged since the last renewal", taskID, status)) continue } if err := w.api.Renew(ctx, taskID, l.Epoch, l.Version, int(domain.LeaseTTL.Seconds())); err != nil { w.recordError(fmt.Errorf("renew lease %s: %w", taskID, err)) log.Printf("renew lease %s: %v", taskID, err) } else { // RenewLease appends one event. Retain that epoch locally until its // replay arrives so a release transaction uses the same version. l.Version++ l.Until = now.Add(domain.LeaseTTL) l.ProgressSHA = progress w.leases[taskID] = l _ = w.save() } } } // publishCaptures makes remote panes observable without allowing the // coordinator to touch their unix herdr socket. func (w *worker) publishCaptures(ctx context.Context) { for taskID, session := range w.sessions { text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, session, "recent") if err != nil { continue } if _, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: taskID, PaneID: session.PaneID, Text: text}); err != nil { log.Printf("publish capture %s: %v", taskID, err) } } } type approvalInput struct { Text string Keys []string } func approvalResponse(text, kind string) (approvalInput, bool) { low := strings.ToLower(text) // Never invent a keystroke. y/n prompts label both decisions directly. if strings.Contains(low, "[y/n]") || strings.Contains(low, "(y/n)") { if kind == "grant_approval" { return approvalInput{Text: "y\n"}, true } return approvalInput{Text: "n\n"}, true } // OpenCode's explicit selector states "Allow once Allow always Reject" // and "enter confirm". Send a real ENTER key, not a newline through // pane.send_text: OpenCode's selector does not treat the latter as input. // Enter is consequently a bounded one-time grant; // rejection would require unobservable selector navigation, so refuse it. if kind == "grant_approval" && strings.Contains(low, "allow once") && strings.Contains(low, "allow always") && strings.Contains(low, "reject") && strings.Contains(low, "enter confirm") { return approvalInput{Keys: []string{"ENTER"}}, true } return approvalInput{}, false } func (w *worker) runCommands(ctx context.Context) { commands, err := w.api.Commands(ctx) if err != nil { log.Printf("poll controls: %v", err) return } for _, command := range commands { session, ok := w.sessions[command.TaskID] if !ok || session.PaneID != command.PaneID { _ = w.api.ResolveCommand(ctx, command.ID, "stale", "session or pane changed") continue } text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, session, "recent") if err != nil { _ = w.api.ResolveCommand(ctx, command.ID, "rejected", "capture unavailable: "+err.Error()) continue } capture, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: command.TaskID, PaneID: session.PaneID, Text: text}) if err != nil { _ = w.api.ResolveCommand(ctx, command.ID, "rejected", "cannot publish capture: "+err.Error()) continue } if capture.Revision != command.CaptureRevision { _ = w.api.ResolveCommand(ctx, command.ID, "stale", "capture revision changed") continue } if command.Kind == "resubmit" { // Not an approval: the pane is holding input Orchestra already // submitted and believes it delivered. Press Enter and say so. if err := w.executionBackend().SendKeys(ctx, session, []string{"Enter"}); err != nil { _ = w.api.ResolveCommand(ctx, command.ID, "rejected", "resubmit not delivered: "+err.Error()) continue } log.Printf("resubmit %s: Enter resent to %s at capture revision %d", command.TaskID, session.PaneID, command.CaptureRevision) if err := w.api.ResolveCommand(ctx, command.ID, "acknowledged", ""); err != nil { log.Printf("ack command %s: %v", command.ID, err) } continue } input, ok := approvalResponse(text, command.Kind) if !ok { _ = w.api.ResolveCommand(ctx, command.ID, "rejected", "prompt does not expose an executable approval control") continue } backend := w.executionBackend() var inputErr error if len(input.Keys) > 0 { inputErr = backend.SendKeys(ctx, session, input.Keys) } else { inputErr = backend.SendText(ctx, session, input.Text) } if inputErr != nil { _ = w.api.ResolveCommand(ctx, command.ID, "rejected", backend.Kind()+" backend did not acknowledge input: "+inputErr.Error()) continue } if err := w.api.ResolveCommand(ctx, command.ID, "acknowledged", ""); err != nil { log.Printf("ack command %s: %v", command.ID, err) } } } func (w *worker) once(ctx context.Context) error { es, _, err := w.api.Events(ctx, w.cursor) if err != nil { return err } for _, e := range es { if t, ok := created(e); ok { w.tasks[t.ID] = t } if e.Type == "TaskLeased" { var p struct { HarnessID string `json:"harness_id"` Epoch string `json:"lease_epoch"` HandoffRef string `json:"handoff_ref"` TransactionID string `json:"transaction_id"` AnchorSHA string `json:"anchor_sha"` } if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID { var until struct { UntilNS int64 `json:"until_ns"` } _ = json.Unmarshal(e.Payload, &until) w.leases[e.TaskID] = lease{Epoch: p.Epoch, HandoffRef: p.HandoffRef, TransactionID: p.TransactionID, AnchorSHA: p.AnchorSHA, Version: e.Version, Until: time.Unix(0, until.UntilNS)} } } if e.Type == "TaskLeaseRenewed" { var p struct { HarnessID string `json:"harness_id"` Epoch string `json:"lease_epoch"` UntilNS int64 `json:"until_ns"` } if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID { l := w.leases[e.TaskID] l.Version, l.Epoch, l.Until = e.Version, p.Epoch, time.Unix(0, p.UntilNS) w.leases[e.TaskID] = l } } if e.Type == "TaskLaunchAcknowledged" { if l, ok := w.leases[e.TaskID]; ok { l.Version = e.Version w.leases[e.TaskID] = l } } if e.Type == "TaskNeedsAttention" { // The diagnostic event increments the aggregate version but leaves // ownership intact. Keep our locally persisted expected version in // sync so a late, otherwise valid completion is not self-staled. if l, ok := w.leases[e.TaskID]; ok { l.Version = e.Version w.leases[e.TaskID] = l } } if e.Type == "TaskCompleted" { delete(w.leases, e.TaskID) if session, active := w.sessions[e.TaskID]; active { if w.executionBackend() == nil { delete(w.sessions, e.TaskID) } else if err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).Kill(ctx, session); err != nil { w.quarantined[e.TaskID] = true w.recordError(fmt.Errorf("close completed pane %s: %w", e.TaskID, err)) continue } else { _ = os.Remove(filepath.Join(session.Worktree, ".orchestra", "done")) _ = os.Remove(filepath.Join(session.Worktree, ".orchestra")) delete(w.sessions, e.TaskID) } } } if e.Type == "TaskPickupValidated" { var p struct { TransactionID string `json:"transaction_id"` } if json.Unmarshal(e.Payload, &p) == nil { if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID { tx.Phase, tx.UpdatedAt = "pickup_validated", time.Now().UTC() w.releases[e.TaskID] = tx } } if l, ok := w.leases[e.TaskID]; ok { l.Version = e.Version w.leases[e.TaskID] = l } } if e.Type == "TaskReleased" || e.Type == "TaskFailed" || e.Type == "TaskBlocked" { if e.Type == "TaskReleased" { var p struct { TransactionID string `json:"transaction_id"` } if json.Unmarshal(e.Payload, &p) == nil { if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID && tx.Phase == "anchor_pushed" { tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC() w.releases[e.TaskID] = tx } } } delete(w.leases, e.TaskID) // A releasing predecessor remains intentionally recoverable until // TaskPickupValidated for its transaction. Do not erase its pane // mapping merely because our own release event was replayed. // // Recoverable means an anchor was actually pushed. Before that // tx.Ref is empty and no successor can pick anything up, so the // mapping protects nothing. F30: a transaction stuck at "prepared" // held the session forever once its pane was gone, health() kept // reporting ActiveTask, and the harness never leased again. tx, releasing := w.releases[e.TaskID] if !releasing || tx.Ref == "" { if releasing { delete(w.releases, e.TaskID) } if session, active := w.sessions[e.TaskID]; active { w.quarantine(ctx, e.TaskID, session) } } } if e.Seq > w.cursor { w.cursor = e.Seq } } // A coordinator restart can restore its task snapshot without retaining // the in-memory event tail. In that state a worker with a persisted cursor // receives an empty page even though a lease is currently assigned to it. // Reconcile the authoritative projection before treating an empty page as // "nothing to do"; otherwise the lease remains invisible until expiry. if len(es) == 0 { if err := w.reconcileLeases(ctx); err != nil { return err } } // State is only a cache. If a lease survived but its TaskCreated event is // older than the worker's cursor (or the event has been compacted), hydrate // the authoritative task projection before deciding whether to start. for taskID := range w.leases { if _, ok := w.tasks[taskID]; !ok { tasks, err := w.api.Tasks(ctx) if err != nil { return fmt.Errorf("hydrate leased task %s: %w", taskID, err) } for _, task := range tasks { w.tasks[task.ID] = task } break } } // Project the whole batch before launching. This prevents a new worker // from resurrecting every historical lease during its initial replay. for taskID, l := range w.leases { if _, started := w.sessions[taskID]; started { continue } if t, ok := w.tasks[taskID]; ok { if err := w.start(ctx, t, l.HandoffRef); err != nil { log.Printf("lease %s: %v", t.ID, err) _, started := w.sessions[taskID] class := classifyLaunchError(err, started) var evidence domain.SessionEvidence if session, ok := w.sessions[taskID]; ok { evidence = w.sessionEvidence(ctx, taskID, session) } if nackErr := w.api.NackStart(ctx, taskID, l.Epoch, l.Version, class, err.Error(), evidence); nackErr != nil { w.recordError(fmt.Errorf("nack launch %s: %w", taskID, nackErr)) continue } if class != "launch_uncertain" { delete(w.leases, taskID) } } } } // Unit/replay-only workers intentionally have no execution backend. A // production worker always does, and only then participates in the live // capture/control protocol. if w.executionBackend() != nil { w.publishCaptures(ctx) w.runCommands(ctx) w.renewLeases(ctx) } w.retryQuarantines(ctx) w.releaseReady(ctx) if err := w.save(); err != nil { return err } return w.api.Ack(ctx, w.cursor) } func (w *worker) reconcileLeases(ctx context.Context) error { tasks, err := w.api.Tasks(ctx) if err != nil { return fmt.Errorf("reconcile leased tasks: %w", err) } active := make(map[string]lease) for _, task := range tasks { w.tasks[task.ID] = task if (task.State == domain.StateLeased || task.State == domain.StateNeedsAttention) && task.Lease != nil && task.Lease.HarnessID == w.harnessID { active[task.ID] = lease{Epoch: task.Lease.Epoch, HandoffRef: task.HandoffRef, TransactionID: task.ReleaseTransaction, AnchorSHA: task.ReleaseAnchor, Version: task.Version, Until: task.Lease.Until} } } for taskID := range w.leases { if _, ok := active[taskID]; !ok { delete(w.leases, taskID) } } for taskID, l := range active { // The coordinator is authoritative for the lease, not for what this // worker has observed under it. Rebuilding the struct wholesale wiped // every worker-local field each tick, which silently disabled the // renewal progress check and reset the usage baseline. Carry them // across, but only while the epoch is the same lease. if prev, ok := w.leases[taskID]; ok && prev.Epoch == l.Epoch { l.ProgressSHA = prev.ProgressSHA l.UsageBaseline = prev.UsageBaseline l.PickupAcknowledged = prev.PickupAcknowledged } w.leases[taskID] = l } return nil } // reRegisterAfterCoordinatorRestart restores the coordinator's in-memory // worker registry. A worker must survive a server restart without operator // intervention; its persisted cursor and sessions remain valid. func (w *worker) reRegisterAfterCoordinatorRestart(ctx context.Context, cause error) bool { if cause == nil || !strings.Contains(cause.Error(), "401 Unauthorized: unknown worker") { return false } if err := w.api.Register(ctx, w.registration); err != nil { log.Printf("re-register: %v", err) return false } log.Printf("re-registered after coordinator restart") return true } func required(k string) string { v := os.Getenv(k) if v == "" { log.Fatalf("%s required", k) } return v } // harnessSpec is one harness identity this process serves. Each spec becomes a // separate federation identity because the coordinator authorizes lease calls by // comparing the URL's worker id against the lease's harness id // (cmd/orchestra/main.go). One process may therefore hold several identities, // but it may never present one identity for several harnesses. type harnessSpec struct { ID string `json:"id"` Harness string `json:"harness"` Token string `json:"token,omitempty"` Backend string `json:"backend,omitempty"` // Herdr is the JSON-RPC address for Backend "herdr". Herdr string `json:"herdr,omitempty"` // TmuxSocket and Command configure Backend "tmux". TmuxSocket string `json:"tmux_socket,omitempty"` Command string `json:"command,omitempty"` State string `json:"state,omitempty"` Address string `json:"address,omitempty"` } // tokenEnvKey maps a harness id onto a per-identity token variable, so a // multi-harness deployment keeps its tokens in the protected environment file // rather than in the harness config file. func tokenEnvKey(id string) string { var b strings.Builder b.WriteString("ORCHESTRA_WORKER_TOKEN_") for _, r := range strings.ToUpper(id) { if (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { b.WriteRune(r) continue } b.WriteRune('_') } return b.String() } // harnessSpecs reads the multi-harness declaration, falling back to the legacy // single-harness environment so an existing deployment upgrades unchanged. func harnessSpecs() []harnessSpec { path := os.Getenv("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE") if path == "" { return []harnessSpec{{ ID: required("ORCHESTRA_WORKER_HERDR_ID"), Harness: required("ORCHESTRA_WORKER_HARNESS"), Token: required("ORCHESTRA_WORKER_TOKEN"), Backend: os.Getenv("ORCHESTRA_WORKER_BACKEND"), Herdr: os.Getenv("ORCHESTRA_WORKER_HERDR"), TmuxSocket: os.Getenv("ORCHESTRA_WORKER_TMUX_SOCKET"), Command: os.Getenv("ORCHESTRA_WORKER_HARNESS_COMMAND"), State: os.Getenv("ORCHESTRA_WORKER_STATE"), Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), }} } b, err := os.ReadFile(path) if err != nil { log.Fatalf("read ORCHESTRA_WORKER_HARNESS_CONFIG_FILE: %v", err) } var specs []harnessSpec if err := json.Unmarshal(b, &specs); err != nil { log.Fatalf("parse ORCHESTRA_WORKER_HARNESS_CONFIG_FILE: %v", err) } if len(specs) == 0 { log.Fatal("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE declares no harnesses") } seen := map[string]bool{} for i, spec := range specs { if spec.ID == "" || spec.Harness == "" { log.Fatalf("harness %d requires id and harness", i) } if seen[spec.ID] { log.Fatalf("harness id %q is declared twice", spec.ID) } seen[spec.ID] = true if spec.Token == "" { key := tokenEnvKey(spec.ID) if spec.Token = os.Getenv(key); spec.Token == "" { log.Fatalf("harness %s has no token: set %s or its config-file token", spec.ID, key) } specs[i] = spec } } return specs } // statePathFor gives every identity its own state file. Sharing one across // harnesses would let a persisted session and its lease cross backends, handing // a tmux backend herdr pane ids it cannot act on. Only the single-harness form // keeps the historical default path, so an existing deployment recovers its // sessions after the upgrade instead of orphaning them. func statePathFor(spec harnessSpec, root, stateDir string, single bool) string { switch { case spec.State != "": return spec.State case stateDir != "": return filepath.Join(stateDir, "state-"+spec.ID+".json") case single: return filepath.Join(root, ".orchestra-worker-state.json") default: return filepath.Join(root, ".orchestra-worker-state-"+spec.ID+".json") } } // backendFor builds the machine-local execution backend for one harness. func backendFor(spec harnessSpec) herdr.Backend { name := strings.ToLower(strings.TrimSpace(spec.Backend)) if name == "" { name = "herdr" } switch name { case "herdr": address := spec.Herdr if address == "" { log.Fatalf("harness %s: backend herdr requires an address", spec.ID) } return herdr.New(address) case "tmux": if spec.Harness != "claude" { log.Fatalf("harness %s: backend tmux currently supports only harness claude, got %q", spec.ID, spec.Harness) } return herdr.NewTmuxBackend(spec.TmuxSocket, spec.Command) default: log.Fatalf("harness %s: unsupported backend %q (want herdr or tmux)", spec.ID, name) return nil } } func main() { workerID := required("ORCHESTRA_WORKER_ID") hard := .75 if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); err == nil && v > 0 && v < 1 { hard = v } projects := map[string]projectConfig{} if path := os.Getenv("ORCHESTRA_WORKER_PROJECT_CONFIG_FILE"); path != "" { b, err := os.ReadFile(path) if err != nil { log.Fatalf("read ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err) } if err := json.Unmarshal(b, &projects); err != nil { log.Fatalf("parse ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err) } } else { for _, project := range strings.Split(os.Getenv("ORCHESTRA_WORKER_PROJECTS"), ",") { if project = strings.TrimSpace(project); project != "" { projects[project] = projectConfig{Repo: required("ORCHESTRA_REPO"), Root: required("ORCHESTRA_WORKTREE_ROOT"), Remote: required("ORCHESTRA_GIT_REMOTE")} } } } if len(projects) == 0 { // Existing deployments may be upgraded before their protected systemd // environment is amended. Stay observable and fail closed in that // interval: an empty declaration makes this worker ineligible for all // new leases instead of turning a configuration rollout into a crash // loop or treating its legacy global checkout as every project. log.Printf("no ORCHESTRA_WORKER_PROJECT_CONFIG_FILE/ORCHESTRA_WORKER_PROJECTS; registering with no supported projects") } for project, config := range projects { if config.Repo == "" || config.Root == "" || config.Remote == "" { log.Fatalf("project %q requires repo, worktree_root, and remote", project) } } supported := make([]string, 0, len(projects)) for project := range projects { supported = append(supported, project) } sort.Strings(supported) repo, root, remote := required("ORCHESTRA_REPO"), required("ORCHESTRA_WORKTREE_ROOT"), required("ORCHESTRA_GIT_REMOTE") if len(supported) > 0 { first := projects[supported[0]] repo, root, remote = first.Repo, first.Root, first.Remote } soft, window := .55, int64(200000) if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_SOFT"), 64); err == nil && v > 0 && v < hard { soft = v } if v, err := strconv.ParseInt(os.Getenv("ORCHESTRA_CONTEXT_WINDOW"), 10, 64); err == nil && v > 0 { window = v } url, admit := required("ORCHESTRA_URL"), os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN") specs := harnessSpecs() if len(specs) == 1 && specs[0].ID != workerID { // A single-harness deployment keeps the historical invariant: its lease // owner and its process identity are the same name. log.Fatal("ORCHESTRA_WORKER_ID must equal ORCHESTRA_WORKER_HERDR_ID so leases and offline recovery have one owner") } stateDir := os.Getenv("ORCHESTRA_WORKER_STATE_DIR") workers := make([]*worker, 0, len(specs)) for _, spec := range specs { w := &worker{ api: federation.Client{BaseURL: url, WorkerID: spec.ID, Token: spec.Token, AdmitToken: admit}, harnessID: spec.ID, harness: spec.Harness, repo: repo, root: root, remote: remote, projects: projects, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, releases: map[string]releaseTransaction{}, quarantined: map[string]bool{}, statePath: spec.State, hard: hard, soft: soft, window: window, // Capacity stays one per identity because a herdr's declared // concurrency is one. Serving N harnesses gives the process N slots. registration: federation.Worker{ID: spec.ID, Address: spec.Address, Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()}, } if w.registration.Address == "" { w.registration.Address = os.Getenv("ORCHESTRA_WORKER_ADDRESS") } w.statePath = statePathFor(spec, root, stateDir, len(specs) == 1) if err := w.load(); err != nil { log.Fatal(err) } w.backend = backendFor(spec) if err := w.api.Register(context.Background(), w.registration); err != nil { log.Fatalf("register %s: %v", spec.ID, err) } // Same reason as the coordinator's: the worker half of the pair must be // checkable from journalctl, not only from the coordinator's // credentialed worker list. b := buildinfo.Current() log.Printf("orchestra-worker revision %s built %s dirty %s", b.Revision, b.Time, b.Dirty) log.Printf("serving harness %s (%s) on %s backend, state %s", spec.ID, spec.Harness, w.backend.Kind(), w.statePath) workers = append(workers, w) } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() for { // Identities are served sequentially. Their checkouts and Git remotes are // shared, so concurrent ticks would race two fetch/worktree operations on // one repository for no useful latency gain at this fan-out. for _, w := range workers { if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil { w.recordError(fmt.Errorf("heartbeat: %w", err)) log.Printf("[%s] heartbeat: %v", w.harnessID, err) if w.reRegisterAfterCoordinatorRestart(ctx, err) { continue } } if err := w.once(ctx); err != nil { w.recordError(fmt.Errorf("poll: %w", err)) log.Printf("[%s] poll: %v", w.harnessID, err) w.reRegisterAfterCoordinatorRestart(ctx, err) } } select { case <-ctx.Done(): return case <-ticker.C: } } } // federatedTurn is the worker half of a turn boundary. The coordinator // reconciles human input and answers with the decisions this session has not // been shown; the worker delivers them into its own pane. // // Nothing is preempted. The boundary is confirmed against the live pane // first, so a correction never lands mid tool call. func (w *worker) federatedTurn(ctx context.Context, id string, a herdr.Adapter, verdict string) { l, ok := w.leases[id] if !ok { return } s, ok := w.sessions[id] if !ok { return } boundary, ok := a.(herdr.TurnBoundary) if !ok { return } at, err := boundary.AtTurnBoundary(ctx, s) if err != nil { w.recordError(fmt.Errorf("turn boundary %s: %w", id, err)) return } if !at { return } // A phase this session no longer runs ends it, whether this worker asked // for the change or an operator made it (F22). Checked before the request // below so a session cannot advance a phase twice. if w.phaseChanged(id, s) { w.rotateForPhase(ctx, id, a, s) return } // A contradiction outranks both requests below. An implementer that has // found the plan wrong should not verify a phase of it or ask to leave the // phase; Orchestra decides what happens to the plan first. if w.reportPlanMismatch(ctx, id, s) { w.rotateForPhase(ctx, id, a, s) return } // A plan phase asks to be verified before the work phase asks to move. // Checked first because verifying the last phase is usually what makes a // session ready to leave implement at all, and this session keeps running // either way: verification is progress within a phase, not a change of one. if w.requestPlanVerification(ctx, id, s) { return } // The agent asks for a phase change here, at a boundary it has reached // (F21). Orchestra decides, and an accepted change ends this session. if w.requestPhase(ctx, id, s) { w.rotateForPhase(ctx, id, a, s) return } answer, err := w.api.Turn(ctx, id, l.Epoch, verdict, s.DeliveredDecisions) if err != nil { // Observable, not fatal. A coordinator that cannot be reached does not // make this session's current intent any more stale than it already is. w.recordError(fmt.Errorf("federated turn %s: %w", id, err)) return } if answer.Verdict == orchestrator.TurnPrepareHandoff && !s.HandoffRequested { // The coordinator has lost the ability to refresh this task's intent. // Ask for a handoff; the release loop takes over as soon as the agent // writes the report, exactly as it does for a local session. requester, ok := a.(herdr.ReasonedHandoffRequester) if !ok { w.recordError(fmt.Errorf("reconcile failure handoff %s: adapter cannot state a reason", id)) return } if err := requester.RequestHandoffReason(ctx, s, "reconcile_failure", nil); err != nil { w.recordError(fmt.Errorf("reconcile failure handoff %s: %w", id, err)) return } s.HandoffRequested, s.HandoffReason = true, "reconcile_failure" w.sessions[id] = s _ = w.save() return } if len(answer.Decisions) == 0 { return } if err := w.sendPrompt(ctx, s, agentctx.DecisionNotice(answer.Decisions)); err != nil { // Not recorded as delivered, so the next boundary retries. w.recordError(fmt.Errorf("deliver decisions %s: %w", id, err)) return } for _, d := range answer.Decisions { s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID) } w.sessions[id] = s _ = w.save() } // phaseRequestFile is the agent's bounded phase-change intent (F21). Prose in // the pane is not a request: matching on it would make the protocol depend on // wording the agent is free to vary, and on Orchestra reading its own echo. const phaseRequestFile = "phase-request.json" // phaseRequest is what the agent writes. It states the phase it believes it // is in as well as the one it wants, so a request written from a stale // context is refused rather than applied to whatever phase is current. type phaseRequest struct { From domain.WorkPhase `json:"from"` To domain.WorkPhase `json:"to"` } // phaseArtifact names the sealed output each phase must produce before it may // be left. Phases absent from this table seal nothing. var phaseArtifact = map[domain.WorkPhase]string{ domain.WorkPhaseResearch: "research.json", domain.WorkPhasePlan: "plan.md", } func currentPhase(t domain.Task) domain.WorkPhase { if t.WorkPhase == "" { return domain.WorkPhaseFrame } return t.WorkPhase } // phaseChanged reports whether this session is running a phase the task has // since left. It covers a change this worker requested and one an operator // made through the coordinator equally, because both leave the same evidence: // a session whose context was built for a phase that is no longer current. func (w *worker) phaseChanged(id string, s herdr.Session) bool { t, ok := w.tasks[id] if !ok || s.Phase == "" { return false } return string(currentPhase(t)) != s.Phase } // rotateForPhase ends the current cognitive session because the phase moved // (F22). A phase change is a change of context, not of instruction: leaving // the old agent running would either waste the lease waiting for it to idle // out, as run 3 did, or let it keep working under a brief that no longer // applies. func (w *worker) rotateForPhase(ctx context.Context, id string, a herdr.Adapter, s herdr.Session) { if s.HandoffRequested { return } requester, ok := a.(herdr.ReasonedHandoffRequester) if !ok { w.recordError(fmt.Errorf("phase rotation %s: adapter cannot state a reason", id)) return } if err := requester.RequestHandoffReason(ctx, s, "phase_changed", nil); err != nil { w.recordError(fmt.Errorf("phase rotation %s: %w", id, err)) return } s.HandoffRequested, s.HandoffReason = true, "phase_changed" w.sessions[id] = s _ = w.save() log.Printf("phase changed for %s: session rotating", id) } // answerRefusedPhase tells the agent why its request was refused and drops the // file so it can write a corrected one. The request survives a failed send, so // the refusal is delivered at the next boundary instead of being lost. func (w *worker) answerRefusedPhase(ctx context.Context, id string, s herdr.Session, path, reason string) { text := "Orchestra refused your phase request: " + reason + "\n\nWrite a corrected .orchestra/phase-request.json, or keep working in the current phase. Do not repeat the refused request." if err := w.sendPrompt(ctx, s, text); err != nil { w.recordError(fmt.Errorf("deliver phase refusal %s: %w", id, err)) return } if err := os.Remove(path); err != nil { w.recordError(fmt.Errorf("phase request %s: %w", id, err)) } log.Printf("phase request %s refused: %s", id, reason) } // requestPhase carries an agent's phase request to the coordinator (F21). // It reports whether the phase moved. // // Everything checkable locally is checked before the call, so an agent that // asked for the wrong thing learns it from a recorded error rather than from // a lease that quietly stops being renewed. func (w *worker) requestPhase(ctx context.Context, id string, s herdr.Session) bool { path := filepath.Join(s.Worktree, ".orchestra", phaseRequestFile) b, err := os.ReadFile(path) if err != nil { return false } var req phaseRequest if err := json.Unmarshal(b, &req); err != nil { w.recordError(fmt.Errorf("phase request %s: %w", id, err)) return false } t, ok := w.tasks[id] if !ok { return false } if req.From != currentPhase(t) { w.recordError(fmt.Errorf("phase request %s: task is in work phase %q, not %q", id, currentPhase(t), req.From)) return false } if !domain.CanTransitionPhase(req.From, req.To) { w.recordError(fmt.Errorf("phase request %s: %q to %q is not a legal transition", id, req.From, req.To)) return false } // The phase being left seals its result before it may be left. Decoding // here means a malformed artifact is reported against the agent that // wrote it, while its session is still alive to be told. var artifact []byte if name := phaseArtifact[req.From]; name != "" { artifact, err = os.ReadFile(filepath.Join(s.Worktree, ".orchestra", name)) if err != nil { w.recordError(fmt.Errorf("phase request %s: work phase %q must seal .orchestra/%s first: %w", id, req.From, name, err)) w.answerRefusedPhase(ctx, id, s, path, fmt.Sprintf("work phase %q must seal .orchestra/%s first: %v", req.From, name, err)) return false } var decErr error switch req.From { case domain.WorkPhaseResearch: _, decErr = workphase.DecodeResearch(artifact) case domain.WorkPhasePlan: // A new seal is markdown. The coordinator re-parses and also // resolves research citations, which the worker cannot check // because it does not hold the accepted research. _, decErr = workphase.ParsePlan(artifact) } if decErr != nil { w.recordError(fmt.Errorf("phase request %s: %s artifact: %w", id, req.From, decErr)) // A local refusal is still a refusal, and the agent is the only // party that can fix it. Recording it in worker health alone left // a live session parked at a boundary forever with nothing telling // it what was wrong (F39) — the silent-loop shape the comment // above this block warns about, reached by the one path that had // no delivery. w.answerRefusedPhase(ctx, id, s, path, fmt.Sprintf(".orchestra/%s does not match the schema: %v", name, decErr)) return false } } l := w.leases[id] // Derived, not random: a redelivery after a lost response must carry the // same id so the coordinator recognises it instead of advancing twice. op := "phase:" + id + ":" + l.Epoch + ":" + string(req.From) + ":" + string(req.To) phase, err := w.api.AdvancePhase(ctx, id, l.Epoch, op, req.From, req.To, artifact) if err != nil { w.recordError(fmt.Errorf("phase request %s: %w", id, err)) // A refusal is an answer, and it names the phase the agent may ask // for. Recording it only in worker health would leave the agent // rewriting the same rejected file at every boundary with nothing // telling it why, which is the silent-loop shape this codebase keeps // producing. A transport failure is not an answer and is retried. var status *federation.StatusError if errors.As(err, &status) && status.Code == http.StatusConflict { w.answerRefusedPhase(ctx, id, s, path, status.Body) } return false } if phase == "" { // Accepted but not advanced: the coordinator raised a trajectory gate // and the human now owns the move. Keep the request so the same ask is // re-sent, under the same operation id, once the gate clears. return false } // Durable before the file is removed. A removal that raced the response // would lose the request and leave the agent waiting on an answer that // already arrived. if err := os.Remove(path); err != nil { w.recordError(fmt.Errorf("phase request %s: %w", id, err)) } if phase == domain.WorkPhaseReview { // Each entry into review starts without the previous cycle's findings. // The worktree survives a changes-requested round trip, so a reviewer // that writes .orchestra/done without rewriting the file would have // the earlier review sealed against the new commit — and submit binds // whatever it reads to the commit it is submitting, so a stale pass // would look exactly like a fresh one. if err := os.Remove(filepath.Join(s.Worktree, ".orchestra", reviewFile)); err != nil && !os.IsNotExist(err) { w.recordError(fmt.Errorf("clear stale review %s: %w", id, err)) } } t.WorkPhase = phase w.tasks[id] = t log.Printf("phase request %s accepted: %s to %s", id, req.From, phase) return true } // sendPrompt delivers Orchestra-originated input and confirms the harness took // it. A phase continuation or a decision notice whose Enter is lost strands the // session exactly as a lost launch does. func (w *worker) sendPrompt(ctx context.Context, s herdr.Session, text string) error { backend := w.executionBackend() if backend == nil { return fmt.Errorf("execution backend is not configured") } if err := backend.Prompt(ctx, s.PaneID, text, time.Minute); err != nil { return err } return w.confirmInput(ctx, s, text) } // confirmInput is the one place Orchestra proves a write landed. A backend // whose own protocol acknowledges input does not implement InputConfirmer and // needs no second opinion. func (w *worker) confirmInput(ctx context.Context, s herdr.Session, text string) error { c, ok := w.executionBackend().(herdr.InputConfirmer) if !ok { return nil } evidence, err := c.ConfirmInput(ctx, s, text) if err != nil { return err } log.Printf("input to %s confirmed: %s", s.PaneID, evidence) return nil } // paneProgress reports pane content with input lines removed where the backend // can separate them, and falls back to the raw capture where it cannot. func (w *worker) paneProgress(ctx context.Context, adapter herdr.CLIAdapter, s herdr.Session) (string, error) { if p, ok := w.executionBackend().(herdr.PaneProgress); ok { return p.PaneProgress(ctx, s) } return adapter.PaneCapture(ctx, s, "recent") } // planProgressFile is the implementer's bounded verification request. It // carries one phase and one status, and the only status it may carry is a // request: an agent that could write "verified" would be marking its own work // done, which is the whole thing this machinery exists to prevent. const planProgressFile = "plan-progress.json" type planProgressRequest struct { Phase string `json:"phase"` Status string `json:"status"` } // requestPlanVerification runs one phase's automated checks and posts the // results. It reports whether anything was done, so the caller does not treat // an absent request as a failure. // // The commands come from the accepted plan, which the coordinator resolves and // authorises. This worker never reads a command out of the request: the // request names a phase, and nothing else about it is trusted. func (w *worker) requestPlanVerification(ctx context.Context, id string, s herdr.Session) bool { path := filepath.Join(s.Worktree, ".orchestra", planProgressFile) b, err := os.ReadFile(path) if err != nil { return false } var req planProgressRequest if err := json.Unmarshal(b, &req); err != nil { w.answerRefusedProgress(ctx, id, s, path, fmt.Sprintf(".orchestra/%s is not valid JSON: %v", planProgressFile, err)) return false } if req.Status != domain.PlanPhaseRequestStatus { // Naming this refusal precisely matters: an agent that wrote // "verified" is trying to establish a fact, and it has to learn that // establishing facts is not its job rather than retry the same file. w.answerRefusedProgress(ctx, id, s, path, fmt.Sprintf("status %q is not writable by you. The only status you may request is %q; Orchestra decides the rest by running the plan's own commands.", req.Status, domain.PlanPhaseRequestStatus)) return false } l := w.leases[id] commands, err := w.api.PlanPhaseCommands(ctx, id, req.Phase, l.Epoch) if err != nil { // A refusal is an answer and reaches the agent. A transport failure is // not, and the request survives for the next boundary. var status *federation.StatusError if errors.As(err, &status) && status.Code == http.StatusConflict { w.answerRefusedProgress(ctx, id, s, path, status.Body) return false } w.recordError(fmt.Errorf("plan verification %s: %w", id, err)) return false } head, err := git(ctx, s.Worktree, "rev-parse", "HEAD") if err != nil { w.recordError(fmt.Errorf("plan verification %s: head: %s: %w", id, head, err)) return false } runs := make([]federation.VerificationRun, 0, len(commands)) for _, argv := range commands { // No shell. The quality gate's envelope is exec in the worktree, and a // plan command is agent-authored, so it gets that envelope and never a // weaker one. A pipe here is a literal argument. cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) cmd.Dir = s.Worktree out, runErr := cmd.CombinedOutput() code := 0 if runErr != nil { code = 1 var exit *exec.ExitError if errors.As(runErr, &exit) { code = exit.ExitCode() } } runs = append(runs, federation.VerificationRun{Command: argv, ExitCode: code, Output: tail(string(out), review.MaxGateOutputBytes)}) } status, err := w.api.RecordPlanPhase(ctx, id, req.Phase, strings.TrimSpace(string(head)), l.Epoch, runs) if err != nil { w.recordError(fmt.Errorf("plan verification %s: %w", id, err)) return false } // Durable before the request is removed. Removing first would lose the // request if the response were lost, and the agent would wait on an answer // that already happened. if err := os.Remove(path); err != nil { w.recordError(fmt.Errorf("plan verification %s: %w", id, err)) } w.tellProgressOutcome(ctx, id, s, req.Phase, status, runs) return true } // tellProgressOutcome delivers what Orchestra established. A phase that did // not verify is the case that matters: without this the agent sees a request // disappear and has to guess whether it worked. func (w *worker) tellProgressOutcome(ctx context.Context, id string, s herdr.Session, phase, status string, runs []federation.VerificationRun) { var b strings.Builder switch status { case string(domain.PlanPhaseVerified): fmt.Fprintf(&b, "Orchestra verified %s. Every automated check passed. Move to the next phase of the plan.", phase) case string(domain.PlanPhaseAwaitingManual): fmt.Fprintf(&b, "Orchestra ran %s's automated checks and they passed. The phase is waiting for the human to confirm its manual steps, which you cannot do. Continue with the next phase; the sign-off arrives on its own.", phase) default: fmt.Fprintf(&b, "Orchestra ran %s's checks and it is not verified. Fix what failed, then write .orchestra/%s again.\n", phase, planProgressFile) for _, r := range runs { if r.ExitCode == 0 { continue } fmt.Fprintf(&b, "\n%s exited %d:\n%s\n", strings.Join(r.Command, " "), r.ExitCode, r.Output) } } if err := w.sendPrompt(ctx, s, b.String()); err != nil { w.recordError(fmt.Errorf("deliver plan verification %s: %w", id, err)) } log.Printf("plan phase %s of %s: %s", phase, id, status) } // answerRefusedProgress tells the implementer why its request was refused and // drops the file so a corrected one can be written. Recording a refusal only // in worker health leaves a live session rewriting the same rejected file at // every boundary, which is the silent-loop shape this codebase keeps producing // (F39, F42, and the completion tail). func (w *worker) answerRefusedProgress(ctx context.Context, id string, s herdr.Session, path, reason string) { w.recordError(fmt.Errorf("plan verification %s refused: %s", id, reason)) text := "Orchestra refused your verification request: " + reason + "\n\nWrite a corrected .orchestra/" + planProgressFile + ", or keep working. Do not repeat the refused request." if err := w.sendPrompt(ctx, s, text); err != nil { w.recordError(fmt.Errorf("deliver plan verification refusal %s: %w", id, err)) return } if err := os.Remove(path); err != nil { w.recordError(fmt.Errorf("plan verification %s: %w", id, err)) } log.Printf("plan verification %s refused: %s", id, reason) } // planMismatchFile is the implementer's bounded report that the accepted plan // is contradicted by the code. It carries an observation, never a replacement // plan: writing the next plan is the planning phase's work, and an implementer // that could supply one would be planning from inside the phase meant to // execute a plan. const planMismatchFile = "plan-mismatch.json" // reportPlanMismatch carries the report to the coordinator, which decides // whether to reopen planning, reopen research, or stop for the human. It // reports whether the phase moved, because an accepted reopen ends this // session exactly as an accepted phase request does. func (w *worker) reportPlanMismatch(ctx context.Context, id string, s herdr.Session) bool { path := filepath.Join(s.Worktree, ".orchestra", planMismatchFile) b, err := os.ReadFile(path) if err != nil { return false } var m domain.PlanMismatch if err := json.Unmarshal(b, &m); err != nil { w.answerRefusedMismatch(ctx, id, s, path, fmt.Sprintf(".orchestra/%s is not valid JSON: %v", planMismatchFile, err)) return false } t, ok := w.tasks[id] if !ok { return false } // Both bindings are filled from what this worker can verify, not from what // the agent wrote. A report is about the plan the task actually works from // and the tree it actually has; letting the agent state either would make // the staleness check meaningless. m.PlanRef = t.PlanRef head, err := git(ctx, s.Worktree, "rev-parse", "HEAD") if err != nil { w.recordError(fmt.Errorf("plan mismatch %s: head: %s: %w", id, head, err)) return false } m.AtSHA = strings.TrimSpace(string(head)) if err := m.Validate(); err != nil { w.answerRefusedMismatch(ctx, id, s, path, err.Error()) return false } l := w.leases[id] phase, err := w.api.ReportPlanMismatch(ctx, id, l.Epoch, m) if err != nil { var status *federation.StatusError if errors.As(err, &status) && status.Code == http.StatusConflict { w.answerRefusedMismatch(ctx, id, s, path, status.Body) return false } w.recordError(fmt.Errorf("plan mismatch %s: %w", id, err)) return false } // Durable before the file goes. A removal that raced the response would // lose the report and leave the agent waiting on an answer that arrived. if err := os.Remove(path); err != nil { w.recordError(fmt.Errorf("plan mismatch %s: %w", id, err)) } if phase == "" { // Recorded, and the task stopped for the human. The session ends: the // contradiction it found is now someone else's to resolve. log.Printf("plan mismatch %s: blocked for the human", id) return true } t.WorkPhase = domain.WorkPhase(phase) w.tasks[id] = t log.Printf("plan mismatch %s: reopened %s", id, phase) return true } // answerRefusedMismatch tells the implementer why its report was refused. A // stale report is the common case, and the agent has to learn that rather than // rewrite the same rejected file at every boundary. func (w *worker) answerRefusedMismatch(ctx context.Context, id string, s herdr.Session, path, reason string) { w.recordError(fmt.Errorf("plan mismatch %s refused: %s", id, reason)) text := "Orchestra refused your plan mismatch report: " + reason + "\n\nA report needs observed, contradicts, the phase id, and requested_action of replan, research, or human_decision. Do not propose a replacement plan: that is the planning phase's work. Write a corrected .orchestra/" + planMismatchFile + ", or keep working." if err := w.sendPrompt(ctx, s, text); err != nil { w.recordError(fmt.Errorf("deliver plan mismatch refusal %s: %w", id, err)) return } if err := os.Remove(path); err != nil { w.recordError(fmt.Errorf("plan mismatch %s: %w", id, err)) } log.Printf("plan mismatch %s refused: %s", id, reason) }