Files
orchestra/cmd/orchestra-worker/main.go
T
kami 063a3ab9ad Clear the previous cycle's findings when review is entered again
The worktree survives a changes-requested round trip, so .orchestra/review.json
from the first review is still there when the second one starts. A reviewer
that writes .orchestra/done without rewriting it would have the earlier
findings sealed against the new commit, and submit binds whatever it reads to
the commit being submitted, so a stale pass is indistinguishable from a fresh
one.

Observed on the 2026-08-28 baseline run: the file from 10:55:45 was still
present when the second review session launched at 10:57:48. That reviewer did
rewrite it, so the run is sound, but nothing enforced it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
2026-08-28 11:03:42 +04:00

2048 lines
75 KiB
Go

// orchestra-worker consumes router-issued leases for one or more local
// execution backends. Each declared harness is a separate federation identity
// with its own token, cursor, backend, and state file, because the coordinator
// authorizes a lease call by comparing the URL's worker id against the lease's
// harness id. Homesrv remains the scheduler and CAS authority. This process
// owns only local Git and pane operations.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"orchestra/internal/agentctx"
"orchestra/internal/buildinfo"
"orchestra/internal/continuity"
"orchestra/internal/domain"
"orchestra/internal/federation"
"orchestra/internal/herdr"
"orchestra/internal/orchestrator"
"orchestra/internal/review"
"orchestra/internal/workphase"
"os"
"os/exec"
"os/signal"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"syscall"
"time"
)
type worker struct {
api federation.Client
backend herdr.Backend
// herdr is retained as a test/backward-compatibility alias. Production
// workers set backend; executionBackend keeps older state-machine tests
// from needing protocol-irrelevant rewrites.
herdr *herdr.Client
harnessID, harness, repo, root, remote string
projects map[string]projectConfig
cursor uint64
tasks map[string]domain.Task
sessions map[string]herdr.Session
leases map[string]lease
releases map[string]releaseTransaction
quarantined map[string]bool
statePath string
hard float64
registration federation.Worker
lastError string
lastErrorAt time.Time
soft float64
window int64
}
func (w *worker) executionBackend() herdr.Backend {
if w.backend != nil {
return w.backend
}
if w.herdr != nil {
return w.herdr
}
return nil
}
func (w *worker) recordError(err error) {
if err == nil {
return
}
w.lastError = err.Error()
w.lastErrorAt = time.Now().UTC()
}
func (w *worker) health(ctx context.Context) federation.WorkerHealth {
h := federation.WorkerHealth{HerdrStatus: "unknown"}
if backend := w.executionBackend(); backend != nil {
h.Backend = backend.Kind()
}
for taskID, session := range w.sessions {
// Workers currently advertise capacity one. Pick deterministically so a
// recovered legacy state with more sessions remains intelligible.
if h.ActiveTask == "" || taskID < h.ActiveTask {
h.ActiveTask, h.ActivePane = taskID, session.PaneID
}
}
if backend := w.executionBackend(); backend != nil {
checkCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
err := backend.Check(checkCtx)
cancel()
h.CheckedAt = time.Now().UTC()
if err == nil {
h.HerdrStatus = "reachable"
} else {
h.HerdrStatus = "unreachable"
w.recordError(fmt.Errorf("local %s backend: %w", backend.Kind(), err))
}
}
h.LastError, h.ErrorAt = w.lastError, w.lastErrorAt
return h
}
type lease struct {
Epoch string `json:"epoch"`
HandoffRef string `json:"handoff_ref,omitempty"`
TransactionID string `json:"transaction_id,omitempty"`
AnchorSHA string `json:"anchor_sha,omitempty"`
PickupAcknowledged bool `json:"pickup_acknowledged,omitempty"`
Version int `json:"version"`
Until time.Time `json:"until"`
UsageBaseline float64 `json:"usage_baseline,omitempty"`
// ProgressSHA hashes the pane capture taken at the last renewal. Renewal
// requires the pane to have changed since then, or the agent to be busy.
ProgressSHA string `json:"progress_sha,omitempty"`
}
type releaseTransaction struct {
ID string `json:"id"`
LeaseVersion int `json:"lease_version"`
Ref string `json:"handoff_ref,omitempty"`
AnchorSHA string `json:"anchor_sha,omitempty"`
Phase string `json:"phase"` // prepared, anchor_pushed, event_committed, pickup_validated, predecessor_retired
AgentReleased bool `json:"agent_released,omitempty"`
LastError string `json:"last_error,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
type projectConfig struct {
Repo string `json:"repo"`
Root string `json:"worktree_root"`
Remote string `json:"remote"`
QualityGate string `json:"quality_gate,omitempty"`
SafeOperations []string `json:"safe_operations,omitempty"`
}
type completionEvidence struct {
TaskID string `json:"task_id"`
Project string `json:"project"`
Worker string `json:"worker"`
Harness string `json:"harness"`
PaneID string `json:"pane_id"`
BaseSHA string `json:"base_sha"`
ResultSHA string `json:"result_sha"`
Branch string `json:"branch"`
Remote string `json:"remote"`
QualityGate string `json:"quality_gate,omitempty"`
GateExit int `json:"gate_exit"`
GateOutput string `json:"gate_output,omitempty"`
CompletedAt time.Time `json:"completed_at"`
}
// tail keeps the end of a gate log, which is where the failure is.
func tail(s string, max int) string {
if len(s) <= max {
return s
}
return s[len(s)-max:]
}
type workerState struct {
Cursor uint64 `json:"cursor"`
Sessions map[string]herdr.Session `json:"sessions"`
Tasks map[string]domain.Task `json:"tasks"`
Leases map[string]lease `json:"leases"`
Releases map[string]releaseTransaction `json:"releases"`
Quarantined map[string]bool `json:"quarantined,omitempty"`
}
func (w *worker) load() error {
b, e := os.ReadFile(w.statePath)
if e == nil {
var s workerState
if err := json.Unmarshal(b, &s); err != nil {
return fmt.Errorf("corrupt worker state %s: %w", w.statePath, err)
}
w.cursor = s.Cursor
w.sessions = s.Sessions
w.tasks = s.Tasks
w.leases = s.Leases
w.releases = s.Releases
w.quarantined = s.Quarantined
} else if !errors.Is(e, os.ErrNotExist) {
return fmt.Errorf("read worker state %s: %w", w.statePath, e)
}
if w.sessions == nil {
w.sessions = map[string]herdr.Session{}
}
if w.tasks == nil {
w.tasks = map[string]domain.Task{}
}
if w.leases == nil {
w.leases = map[string]lease{}
}
if w.releases == nil {
w.releases = map[string]releaseTransaction{}
}
if w.quarantined == nil {
w.quarantined = map[string]bool{}
}
return nil
}
func (w *worker) save() error {
b, e := json.Marshal(workerState{Cursor: w.cursor, Sessions: w.sessions, Tasks: w.tasks, Leases: w.leases, Releases: w.releases, Quarantined: w.quarantined})
if e != nil {
return e
}
if e := os.MkdirAll(filepath.Dir(w.statePath), 0700); e != nil {
return e
}
tmp := w.statePath + ".tmp"
f, e := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
if e != nil {
return e
}
if _, e = f.Write(b); e == nil {
e = f.Sync()
}
if closeErr := f.Close(); e == nil {
e = closeErr
}
if e != nil {
_ = os.Remove(tmp)
return e
}
if e = os.Rename(tmp, w.statePath); e != nil {
return e
}
dir, e := os.Open(filepath.Dir(w.statePath))
if e != nil {
return e
}
defer dir.Close()
return dir.Sync()
}
// quarantine stops a pane before its lost lease mapping can be forgotten.
// A failed close remains durable and is retried; it is never treated as a
// harmless cleanup error while the old harness could still be working.
func (w *worker) quarantine(ctx context.Context, taskID string, s herdr.Session) {
if w.executionBackend() == nil {
w.quarantined[taskID] = true
return
}
if err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).Kill(ctx, s); err != nil {
w.quarantined[taskID] = true
w.recordError(fmt.Errorf("quarantine %s: %w", taskID, err))
return
}
delete(w.sessions, taskID)
delete(w.quarantined, taskID)
}
func (w *worker) retryQuarantines(ctx context.Context) {
for taskID := range w.quarantined {
if s, ok := w.sessions[taskID]; ok {
w.quarantine(ctx, taskID, s)
}
}
}
type artifactCAS struct{ api federation.Client }
func (c artifactCAS) PutArtifact(b []byte) (string, error) {
return c.api.PutArtifact(context.Background(), b)
}
func (c artifactCAS) Artifact(ref string) ([]byte, error) {
return c.api.Artifact(context.Background(), ref)
}
func created(e domain.Event) (domain.Task, bool) {
if e.Type != "TaskCreated" {
return domain.Task{}, false
}
var p struct {
Source string `json:"source"`
ExternalID string `json:"external_id"`
Project string `json:"project"`
Capability []string `json:"capability"`
Title string `json:"title"`
Description string `json:"description"`
Acceptance []string `json:"acceptance"`
QualityGate string `json:"quality_gate"`
}
if json.Unmarshal(e.Payload, &p) != nil || p.Source == "" || p.ExternalID == "" || p.Project == "" {
return domain.Task{}, false
}
return domain.Task{ID: e.TaskID, Source: p.Source, ExternalID: p.ExternalID, Project: p.Project, Capability: p.Capability, Title: p.Title, Description: p.Description, Acceptance: p.Acceptance, QualityGate: p.QualityGate}, true
}
func (w *worker) project(t domain.Task) (projectConfig, error) {
if w.projects != nil {
if p, ok := w.projects[t.Project]; ok && p.Repo != "" && p.Root != "" && p.Remote != "" {
return p, nil
}
return projectConfig{}, fmt.Errorf("project %q is not configured on worker", t.Project)
}
return projectConfig{Repo: w.repo, Root: w.root, Remote: w.remote}, nil
}
func (w *worker) syncBase(ctx context.Context, p projectConfig) error {
if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil {
return fmt.Errorf("fetch base checkout: %s: %w", out, err)
}
branch, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "symbolic-ref", "--quiet", "--short", "HEAD").Output()
if err != nil {
return fmt.Errorf("identify base branch: %w", err)
}
branchName := strings.TrimSpace(string(branch))
if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "merge", "--ff-only", p.Remote+"/"+branchName).CombinedOutput(); err != nil {
return fmt.Errorf("fast-forward base checkout: %s: %w", out, err)
}
return nil
}
func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
var wt string
var h continuity.Handoff
var err error
p, err := w.project(t)
if err != nil {
return err
}
// Synchronize the local base before any worktree operation. A worker never
// treats a coordinator-side path as truth; the Git remote is the only
// cross-machine transport.
if err := w.syncBase(ctx, p); err != nil {
return err
}
if ref != "" {
b, err := w.api.Artifact(ctx, ref)
if err != nil {
return err
}
h, err = continuity.Decode(b)
if err != nil {
return err
}
if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "fetch", p.Remote, "--prune").CombinedOutput(); err != nil {
return fmt.Errorf("fetch pickup anchor: %s: %w", out, err)
}
wt = filepath.Join(p.Root, t.ID)
if _, err := os.Stat(wt); os.IsNotExist(err) {
if out, err := exec.CommandContext(ctx, "git", "-C", p.Repo, "worktree", "add", "-b", "orchestra/"+t.ID, wt, h.Anchor.GitSHA).CombinedOutput(); err != nil {
return fmt.Errorf("create pickup worktree: %s: %w", out, err)
}
}
if err = continuity.ValidatePickup(wt, h, taskHash(t)); err != nil {
return err
}
} else {
wt, err = (orchestrator.GitWorktrees{Repo: p.Repo, Root: p.Root}).Create(ctx, t)
if err != nil {
return err
}
}
backend := w.executionBackend()
if backend == nil {
return fmt.Errorf("execution backend is not configured")
}
if _, err = backend.Worktree(ctx, p.Repo, wt, "orchestra/"+t.ID); err != nil {
return err
}
s, err := backend.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID)
if err != nil {
return err
}
// The phase this session was launched to run. A later phase change makes
// this session's context the wrong one, which is what rotates it (F22).
s.Phase = string(currentPhase(t))
s.TaskFileSHA = taskHash(t)
if w.harness == "claude" {
s.ContextHandoffSHA, _ = fileSHA256(filepath.Join(wt, "HANDOFF.md"))
}
// One renderer, on both machines. The worker fetches the reduced
// authority as data and renders it with agentctx, so a decision the human
// recorded before this session existed is visible from its first turn.
intent, err := w.api.Intent(ctx, t.ID)
if err != nil {
return fmt.Errorf("effective intent: %w", err)
}
in := agentctx.Input{
Task: t, Intent: intent, Phase: t.WorkPhase, DecisionRequest: t.DecisionRequest,
Git: agentctx.GitState{Worktree: wt, Branch: "orchestra/" + t.ID},
RepoRules: agentctx.DiscoverRepoRules(wt),
}
if sha, shaErr := herdr.HeadSHA(wt); shaErr == nil {
in.Git.HeadSHA = sha
}
if ref != "" {
in.Handoff = &h
}
if len(p.SafeOperations) > 0 {
in.Policy = []string{
"Permitted without an operator grant, inside this worktree only: " + strings.Join(p.SafeOperations, ", ") + ".",
"Network access, secrets, destructive actions, and paths outside this worktree require an explicit operator approval.",
}
}
if t.ResearchRef != "" {
b, artErr := w.api.Artifact(ctx, t.ResearchRef)
if artErr != nil {
return fmt.Errorf("research artifact: %w", artErr)
}
r, decErr := workphase.DecodeResearch(b)
if decErr != nil {
return fmt.Errorf("research artifact: %w", decErr)
}
in.Research = &r
}
if t.PlanRef != "" {
b, artErr := w.api.Artifact(ctx, t.PlanRef)
if artErr != nil {
return fmt.Errorf("plan artifact: %w", artErr)
}
pl, decErr := workphase.DecodePlan(b)
if decErr != nil {
return fmt.Errorf("plan artifact: %w", decErr)
}
in.Plan = &pl
}
if t.Review != nil {
b, artErr := w.api.Artifact(ctx, t.Review.ArtifactRef)
if artErr != nil {
return fmt.Errorf("review artifact: %w", artErr)
}
r, decErr := review.Decode(b)
if decErr != nil {
return fmt.Errorf("review artifact: %w", decErr)
}
in.Review = &r
}
built, err := agentctx.Build(in)
if err != nil {
return fmt.Errorf("build context: %w", err)
}
prompt := built.System + "\n\n" + built.Task
// The transport decides what is submitted, never what the agent receives:
// the file holds the exact bytes agentctx rendered either way.
submitted, transport := prompt, herdr.LaunchInline
if lt, ok := backend.(herdr.LaunchTransporter); ok {
transport = lt.LaunchTransport(w.harness)
}
if transport == herdr.LaunchFileRef {
submitted = herdr.LaunchReference
}
if writeErr := herdr.WriteLaunchContext(s.Worktree, prompt); writeErr != nil {
// Under LaunchFileRef the file is the instruction, so a failed write
// is a failed launch rather than lost evidence.
if transport == herdr.LaunchFileRef {
return fmt.Errorf("launch context %s: %w", t.ID, writeErr)
}
w.recordError(fmt.Errorf("launch context %s: %w", t.ID, writeErr))
}
// The launch instruction carried these, so the first turn boundary must
// not re-announce them as news.
for _, d := range intent.Decisions {
s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID)
}
w.sessions[t.ID] = s
if err := w.save(); err != nil {
return err
}
// A prompt response can be lost after the backend accepted it. Persist the
// session first so the worker can reconcile/release it after restart.
if err := backend.Prompt(ctx, s.PaneID, submitted, 0); err != nil {
return err
}
// Acknowledging a launch means the harness accepted the instruction, not
// that the adapter call returned nil. Without this the worker reported a
// started agent while the prompt sat unsubmitted in the input editor.
if c, ok := backend.(herdr.InputConfirmer); ok {
evidence, confirmErr := c.ConfirmInput(ctx, s, submitted)
if confirmErr != nil {
// An unsubmitted prompt leaves a live pane that nothing owns, and
// a retained session would make the retry skip this task
// entirely. Reclaim both so the released lease can be re-leased.
if killErr := backend.Kill(ctx, s); killErr != nil {
w.recordError(fmt.Errorf("kill unlaunched pane %s: %w", t.ID, killErr))
}
delete(w.sessions, t.ID)
if saveErr := w.save(); saveErr != nil {
w.recordError(fmt.Errorf("save after failed launch %s: %w", t.ID, saveErr))
}
return fmt.Errorf("launch %s: %w", t.ID, confirmErr)
}
log.Printf("launch %s confirmed: %s", t.ID, evidence)
}
// Baseline the progress check at launch, not at the first renewal. The
// renewal gate exempts a lease with no baseline, which handed a pane that
// opened and never started a full free renewal period (F34) — the exact
// case the gate exists to catch. Capturing here costs one pane read and
// makes the first renewal a real comparison.
if l, ok := w.leases[t.ID]; ok && l.ProgressSHA == "" {
if text, progressErr := w.paneProgress(ctx, herdr.CLIAdapter{Backend: backend, Harness: w.harness}, s); progressErr == nil {
l.ProgressSHA = domain.Hash([]byte(text))
w.leases[t.ID] = l
} else {
w.recordError(fmt.Errorf("baseline launch progress %s: %w", t.ID, progressErr))
}
}
if l, ok := w.leases[t.ID]; ok {
if err := w.api.Start(ctx, t.ID, l.Epoch, l.Version, w.sessionEvidence(ctx, t.ID, s)); err != nil {
return fmt.Errorf("ack start: %w", err)
}
l.Version++
w.leases[t.ID] = l
if err := w.save(); err != nil {
return err
}
}
if ref != "" {
return w.ackPickup(ctx, t.ID, s)
}
return nil
}
func classifyLaunchError(err error, sessionStarted bool) string {
// Positive evidence that the harness never accepted the prompt is not
// uncertainty. Release the lease so the existing retry path can take it.
if errors.Is(err, herdr.ErrPromptNotSubmitted) {
return "prompt_not_submitted"
}
if sessionStarted {
// A prompt response can be lost after herdr accepted it. Never reclaim
// that pane just because its acknowledgement was uncertain.
return "launch_uncertain"
}
text := strings.ToLower(err.Error())
if strings.Contains(text, "handoff") || strings.Contains(text, "pickup") || strings.Contains(text, "task.md") {
return "invalid_handoff"
}
return "launch_transient"
}
func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) }
func fileSHA256(path string) (string, error) {
b, err := os.ReadFile(path)
if err != nil {
return "", err
}
return domain.Hash(b), nil
}
func (w *worker) releaseReady(ctx context.Context) {
for id, s := range w.sessions {
if w.quarantined[id] {
continue
}
if l := w.leases[id]; l.HandoffRef != "" && !l.PickupAcknowledged {
if err := w.ackPickup(ctx, id, s); err != nil {
w.recordError(err)
continue
}
}
if _, err := os.Stat(filepath.Join(s.Worktree, ".orchestra", "done")); err == nil {
// A done marker is an intent, not enough on its own: do not race a
// still-running native harness into committing half-written work.
status, statusErr := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).AgentStatus(ctx, s)
if statusErr != nil {
w.recordError(fmt.Errorf("completion identity %s: %w", id, statusErr))
continue
}
if herdr.IsBusy(status) {
w.recordError(fmt.Errorf("completion %s deferred: agent status %s", id, status))
continue
}
evidence, err := w.finalize(ctx, id, s)
if err != nil {
w.recordError(fmt.Errorf("complete %s: %w", id, err))
log.Printf("complete %s: %v", id, err)
continue
}
report, _ := json.Marshal(evidence)
ref, err := w.api.PutArtifact(ctx, report)
if err != nil {
w.recordError(fmt.Errorf("upload completion %s: %w", id, err))
log.Printf("upload completion %s: %v", id, err)
continue
}
outcome, err := w.submit(ctx, id, s, evidence)
if errors.Is(err, errReviewArtifact) {
// Only the reviewing agent can fix this, and it is still alive
// to be told. Dropping the done marker stops the five-second
// retry and makes the corrected file the thing that finishes.
w.recordError(fmt.Errorf("submit %s: %w", id, err))
w.answerRefusedReview(ctx, id, s, err)
continue
}
if err != nil {
w.recordError(fmt.Errorf("submit %s: %w", id, err))
log.Printf("submit %s: %v", id, err)
continue
}
if outcome == federation.SubmitChangesRequested {
// The sealed review sent the work back. The task is in
// implement again, so the done marker is stale and the phase
// change rotates this session on the next tick.
_ = os.Remove(filepath.Join(s.Worktree, ".orchestra", "done"))
log.Printf("review returned %s to implementation", id)
continue
}
if outcome == federation.SubmitNoPublisher {
if err = w.api.Complete(ctx, id, ref, evidence.ResultSHA, evidence.Branch, evidence.Remote, w.leases[id].Epoch, w.leases[id].Version, w.usageReceipt(s, w.leases[id]), w.sessionEvidence(ctx, id, s)); err != nil {
w.recordError(fmt.Errorf("complete %s: %w", id, err))
log.Printf("complete %s: %v", id, err)
continue
}
}
// Completion is durable before closing the exact pane. If close
// fails, retain the session mapping for a later explicit cleanup.
a := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}
if err := a.Kill(ctx, s); err != nil {
w.recordError(fmt.Errorf("close completed pane %s: %w", id, err))
log.Printf("close completed pane %s: %v", id, err)
continue
}
_ = os.Remove(filepath.Join(s.Worktree, ".orchestra", "done"))
_ = os.Remove(filepath.Join(s.Worktree, ".orchestra"))
delete(w.sessions, id)
delete(w.leases, id)
_ = w.save()
continue
}
if _, err := os.Stat(filepath.Join(s.Worktree, herdr.HandoffReportFile)); err == nil || w.releases[id].ID != "" {
w.advanceRelease(ctx, id, s)
continue
}
w.rotationTick(ctx, id, s)
}
}
func (w *worker) adapter(s herdr.Session, remote string) herdr.CLIAdapter {
a := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness, Window: w.window, CAS: artifactCAS{w.api}, Remote: remote}
switch w.harness {
case "claude":
a.Usage = herdr.ClaudeUsage
case "codex":
a.Usage = herdr.CodexUsage
case "opencode":
a.Usage = herdr.OpenCodeUsage
}
return a
}
// rotationTick is the checkout-owner state machine. Occupancy, tool activity,
// and pane status are all read from the persisted harness session identity;
// any unknown source is recorded and never treated as zero usage.
func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
// Claude Code owns its context threshold through the installed
// context-handoff hook. A changed HANDOFF.md means that hook has landed a
// durable local continuation. Resume in the same process with Claude's
// native context reset instead of manufacturing Orchestra's cross-worker
// release artifact. Codex and OpenCode continue through the existing
// occupancy/release state machine below.
if w.harness == "claude" {
if err := w.advanceClaudeContextReset(ctx, id, s); err != nil {
w.recordError(fmt.Errorf("Claude context reset %s: %w", id, err))
}
// A turn boundary is not a rotation. Claude owns its context rollover
// through the installed hook, which is why the occupancy state machine
// below is skipped, but phase requests and human decisions are carried
// at the boundary and returning here left both unreachable on this
// harness. Every decision recorded against a live Claude session went
// undelivered, and no phase request could ever be read.
t, ok := w.tasks[id]
if !ok {
w.recordError(fmt.Errorf("turn boundary %s: task cache missing", id))
return
}
p, err := w.project(t)
if err != nil {
w.recordError(err)
return
}
w.federatedTurn(ctx, id, w.adapter(s, p.Remote), orchestrator.TurnContinue)
return
}
t, ok := w.tasks[id]
if !ok {
w.recordError(fmt.Errorf("rotation %s: task cache missing", id))
return
}
p, err := w.project(t)
if err != nil {
w.recordError(err)
return
}
a := w.adapter(s, p.Remote)
resolved, err := a.ResolveSessionIdentity(s)
if err != nil {
w.recordError(fmt.Errorf("rotation %s occupancy degraded: %w", id, err))
return
}
// DeepEqual, not !=: Session carries a slice since decisions are tracked
// per session, so it is no longer comparable with ==.
if !reflect.DeepEqual(resolved, s) {
w.sessions[id] = resolved
s = resolved
_ = w.save()
}
d := (orchestrator.RotationStateMachine{Soft: w.soft, Hard: w.hard}).Evaluate(ctx, a, s)
if d.ActivityDegraded != nil {
w.recordError(fmt.Errorf("rotation %s activity degraded: %w", id, d.ActivityDegraded))
}
if d.Degraded != nil {
w.recordError(fmt.Errorf("rotation %s degraded: %w", id, d.Degraded))
if d.Action == orchestrator.TurnContinue || d.Action == "" {
return
}
}
if d.Action == orchestrator.TurnContinue || d.Action == "" {
w.federatedTurn(ctx, id, a, orchestrator.TurnContinue)
return
}
if d.Action == orchestrator.TurnRefuse || s.HandoffRequested {
return
}
if d.Reason == "milestone" || d.Reason == "thrash" {
if err := a.RequestHandoffReason(ctx, s, d.Reason, d.DeadEnds); err != nil {
w.recordError(fmt.Errorf("rotation %s %s prompt: %w", id, d.Reason, err))
return
}
} else if err := a.RequestHandoff(ctx, s); err != nil {
w.recordError(fmt.Errorf("rotation %s threshold prompt: %w", id, err))
return
}
s.HandoffRequested, s.HandoffReason = true, d.Reason
w.sessions[id] = s
_ = w.save()
}
func (w *worker) sendLine(ctx context.Context, s herdr.Session, line string) error {
backend := w.executionBackend()
if backend == nil {
return fmt.Errorf("execution backend is not configured")
}
if err := backend.SendText(ctx, s, line); err != nil {
return fmt.Errorf("send %q: %w", line, err)
}
if err := backend.SendKeys(ctx, s, []string{"ENTER"}); err != nil {
return fmt.Errorf("submit %q: %w", line, err)
}
// A lost Enter here leaves the session mid-rollover with /clear sitting in
// the editor, which is worse than a lost launch: nothing retries it.
if err := w.confirmInput(ctx, s, line); err != nil {
return fmt.Errorf("submit %q: %w", line, err)
}
return nil
}
func (w *worker) advanceClaudeContextReset(ctx context.Context, id string, s herdr.Session) error {
backend := w.executionBackend()
if backend == nil {
return fmt.Errorf("execution backend is not configured")
}
if s.ContextResetSHA == "" {
sha, err := fileSHA256(filepath.Join(s.Worktree, "HANDOFF.md"))
if os.IsNotExist(err) {
return nil
}
if err != nil {
return fmt.Errorf("read HANDOFF.md: %w", err)
}
if sha == s.ContextHandoffSHA {
return nil
}
status, err := backend.AgentStatus(ctx, s)
if err != nil {
return fmt.Errorf("confirm Claude stopped after handoff: %w", err)
}
if status != "idle" {
return nil
}
s.ContextResetSHA = sha
s.ContextResetPhase = "clear"
w.sessions[id] = s
if err := w.save(); err != nil {
return err
}
}
switch s.ContextResetPhase {
case "clear":
if err := w.sendLine(ctx, s, "/clear"); err != nil {
return err
}
s.ContextResetPhase = "handoff"
w.sessions[id] = s
if err := w.save(); err != nil {
return err
}
// /clear redraws Claude's input UI asynchronously. Give it a small,
// bounded interval before submitting the new-session file mention.
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(500 * time.Millisecond):
}
fallthrough
case "handoff":
if err := w.sendLine(ctx, s, "@HANDOFF.md"); err != nil {
return err
}
s.ContextHandoffSHA = s.ContextResetSHA
s.ContextResetSHA = ""
s.ContextResetPhase = ""
// Claude normally opens a fresh transcript for /clear. Force the next
// observation to discover it instead of retaining the exhausted path.
s.SessionFile = ""
w.sessions[id] = s
return w.save()
default:
return fmt.Errorf("unknown persisted context-reset phase %q", s.ContextResetPhase)
}
}
func (w *worker) advanceRelease(ctx context.Context, id string, s herdr.Session) {
t, ok := w.tasks[id]
if !ok {
w.recordError(fmt.Errorf("release %s: task cache missing", id))
return
}
p, err := w.project(t)
if err != nil {
w.recordError(fmt.Errorf("release %s: %w", id, err))
return
}
if w.releases == nil {
w.releases = map[string]releaseTransaction{}
}
tx := w.releases[id]
if tx.ID == "" {
l, ok := w.leases[id]
if !ok {
w.recordError(fmt.Errorf("release %s: lease missing", id))
return
}
tx = releaseTransaction{ID: domain.NewID(), LeaseVersion: l.Version, Phase: "prepared", UpdatedAt: time.Now().UTC()}
w.releases[id] = tx
_ = w.save()
}
a := w.adapter(s, p.Remote)
if tx.Phase == "prepared" {
prepared, err := a.PrepareRelease(ctx, s)
if err != nil {
tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
w.releases[id] = tx
_ = w.save()
w.recordError(fmt.Errorf("release %s prepare: %w", id, err))
return
}
tx.Ref, tx.AnchorSHA, tx.Phase, tx.LastError, tx.UpdatedAt = prepared.Ref, prepared.AnchorSHA, "anchor_pushed", "", time.Now().UTC()
w.releases[id] = tx
_ = w.save()
}
if tx.Phase == "anchor_pushed" {
l := w.leases[id]
if err := w.api.Release(ctx, id, tx.Ref, tx.AnchorSHA, tx.ID, l.Epoch, tx.LeaseVersion, w.sessionEvidence(ctx, id, s)); err != nil {
tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
w.releases[id] = tx
_ = w.save()
w.recordError(fmt.Errorf("release %s commit: %w", id, err))
return
}
tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC()
w.releases[id] = tx
_ = w.save()
}
if tx.Phase == "event_committed" && !tx.AgentReleased {
if err := a.ReleaseAgent(ctx, s); err != nil {
tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
w.releases[id] = tx
_ = w.save()
w.recordError(fmt.Errorf("release %s release agent: %w", id, err))
return
}
tx.AgentReleased, tx.LastError, tx.UpdatedAt = true, "", time.Now().UTC()
w.releases[id] = tx
_ = w.save()
}
if tx.Phase == "pickup_validated" {
if err := a.Kill(ctx, s); err != nil {
tx.LastError, tx.UpdatedAt = err.Error(), time.Now().UTC()
w.releases[id] = tx
_ = w.save()
w.recordError(fmt.Errorf("release %s retire predecessor: %w", id, err))
return
}
_ = os.Remove(filepath.Join(s.Worktree, herdr.HandoffReportFile))
tx.Phase, tx.UpdatedAt = "predecessor_retired", time.Now().UTC()
w.releases[id] = tx
_ = w.save()
delete(w.sessions, id)
delete(w.releases, id)
_ = w.save()
}
}
// ackPickup retries the successor acknowledgement from persisted lease state.
// It is safe after a lost response: the coordinator recognizes the exact
// transaction/lease epoch as an idempotent pickup.
func (w *worker) ackPickup(ctx context.Context, id string, s herdr.Session) error {
l, ok := w.leases[id]
if !ok || l.HandoffRef == "" || l.TransactionID == "" || l.AnchorSHA == "" {
return fmt.Errorf("pickup %s: lease is missing its release transaction", id)
}
if l.PickupAcknowledged {
return nil
}
if err := w.api.Pickup(ctx, id, l.HandoffRef, l.AnchorSHA, l.TransactionID, l.Epoch, l.Version, w.sessionEvidence(ctx, id, s)); err != nil {
return fmt.Errorf("pickup %s acknowledgement: %w", id, err)
}
l.PickupAcknowledged = true
l.Version++ // TaskPickupValidated increments the task version.
w.leases[id] = l
return w.save()
}
func (w *worker) sessionEvidence(ctx context.Context, taskID string, s herdr.Session) domain.SessionEvidence {
e := domain.SessionEvidence{PaneID: s.PaneID, HarnessID: w.harnessID, PaneState: "open", Source: "worker", CheckedAt: time.Now().UTC()}
text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, s, "recent")
if err != nil {
e.PaneState = "unreachable"
return e
}
if capture, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: taskID, PaneID: s.PaneID, Text: text}); err == nil {
e.CapturedAt = capture.At
}
return e
}
func (w *worker) usageReceipt(s herdr.Session, l lease) map[string]any {
if s.SessionFile == "" && !(w.harness == "opencode" && s.SessionID != "") {
return map[string]any{"harness_id": w.harnessID, "consumed": 0, "known": false, "error": "native usage identity missing"}
}
var usage herdr.Usage
var err error
switch w.harness {
case "claude":
usage, err = herdr.ClaudeUsage(s.SessionFile)
case "codex":
usage, err = herdr.CodexUsage(s.SessionFile)
case "opencode":
usage, err = herdr.OpenCodeSessionUsage(s.SessionID)
}
if err != nil {
return map[string]any{"harness_id": w.harnessID, "consumed": 0, "known": false, "error": err.Error()}
}
delta := float64(usage.Numerator()) - l.UsageBaseline
if delta < 0 {
delta = 0
}
return map[string]any{"harness_id": w.harnessID, "input_tokens": usage.Input, "cache_read_tokens": usage.CacheRead, "cache_write_tokens": usage.CacheWrite, "consumed": delta, "lease_usage_delta": delta, "known": true}
}
func git(ctx context.Context, dir string, args ...string) ([]byte, error) {
return exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...).CombinedOutput()
}
// finalize performs only mechanical delivery work. It never asks the harness
// to narrate Git state, gates, or a report; those are generated from the
// worker-owned checkout and then verified against the configured remote.
func (w *worker) finalize(ctx context.Context, id string, s herdr.Session) (completionEvidence, error) {
t, ok := w.tasks[id]
if !ok {
return completionEvidence{}, fmt.Errorf("task cache missing")
}
p, err := w.project(t)
if err != nil {
return completionEvidence{}, err
}
if s.TaskFileSHA != "" {
if err := continuity.VerifyTaskFile(s.Worktree, s.TaskFileSHA); err != nil {
return completionEvidence{}, fmt.Errorf("verify immutable TASK.md: %w", err)
}
}
base, err := git(ctx, s.Worktree, "rev-parse", "HEAD")
if err != nil {
return completionEvidence{}, fmt.Errorf("base sha: %s: %w", base, err)
}
e := completionEvidence{TaskID: id, Project: t.Project, Worker: w.harnessID, Harness: w.harness, PaneID: s.PaneID, BaseSHA: strings.TrimSpace(string(base)), Remote: p.Remote, QualityGate: t.QualityGate, CompletedAt: time.Now().UTC()}
if _, err := git(ctx, s.Worktree, "diff", "--quiet", "--", "TASK.md"); err != nil {
return completionEvidence{}, errors.New("TASK.md was modified")
}
if out, err := git(ctx, s.Worktree, "add", "-A", "--", ".", stageExclude); err != nil {
return completionEvidence{}, fmt.Errorf("stage result: %s: %w", out, err)
}
if _, err := git(ctx, s.Worktree, "diff", "--cached", "--quiet"); err != nil {
if out, err := git(ctx, s.Worktree, "commit", "-m", "orchestra: complete "+id); err != nil {
return completionEvidence{}, fmt.Errorf("commit result: %s: %w", out, err)
}
}
sha, err := git(ctx, s.Worktree, "rev-parse", "HEAD")
if err != nil {
return completionEvidence{}, fmt.Errorf("result sha: %s: %w", sha, err)
}
e.ResultSHA = strings.TrimSpace(string(sha))
// The gate runs after the result commit, against a clean tree that is
// byte-for-byte the commit being submitted. Running it first bound the
// evidence to the pre-commit HEAD, which CheckSubmission rejects because
// gate sha, review sha and head sha must be one commit.
gateCommand := t.QualityGate
if gateCommand == "" {
gateCommand = p.QualityGate
}
e.QualityGate = gateCommand
if gateCommand != "" {
gate := exec.CommandContext(ctx, "sh", "-c", gateCommand)
gate.Dir = s.Worktree
out, gateErr := gate.CombinedOutput()
e.GateOutput = tail(string(out), review.MaxGateOutputBytes)
if gateErr != nil {
e.GateExit = 1
return completionEvidence{}, fmt.Errorf("quality gate %q: %s: %w", gateCommand, out, gateErr)
}
}
// The submission branch is derived, not the worktree's local name: the
// coordinator computes "orchestra/<task>" independently, and a pull
// request can only be opened for a branch both halves name the same way.
e.Branch = "orchestra/" + id
if out, err := git(ctx, s.Worktree, "push", p.Remote, "HEAD:refs/heads/"+e.Branch); err != nil {
return completionEvidence{}, fmt.Errorf("push result: %s: %w", out, err)
}
remote, err := git(ctx, s.Worktree, "ls-remote", p.Remote, "refs/heads/"+e.Branch)
if err != nil || !strings.HasPrefix(string(remote), e.ResultSHA+"\t") {
return completionEvidence{}, fmt.Errorf("verify pushed sha: got %q: %w", strings.TrimSpace(string(remote)), err)
}
return e, nil
}
// reviewFile is where the reviewing session leaves its findings. The reviewer
// supplies findings and nothing else: the commit they are bound to is the
// result commit the worker just made, which the agent cannot know and must not
// assert.
const reviewFile = "review.json"
// errReviewArtifact is a refusal only the reviewing agent can fix. It is
// separated from every other submission failure because recording it in worker
// health alone would leave a live session being retried every five seconds
// with nothing telling it what is wrong — the silent-loop shape this codebase
// keeps producing (F39, F42).
var errReviewArtifact = errors.New("review artifact")
// answerRefusedReview tells the reviewing session why its finish was refused
// and clears the done marker so a corrected artifact is what finishes the
// phase. The refusal survives a failed send: the marker is dropped either way,
// and the agent writes it again when it has written the file.
func (w *worker) answerRefusedReview(ctx context.Context, id string, s herdr.Session, cause error) {
text := "Orchestra refused your completion: " + cause.Error() +
"\n\nWrite .orchestra/" + reviewFile + " as {\"findings\": [...]}, an empty list if you found nothing, then write .orchestra/done again. Do not set a commit sha."
if err := w.sendPrompt(ctx, s, text); err != nil {
w.recordError(fmt.Errorf("deliver review refusal %s: %w", id, err))
}
if err := os.Remove(filepath.Join(s.Worktree, ".orchestra", "done")); err != nil {
w.recordError(fmt.Errorf("clear done marker %s: %w", id, err))
}
log.Printf("completion %s refused: %v", id, cause)
}
// submit seals the review and hands the result to the human through a pull
// request. It is the only path from a reviewed change to TaskSubmitted; the
// direct completion call remains for a project with no forge configured.
func (w *worker) submit(ctx context.Context, id string, s herdr.Session, e completionEvidence) (string, error) {
var result review.Result
b, err := os.ReadFile(filepath.Join(s.Worktree, ".orchestra", reviewFile))
if err != nil {
// An absent review file is not a pass. Submission needs a sealed
// review, and inventing an empty one would launder "the reviewer wrote
// nothing" into "the reviewer found nothing".
return "", fmt.Errorf("%w: the review phase sealed no .orchestra/%s: %v", errReviewArtifact, reviewFile, err)
}
if err := json.Unmarshal(b, &result); err != nil {
return "", fmt.Errorf("%w: .orchestra/%s is not valid JSON: %v", errReviewArtifact, reviewFile, err)
}
result.ResultSHA = e.ResultSHA
if err := result.Validate(); err != nil {
return "", fmt.Errorf("%w: .orchestra/%s: %v", errReviewArtifact, reviewFile, err)
}
l := w.leases[id]
// The gate result is bound to the commit the gate ran against, which
// finalize guarantees is the commit being submitted. A project with no
// quality gate still produces a bound result: Passed() needs the sha.
gate := domain.GateResult{Command: e.QualityGate, ExitCode: e.GateExit, SHA: e.ResultSHA, Output: e.GateOutput}
return w.api.Submit(ctx, id, l.Epoch, l.Version, e.ResultSHA, e.Remote, result, gate)
}
// stageExclude keeps Orchestra's own control files out of the result commit.
// It names the directory, not the marker inside it: .orchestra carries a
// .gitignore of "*" (herdr/adapter.go), and naming an ignored file in a
// pathspec makes git refuse the whole add with "The following paths are
// ignored by one of your .gitignore files". F42, live on run 5: the review
// agent wrote .orchestra/done, and every completion attempt failed on that
// refusal, once every five seconds, with the task stuck in review. Excluding
// the directory also holds for a worktree that has no inner .gitignore.
const stageExclude = ":!.orchestra"
func (w *worker) renewLeases(ctx context.Context) {
if w.executionBackend() == nil {
return
}
now := time.Now()
for taskID, l := range w.leases {
s, ok := w.sessions[taskID]
if !ok || l.Version == 0 || l.Until.After(now.Add(domain.LeaseRenewAt)) {
continue
}
adapter := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}
// Input lines are excluded: keystrokes arriving at a pane, from
// Orchestra or from anyone else, are not the agent doing work.
text, err := w.paneProgress(ctx, adapter, s)
if err != nil {
w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err))
continue
}
// A live pane is not progress. Renewing on pane existence alone let a
// pane that opened and never started hold its lease forever, which is
// what orphaned the July task once the launch itself had failed.
status, err := adapter.AgentStatus(ctx, s)
if err != nil {
w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err))
continue
}
progress := domain.Hash([]byte(text))
switch {
case herdr.IsBusy(status):
case progress != l.ProgressSHA && l.ProgressSHA != "":
case l.ProgressSHA == "":
// First renewal has no baseline to compare against. Record one and
// allow this renewal; the next one must show real movement.
default:
w.recordError(fmt.Errorf("lease %s not renewed: agent status %s and pane unchanged since the last renewal", taskID, status))
continue
}
if err := w.api.Renew(ctx, taskID, l.Epoch, l.Version, int(domain.LeaseTTL.Seconds())); err != nil {
w.recordError(fmt.Errorf("renew lease %s: %w", taskID, err))
log.Printf("renew lease %s: %v", taskID, err)
} else {
// RenewLease appends one event. Retain that epoch locally until its
// replay arrives so a release transaction uses the same version.
l.Version++
l.Until = now.Add(domain.LeaseTTL)
l.ProgressSHA = progress
w.leases[taskID] = l
_ = w.save()
}
}
}
// publishCaptures makes remote panes observable without allowing the
// coordinator to touch their unix herdr socket.
func (w *worker) publishCaptures(ctx context.Context) {
for taskID, session := range w.sessions {
text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, session, "recent")
if err != nil {
continue
}
if _, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: taskID, PaneID: session.PaneID, Text: text}); err != nil {
log.Printf("publish capture %s: %v", taskID, err)
}
}
}
type approvalInput struct {
Text string
Keys []string
}
func approvalResponse(text, kind string) (approvalInput, bool) {
low := strings.ToLower(text)
// Never invent a keystroke. y/n prompts label both decisions directly.
if strings.Contains(low, "[y/n]") || strings.Contains(low, "(y/n)") {
if kind == "grant_approval" {
return approvalInput{Text: "y\n"}, true
}
return approvalInput{Text: "n\n"}, true
}
// OpenCode's explicit selector states "Allow once Allow always Reject"
// and "enter confirm". Send a real ENTER key, not a newline through
// pane.send_text: OpenCode's selector does not treat the latter as input.
// Enter is consequently a bounded one-time grant;
// rejection would require unobservable selector navigation, so refuse it.
if kind == "grant_approval" && strings.Contains(low, "allow once") && strings.Contains(low, "allow always") && strings.Contains(low, "reject") && strings.Contains(low, "enter confirm") {
return approvalInput{Keys: []string{"ENTER"}}, true
}
return approvalInput{}, false
}
func (w *worker) runCommands(ctx context.Context) {
commands, err := w.api.Commands(ctx)
if err != nil {
log.Printf("poll controls: %v", err)
return
}
for _, command := range commands {
session, ok := w.sessions[command.TaskID]
if !ok || session.PaneID != command.PaneID {
_ = w.api.ResolveCommand(ctx, command.ID, "stale", "session or pane changed")
continue
}
text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, session, "recent")
if err != nil {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "capture unavailable: "+err.Error())
continue
}
capture, err := w.api.PublishCapture(ctx, federation.Capture{TaskID: command.TaskID, PaneID: session.PaneID, Text: text})
if err != nil {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "cannot publish capture: "+err.Error())
continue
}
if capture.Revision != command.CaptureRevision {
_ = w.api.ResolveCommand(ctx, command.ID, "stale", "capture revision changed")
continue
}
if command.Kind == "resubmit" {
// Not an approval: the pane is holding input Orchestra already
// submitted and believes it delivered. Press Enter and say so.
if err := w.executionBackend().SendKeys(ctx, session, []string{"Enter"}); err != nil {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "resubmit not delivered: "+err.Error())
continue
}
log.Printf("resubmit %s: Enter resent to %s at capture revision %d", command.TaskID, session.PaneID, command.CaptureRevision)
if err := w.api.ResolveCommand(ctx, command.ID, "acknowledged", ""); err != nil {
log.Printf("ack command %s: %v", command.ID, err)
}
continue
}
input, ok := approvalResponse(text, command.Kind)
if !ok {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "prompt does not expose an executable approval control")
continue
}
backend := w.executionBackend()
var inputErr error
if len(input.Keys) > 0 {
inputErr = backend.SendKeys(ctx, session, input.Keys)
} else {
inputErr = backend.SendText(ctx, session, input.Text)
}
if inputErr != nil {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", backend.Kind()+" backend did not acknowledge input: "+inputErr.Error())
continue
}
if err := w.api.ResolveCommand(ctx, command.ID, "acknowledged", ""); err != nil {
log.Printf("ack command %s: %v", command.ID, err)
}
}
}
func (w *worker) once(ctx context.Context) error {
es, _, err := w.api.Events(ctx, w.cursor)
if err != nil {
return err
}
for _, e := range es {
if t, ok := created(e); ok {
w.tasks[t.ID] = t
}
if e.Type == "TaskLeased" {
var p struct {
HarnessID string `json:"harness_id"`
Epoch string `json:"lease_epoch"`
HandoffRef string `json:"handoff_ref"`
TransactionID string `json:"transaction_id"`
AnchorSHA string `json:"anchor_sha"`
}
if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
var until struct {
UntilNS int64 `json:"until_ns"`
}
_ = json.Unmarshal(e.Payload, &until)
w.leases[e.TaskID] = lease{Epoch: p.Epoch, HandoffRef: p.HandoffRef, TransactionID: p.TransactionID, AnchorSHA: p.AnchorSHA, Version: e.Version, Until: time.Unix(0, until.UntilNS)}
}
}
if e.Type == "TaskLeaseRenewed" {
var p struct {
HarnessID string `json:"harness_id"`
Epoch string `json:"lease_epoch"`
UntilNS int64 `json:"until_ns"`
}
if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == w.harnessID {
l := w.leases[e.TaskID]
l.Version, l.Epoch, l.Until = e.Version, p.Epoch, time.Unix(0, p.UntilNS)
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskLaunchAcknowledged" {
if l, ok := w.leases[e.TaskID]; ok {
l.Version = e.Version
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskNeedsAttention" {
// The diagnostic event increments the aggregate version but leaves
// ownership intact. Keep our locally persisted expected version in
// sync so a late, otherwise valid completion is not self-staled.
if l, ok := w.leases[e.TaskID]; ok {
l.Version = e.Version
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskCompleted" {
delete(w.leases, e.TaskID)
if session, active := w.sessions[e.TaskID]; active {
if w.executionBackend() == nil {
delete(w.sessions, e.TaskID)
} else if err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).Kill(ctx, session); err != nil {
w.quarantined[e.TaskID] = true
w.recordError(fmt.Errorf("close completed pane %s: %w", e.TaskID, err))
continue
} else {
_ = os.Remove(filepath.Join(session.Worktree, ".orchestra", "done"))
_ = os.Remove(filepath.Join(session.Worktree, ".orchestra"))
delete(w.sessions, e.TaskID)
}
}
}
if e.Type == "TaskPickupValidated" {
var p struct {
TransactionID string `json:"transaction_id"`
}
if json.Unmarshal(e.Payload, &p) == nil {
if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID {
tx.Phase, tx.UpdatedAt = "pickup_validated", time.Now().UTC()
w.releases[e.TaskID] = tx
}
}
if l, ok := w.leases[e.TaskID]; ok {
l.Version = e.Version
w.leases[e.TaskID] = l
}
}
if e.Type == "TaskReleased" || e.Type == "TaskFailed" || e.Type == "TaskBlocked" {
if e.Type == "TaskReleased" {
var p struct {
TransactionID string `json:"transaction_id"`
}
if json.Unmarshal(e.Payload, &p) == nil {
if tx := w.releases[e.TaskID]; tx.ID != "" && tx.ID == p.TransactionID && tx.Phase == "anchor_pushed" {
tx.Phase, tx.LastError, tx.UpdatedAt = "event_committed", "", time.Now().UTC()
w.releases[e.TaskID] = tx
}
}
}
delete(w.leases, e.TaskID)
// A releasing predecessor remains intentionally recoverable until
// TaskPickupValidated for its transaction. Do not erase its pane
// mapping merely because our own release event was replayed.
//
// Recoverable means an anchor was actually pushed. Before that
// tx.Ref is empty and no successor can pick anything up, so the
// mapping protects nothing. F30: a transaction stuck at "prepared"
// held the session forever once its pane was gone, health() kept
// reporting ActiveTask, and the harness never leased again.
tx, releasing := w.releases[e.TaskID]
if !releasing || tx.Ref == "" {
if releasing {
delete(w.releases, e.TaskID)
}
if session, active := w.sessions[e.TaskID]; active {
w.quarantine(ctx, e.TaskID, session)
}
}
}
if e.Seq > w.cursor {
w.cursor = e.Seq
}
}
// A coordinator restart can restore its task snapshot without retaining
// the in-memory event tail. In that state a worker with a persisted cursor
// receives an empty page even though a lease is currently assigned to it.
// Reconcile the authoritative projection before treating an empty page as
// "nothing to do"; otherwise the lease remains invisible until expiry.
if len(es) == 0 {
if err := w.reconcileLeases(ctx); err != nil {
return err
}
}
// State is only a cache. If a lease survived but its TaskCreated event is
// older than the worker's cursor (or the event has been compacted), hydrate
// the authoritative task projection before deciding whether to start.
for taskID := range w.leases {
if _, ok := w.tasks[taskID]; !ok {
tasks, err := w.api.Tasks(ctx)
if err != nil {
return fmt.Errorf("hydrate leased task %s: %w", taskID, err)
}
for _, task := range tasks {
w.tasks[task.ID] = task
}
break
}
}
// Project the whole batch before launching. This prevents a new worker
// from resurrecting every historical lease during its initial replay.
for taskID, l := range w.leases {
if _, started := w.sessions[taskID]; started {
continue
}
if t, ok := w.tasks[taskID]; ok {
if err := w.start(ctx, t, l.HandoffRef); err != nil {
log.Printf("lease %s: %v", t.ID, err)
_, started := w.sessions[taskID]
class := classifyLaunchError(err, started)
var evidence domain.SessionEvidence
if session, ok := w.sessions[taskID]; ok {
evidence = w.sessionEvidence(ctx, taskID, session)
}
if nackErr := w.api.NackStart(ctx, taskID, l.Epoch, l.Version, class, err.Error(), evidence); nackErr != nil {
w.recordError(fmt.Errorf("nack launch %s: %w", taskID, nackErr))
continue
}
if class != "launch_uncertain" {
delete(w.leases, taskID)
}
}
}
}
// Unit/replay-only workers intentionally have no execution backend. A
// production worker always does, and only then participates in the live
// capture/control protocol.
if w.executionBackend() != nil {
w.publishCaptures(ctx)
w.runCommands(ctx)
w.renewLeases(ctx)
}
w.retryQuarantines(ctx)
w.releaseReady(ctx)
if err := w.save(); err != nil {
return err
}
return w.api.Ack(ctx, w.cursor)
}
func (w *worker) reconcileLeases(ctx context.Context) error {
tasks, err := w.api.Tasks(ctx)
if err != nil {
return fmt.Errorf("reconcile leased tasks: %w", err)
}
active := make(map[string]lease)
for _, task := range tasks {
w.tasks[task.ID] = task
if (task.State == domain.StateLeased || task.State == domain.StateNeedsAttention) && task.Lease != nil && task.Lease.HarnessID == w.harnessID {
active[task.ID] = lease{Epoch: task.Lease.Epoch, HandoffRef: task.HandoffRef, TransactionID: task.ReleaseTransaction, AnchorSHA: task.ReleaseAnchor, Version: task.Version, Until: task.Lease.Until}
}
}
for taskID := range w.leases {
if _, ok := active[taskID]; !ok {
delete(w.leases, taskID)
}
}
for taskID, l := range active {
// The coordinator is authoritative for the lease, not for what this
// worker has observed under it. Rebuilding the struct wholesale wiped
// every worker-local field each tick, which silently disabled the
// renewal progress check and reset the usage baseline. Carry them
// across, but only while the epoch is the same lease.
if prev, ok := w.leases[taskID]; ok && prev.Epoch == l.Epoch {
l.ProgressSHA = prev.ProgressSHA
l.UsageBaseline = prev.UsageBaseline
l.PickupAcknowledged = prev.PickupAcknowledged
}
w.leases[taskID] = l
}
return nil
}
// reRegisterAfterCoordinatorRestart restores the coordinator's in-memory
// worker registry. A worker must survive a server restart without operator
// intervention; its persisted cursor and sessions remain valid.
func (w *worker) reRegisterAfterCoordinatorRestart(ctx context.Context, cause error) bool {
if cause == nil || !strings.Contains(cause.Error(), "401 Unauthorized: unknown worker") {
return false
}
if err := w.api.Register(ctx, w.registration); err != nil {
log.Printf("re-register: %v", err)
return false
}
log.Printf("re-registered after coordinator restart")
return true
}
func required(k string) string {
v := os.Getenv(k)
if v == "" {
log.Fatalf("%s required", k)
}
return v
}
// harnessSpec is one harness identity this process serves. Each spec becomes a
// separate federation identity because the coordinator authorizes lease calls by
// comparing the URL's worker id against the lease's harness id
// (cmd/orchestra/main.go). One process may therefore hold several identities,
// but it may never present one identity for several harnesses.
type harnessSpec struct {
ID string `json:"id"`
Harness string `json:"harness"`
Token string `json:"token,omitempty"`
Backend string `json:"backend,omitempty"`
// Herdr is the JSON-RPC address for Backend "herdr".
Herdr string `json:"herdr,omitempty"`
// TmuxSocket and Command configure Backend "tmux".
TmuxSocket string `json:"tmux_socket,omitempty"`
Command string `json:"command,omitempty"`
State string `json:"state,omitempty"`
Address string `json:"address,omitempty"`
}
// tokenEnvKey maps a harness id onto a per-identity token variable, so a
// multi-harness deployment keeps its tokens in the protected environment file
// rather than in the harness config file.
func tokenEnvKey(id string) string {
var b strings.Builder
b.WriteString("ORCHESTRA_WORKER_TOKEN_")
for _, r := range strings.ToUpper(id) {
if (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
continue
}
b.WriteRune('_')
}
return b.String()
}
// harnessSpecs reads the multi-harness declaration, falling back to the legacy
// single-harness environment so an existing deployment upgrades unchanged.
func harnessSpecs() []harnessSpec {
path := os.Getenv("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE")
if path == "" {
return []harnessSpec{{
ID: required("ORCHESTRA_WORKER_HERDR_ID"),
Harness: required("ORCHESTRA_WORKER_HARNESS"),
Token: required("ORCHESTRA_WORKER_TOKEN"),
Backend: os.Getenv("ORCHESTRA_WORKER_BACKEND"),
Herdr: os.Getenv("ORCHESTRA_WORKER_HERDR"),
TmuxSocket: os.Getenv("ORCHESTRA_WORKER_TMUX_SOCKET"),
Command: os.Getenv("ORCHESTRA_WORKER_HARNESS_COMMAND"),
State: os.Getenv("ORCHESTRA_WORKER_STATE"),
Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"),
}}
}
b, err := os.ReadFile(path)
if err != nil {
log.Fatalf("read ORCHESTRA_WORKER_HARNESS_CONFIG_FILE: %v", err)
}
var specs []harnessSpec
if err := json.Unmarshal(b, &specs); err != nil {
log.Fatalf("parse ORCHESTRA_WORKER_HARNESS_CONFIG_FILE: %v", err)
}
if len(specs) == 0 {
log.Fatal("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE declares no harnesses")
}
seen := map[string]bool{}
for i, spec := range specs {
if spec.ID == "" || spec.Harness == "" {
log.Fatalf("harness %d requires id and harness", i)
}
if seen[spec.ID] {
log.Fatalf("harness id %q is declared twice", spec.ID)
}
seen[spec.ID] = true
if spec.Token == "" {
key := tokenEnvKey(spec.ID)
if spec.Token = os.Getenv(key); spec.Token == "" {
log.Fatalf("harness %s has no token: set %s or its config-file token", spec.ID, key)
}
specs[i] = spec
}
}
return specs
}
// statePathFor gives every identity its own state file. Sharing one across
// harnesses would let a persisted session and its lease cross backends, handing
// a tmux backend herdr pane ids it cannot act on. Only the single-harness form
// keeps the historical default path, so an existing deployment recovers its
// sessions after the upgrade instead of orphaning them.
func statePathFor(spec harnessSpec, root, stateDir string, single bool) string {
switch {
case spec.State != "":
return spec.State
case stateDir != "":
return filepath.Join(stateDir, "state-"+spec.ID+".json")
case single:
return filepath.Join(root, ".orchestra-worker-state.json")
default:
return filepath.Join(root, ".orchestra-worker-state-"+spec.ID+".json")
}
}
// backendFor builds the machine-local execution backend for one harness.
func backendFor(spec harnessSpec) herdr.Backend {
name := strings.ToLower(strings.TrimSpace(spec.Backend))
if name == "" {
name = "herdr"
}
switch name {
case "herdr":
address := spec.Herdr
if address == "" {
log.Fatalf("harness %s: backend herdr requires an address", spec.ID)
}
return herdr.New(address)
case "tmux":
if spec.Harness != "claude" {
log.Fatalf("harness %s: backend tmux currently supports only harness claude, got %q", spec.ID, spec.Harness)
}
return herdr.NewTmuxBackend(spec.TmuxSocket, spec.Command)
default:
log.Fatalf("harness %s: unsupported backend %q (want herdr or tmux)", spec.ID, name)
return nil
}
}
func main() {
workerID := required("ORCHESTRA_WORKER_ID")
hard := .75
if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); err == nil && v > 0 && v < 1 {
hard = v
}
projects := map[string]projectConfig{}
if path := os.Getenv("ORCHESTRA_WORKER_PROJECT_CONFIG_FILE"); path != "" {
b, err := os.ReadFile(path)
if err != nil {
log.Fatalf("read ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err)
}
if err := json.Unmarshal(b, &projects); err != nil {
log.Fatalf("parse ORCHESTRA_WORKER_PROJECT_CONFIG_FILE: %v", err)
}
} else {
for _, project := range strings.Split(os.Getenv("ORCHESTRA_WORKER_PROJECTS"), ",") {
if project = strings.TrimSpace(project); project != "" {
projects[project] = projectConfig{Repo: required("ORCHESTRA_REPO"), Root: required("ORCHESTRA_WORKTREE_ROOT"), Remote: required("ORCHESTRA_GIT_REMOTE")}
}
}
}
if len(projects) == 0 {
// Existing deployments may be upgraded before their protected systemd
// environment is amended. Stay observable and fail closed in that
// interval: an empty declaration makes this worker ineligible for all
// new leases instead of turning a configuration rollout into a crash
// loop or treating its legacy global checkout as every project.
log.Printf("no ORCHESTRA_WORKER_PROJECT_CONFIG_FILE/ORCHESTRA_WORKER_PROJECTS; registering with no supported projects")
}
for project, config := range projects {
if config.Repo == "" || config.Root == "" || config.Remote == "" {
log.Fatalf("project %q requires repo, worktree_root, and remote", project)
}
}
supported := make([]string, 0, len(projects))
for project := range projects {
supported = append(supported, project)
}
sort.Strings(supported)
repo, root, remote := required("ORCHESTRA_REPO"), required("ORCHESTRA_WORKTREE_ROOT"), required("ORCHESTRA_GIT_REMOTE")
if len(supported) > 0 {
first := projects[supported[0]]
repo, root, remote = first.Repo, first.Root, first.Remote
}
soft, window := .55, int64(200000)
if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_SOFT"), 64); err == nil && v > 0 && v < hard {
soft = v
}
if v, err := strconv.ParseInt(os.Getenv("ORCHESTRA_CONTEXT_WINDOW"), 10, 64); err == nil && v > 0 {
window = v
}
url, admit := required("ORCHESTRA_URL"), os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")
specs := harnessSpecs()
if len(specs) == 1 && specs[0].ID != workerID {
// A single-harness deployment keeps the historical invariant: its lease
// owner and its process identity are the same name.
log.Fatal("ORCHESTRA_WORKER_ID must equal ORCHESTRA_WORKER_HERDR_ID so leases and offline recovery have one owner")
}
stateDir := os.Getenv("ORCHESTRA_WORKER_STATE_DIR")
workers := make([]*worker, 0, len(specs))
for _, spec := range specs {
w := &worker{
api: federation.Client{BaseURL: url, WorkerID: spec.ID, Token: spec.Token, AdmitToken: admit},
harnessID: spec.ID,
harness: spec.Harness,
repo: repo,
root: root,
remote: remote,
projects: projects,
tasks: map[string]domain.Task{},
sessions: map[string]herdr.Session{},
leases: map[string]lease{},
releases: map[string]releaseTransaction{},
quarantined: map[string]bool{},
statePath: spec.State,
hard: hard,
soft: soft,
window: window,
// Capacity stays one per identity because a herdr's declared
// concurrency is one. Serving N harnesses gives the process N slots.
registration: federation.Worker{ID: spec.ID, Address: spec.Address, Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()},
}
if w.registration.Address == "" {
w.registration.Address = os.Getenv("ORCHESTRA_WORKER_ADDRESS")
}
w.statePath = statePathFor(spec, root, stateDir, len(specs) == 1)
if err := w.load(); err != nil {
log.Fatal(err)
}
w.backend = backendFor(spec)
if err := w.api.Register(context.Background(), w.registration); err != nil {
log.Fatalf("register %s: %v", spec.ID, err)
}
// Same reason as the coordinator's: the worker half of the pair must be
// checkable from journalctl, not only from the coordinator's
// credentialed worker list.
b := buildinfo.Current()
log.Printf("orchestra-worker revision %s built %s dirty %s", b.Revision, b.Time, b.Dirty)
log.Printf("serving harness %s (%s) on %s backend, state %s", spec.ID, spec.Harness, w.backend.Kind(), w.statePath)
workers = append(workers, w)
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
// Identities are served sequentially. Their checkouts and Git remotes are
// shared, so concurrent ticks would race two fetch/worktree operations on
// one repository for no useful latency gain at this fan-out.
for _, w := range workers {
if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil {
w.recordError(fmt.Errorf("heartbeat: %w", err))
log.Printf("[%s] heartbeat: %v", w.harnessID, err)
if w.reRegisterAfterCoordinatorRestart(ctx, err) {
continue
}
}
if err := w.once(ctx); err != nil {
w.recordError(fmt.Errorf("poll: %w", err))
log.Printf("[%s] poll: %v", w.harnessID, err)
w.reRegisterAfterCoordinatorRestart(ctx, err)
}
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
// federatedTurn is the worker half of a turn boundary. The coordinator
// reconciles human input and answers with the decisions this session has not
// been shown; the worker delivers them into its own pane.
//
// Nothing is preempted. The boundary is confirmed against the live pane
// first, so a correction never lands mid tool call.
func (w *worker) federatedTurn(ctx context.Context, id string, a herdr.Adapter, verdict string) {
l, ok := w.leases[id]
if !ok {
return
}
s, ok := w.sessions[id]
if !ok {
return
}
boundary, ok := a.(herdr.TurnBoundary)
if !ok {
return
}
at, err := boundary.AtTurnBoundary(ctx, s)
if err != nil {
w.recordError(fmt.Errorf("turn boundary %s: %w", id, err))
return
}
if !at {
return
}
// A phase this session no longer runs ends it, whether this worker asked
// for the change or an operator made it (F22). Checked before the request
// below so a session cannot advance a phase twice.
if w.phaseChanged(id, s) {
w.rotateForPhase(ctx, id, a, s)
return
}
// The agent asks for a phase change here, at a boundary it has reached
// (F21). Orchestra decides, and an accepted change ends this session.
if w.requestPhase(ctx, id, s) {
w.rotateForPhase(ctx, id, a, s)
return
}
answer, err := w.api.Turn(ctx, id, l.Epoch, verdict, s.DeliveredDecisions)
if err != nil {
// Observable, not fatal. A coordinator that cannot be reached does not
// make this session's current intent any more stale than it already is.
w.recordError(fmt.Errorf("federated turn %s: %w", id, err))
return
}
if answer.Verdict == orchestrator.TurnPrepareHandoff && !s.HandoffRequested {
// The coordinator has lost the ability to refresh this task's intent.
// Ask for a handoff; the release loop takes over as soon as the agent
// writes the report, exactly as it does for a local session.
requester, ok := a.(herdr.ReasonedHandoffRequester)
if !ok {
w.recordError(fmt.Errorf("reconcile failure handoff %s: adapter cannot state a reason", id))
return
}
if err := requester.RequestHandoffReason(ctx, s, "reconcile_failure", nil); err != nil {
w.recordError(fmt.Errorf("reconcile failure handoff %s: %w", id, err))
return
}
s.HandoffRequested, s.HandoffReason = true, "reconcile_failure"
w.sessions[id] = s
_ = w.save()
return
}
if len(answer.Decisions) == 0 {
return
}
if err := w.sendPrompt(ctx, s, agentctx.DecisionNotice(answer.Decisions)); err != nil {
// Not recorded as delivered, so the next boundary retries.
w.recordError(fmt.Errorf("deliver decisions %s: %w", id, err))
return
}
for _, d := range answer.Decisions {
s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID)
}
w.sessions[id] = s
_ = w.save()
}
// phaseRequestFile is the agent's bounded phase-change intent (F21). Prose in
// the pane is not a request: matching on it would make the protocol depend on
// wording the agent is free to vary, and on Orchestra reading its own echo.
const phaseRequestFile = "phase-request.json"
// phaseRequest is what the agent writes. It states the phase it believes it
// is in as well as the one it wants, so a request written from a stale
// context is refused rather than applied to whatever phase is current.
type phaseRequest struct {
From domain.WorkPhase `json:"from"`
To domain.WorkPhase `json:"to"`
}
// phaseArtifact names the sealed output each phase must produce before it may
// be left. Phases absent from this table seal nothing.
var phaseArtifact = map[domain.WorkPhase]string{
domain.WorkPhaseResearch: "research.json",
domain.WorkPhasePlan: "plan.json",
}
func currentPhase(t domain.Task) domain.WorkPhase {
if t.WorkPhase == "" {
return domain.WorkPhaseFrame
}
return t.WorkPhase
}
// phaseChanged reports whether this session is running a phase the task has
// since left. It covers a change this worker requested and one an operator
// made through the coordinator equally, because both leave the same evidence:
// a session whose context was built for a phase that is no longer current.
func (w *worker) phaseChanged(id string, s herdr.Session) bool {
t, ok := w.tasks[id]
if !ok || s.Phase == "" {
return false
}
return string(currentPhase(t)) != s.Phase
}
// rotateForPhase ends the current cognitive session because the phase moved
// (F22). A phase change is a change of context, not of instruction: leaving
// the old agent running would either waste the lease waiting for it to idle
// out, as run 3 did, or let it keep working under a brief that no longer
// applies.
func (w *worker) rotateForPhase(ctx context.Context, id string, a herdr.Adapter, s herdr.Session) {
if s.HandoffRequested {
return
}
requester, ok := a.(herdr.ReasonedHandoffRequester)
if !ok {
w.recordError(fmt.Errorf("phase rotation %s: adapter cannot state a reason", id))
return
}
if err := requester.RequestHandoffReason(ctx, s, "phase_changed", nil); err != nil {
w.recordError(fmt.Errorf("phase rotation %s: %w", id, err))
return
}
s.HandoffRequested, s.HandoffReason = true, "phase_changed"
w.sessions[id] = s
_ = w.save()
log.Printf("phase changed for %s: session rotating", id)
}
// answerRefusedPhase tells the agent why its request was refused and drops the
// file so it can write a corrected one. The request survives a failed send, so
// the refusal is delivered at the next boundary instead of being lost.
func (w *worker) answerRefusedPhase(ctx context.Context, id string, s herdr.Session, path, reason string) {
text := "Orchestra refused your phase request: " + reason +
"\n\nWrite a corrected .orchestra/phase-request.json, or keep working in the current phase. Do not repeat the refused request."
if err := w.sendPrompt(ctx, s, text); err != nil {
w.recordError(fmt.Errorf("deliver phase refusal %s: %w", id, err))
return
}
if err := os.Remove(path); err != nil {
w.recordError(fmt.Errorf("phase request %s: %w", id, err))
}
log.Printf("phase request %s refused: %s", id, reason)
}
// requestPhase carries an agent's phase request to the coordinator (F21).
// It reports whether the phase moved.
//
// Everything checkable locally is checked before the call, so an agent that
// asked for the wrong thing learns it from a recorded error rather than from
// a lease that quietly stops being renewed.
func (w *worker) requestPhase(ctx context.Context, id string, s herdr.Session) bool {
path := filepath.Join(s.Worktree, ".orchestra", phaseRequestFile)
b, err := os.ReadFile(path)
if err != nil {
return false
}
var req phaseRequest
if err := json.Unmarshal(b, &req); err != nil {
w.recordError(fmt.Errorf("phase request %s: %w", id, err))
return false
}
t, ok := w.tasks[id]
if !ok {
return false
}
if req.From != currentPhase(t) {
w.recordError(fmt.Errorf("phase request %s: task is in work phase %q, not %q", id, currentPhase(t), req.From))
return false
}
if !domain.CanTransitionPhase(req.From, req.To) {
w.recordError(fmt.Errorf("phase request %s: %q to %q is not a legal transition", id, req.From, req.To))
return false
}
// The phase being left seals its result before it may be left. Decoding
// here means a malformed artifact is reported against the agent that
// wrote it, while its session is still alive to be told.
var artifact []byte
if name := phaseArtifact[req.From]; name != "" {
artifact, err = os.ReadFile(filepath.Join(s.Worktree, ".orchestra", name))
if err != nil {
w.recordError(fmt.Errorf("phase request %s: work phase %q must seal .orchestra/%s first: %w", id, req.From, name, err))
w.answerRefusedPhase(ctx, id, s, path, fmt.Sprintf("work phase %q must seal .orchestra/%s first: %v", req.From, name, err))
return false
}
var decErr error
switch req.From {
case domain.WorkPhaseResearch:
_, decErr = workphase.DecodeResearch(artifact)
case domain.WorkPhasePlan:
_, decErr = workphase.DecodePlan(artifact)
}
if decErr != nil {
w.recordError(fmt.Errorf("phase request %s: %s artifact: %w", id, req.From, decErr))
// A local refusal is still a refusal, and the agent is the only
// party that can fix it. Recording it in worker health alone left
// a live session parked at a boundary forever with nothing telling
// it what was wrong (F39) — the silent-loop shape the comment
// above this block warns about, reached by the one path that had
// no delivery.
w.answerRefusedPhase(ctx, id, s, path, fmt.Sprintf(".orchestra/%s does not match the schema: %v", name, decErr))
return false
}
}
l := w.leases[id]
// Derived, not random: a redelivery after a lost response must carry the
// same id so the coordinator recognises it instead of advancing twice.
op := "phase:" + id + ":" + l.Epoch + ":" + string(req.From) + ":" + string(req.To)
phase, err := w.api.AdvancePhase(ctx, id, l.Epoch, op, req.From, req.To, artifact)
if err != nil {
w.recordError(fmt.Errorf("phase request %s: %w", id, err))
// A refusal is an answer, and it names the phase the agent may ask
// for. Recording it only in worker health would leave the agent
// rewriting the same rejected file at every boundary with nothing
// telling it why, which is the silent-loop shape this codebase keeps
// producing. A transport failure is not an answer and is retried.
var status *federation.StatusError
if errors.As(err, &status) && status.Code == http.StatusConflict {
w.answerRefusedPhase(ctx, id, s, path, status.Body)
}
return false
}
if phase == "" {
// Accepted but not advanced: the coordinator raised a trajectory gate
// and the human now owns the move. Keep the request so the same ask is
// re-sent, under the same operation id, once the gate clears.
return false
}
// Durable before the file is removed. A removal that raced the response
// would lose the request and leave the agent waiting on an answer that
// already arrived.
if err := os.Remove(path); err != nil {
w.recordError(fmt.Errorf("phase request %s: %w", id, err))
}
if phase == domain.WorkPhaseReview {
// Each entry into review starts without the previous cycle's findings.
// The worktree survives a changes-requested round trip, so a reviewer
// that writes .orchestra/done without rewriting the file would have
// the earlier review sealed against the new commit — and submit binds
// whatever it reads to the commit it is submitting, so a stale pass
// would look exactly like a fresh one.
if err := os.Remove(filepath.Join(s.Worktree, ".orchestra", reviewFile)); err != nil && !os.IsNotExist(err) {
w.recordError(fmt.Errorf("clear stale review %s: %w", id, err))
}
}
t.WorkPhase = phase
w.tasks[id] = t
log.Printf("phase request %s accepted: %s to %s", id, req.From, phase)
return true
}
// sendPrompt delivers Orchestra-originated input and confirms the harness took
// it. A phase continuation or a decision notice whose Enter is lost strands the
// session exactly as a lost launch does.
func (w *worker) sendPrompt(ctx context.Context, s herdr.Session, text string) error {
backend := w.executionBackend()
if backend == nil {
return fmt.Errorf("execution backend is not configured")
}
if err := backend.Prompt(ctx, s.PaneID, text, time.Minute); err != nil {
return err
}
return w.confirmInput(ctx, s, text)
}
// confirmInput is the one place Orchestra proves a write landed. A backend
// whose own protocol acknowledges input does not implement InputConfirmer and
// needs no second opinion.
func (w *worker) confirmInput(ctx context.Context, s herdr.Session, text string) error {
c, ok := w.executionBackend().(herdr.InputConfirmer)
if !ok {
return nil
}
evidence, err := c.ConfirmInput(ctx, s, text)
if err != nil {
return err
}
log.Printf("input to %s confirmed: %s", s.PaneID, evidence)
return nil
}
// paneProgress reports pane content with input lines removed where the backend
// can separate them, and falls back to the raw capture where it cannot.
func (w *worker) paneProgress(ctx context.Context, adapter herdr.CLIAdapter, s herdr.Session) (string, error) {
if p, ok := w.executionBackend().(herdr.PaneProgress); ok {
return p.PaneProgress(ctx, s)
}
return adapter.PaneCapture(ctx, s, "recent")
}