v3 workflow: intent, phases, review, submission, enforcement, burn-in
The v3 stack, previously an uncommitted working tree, plus this session's two units and the burn-in instrument. This commit is the burn-in build identity: coordinator and worker must both report this revision before a task is created. Workflow (earlier sessions, uncommitted until now): human decision events and reduction, source cursors and reconcile-before-launch, turn-boundary reconciliation, internal/agentctx as the single renderer, ace-fca phases with sealed artifacts, the trajectory gate, bounded grilling, independent review, task pr enforcement, and human review reflection. Capability restrictions at the agent boundary: an authz.Agent surface at GatedWrite may ask and may not act. It also fixes two bugs the unit exposed -- gated surfaces could not reach the two endpoints written for them, and RequestHumanDecision would block an unowned task while rejecting a question from the session that did own it. Turn-boundary reconcile-failure escalation: a streak of consecutive failures asks the session to hand off, fenced on the lease epoch, with reconcile_failure as a real handoff reason. The worker was dropping the coordinator's verdict on the floor; it now acts on it. Burn-in: herdr.WriteLaunchContext dumps the exact agentctx.Build result to <worktree>/.orchestra/launch.md at every launch, local and federated. BURNIN.md is the runbook. deploy/build.sh stamps both binaries from one commit. go build, go vet and go test ./... pass, 20 packages. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+497
-57
@@ -1,6 +1,9 @@
|
||||
// orchestra-worker consumes router-issued leases for one local herdr. Homesrv
|
||||
// remains the scheduler and CAS authority; this process owns only local Git
|
||||
// and pane operations.
|
||||
// orchestra-worker consumes router-issued leases for one or more local
|
||||
// execution backends. Each declared harness is a separate federation identity
|
||||
// with its own token, cursor, backend, and state file, because the coordinator
|
||||
// authorizes a lease call by comparing the URL's worker id against the lease's
|
||||
// harness id. Homesrv remains the scheduler and CAS authority. This process
|
||||
// owns only local Git and pane operations.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -9,16 +12,20 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"orchestra/internal/agentctx"
|
||||
"orchestra/internal/buildinfo"
|
||||
"orchestra/internal/continuity"
|
||||
"orchestra/internal/domain"
|
||||
"orchestra/internal/federation"
|
||||
"orchestra/internal/herdr"
|
||||
"orchestra/internal/orchestrator"
|
||||
"orchestra/internal/review"
|
||||
"orchestra/internal/workphase"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -27,7 +34,11 @@ import (
|
||||
)
|
||||
|
||||
type worker struct {
|
||||
api federation.Client
|
||||
api federation.Client
|
||||
backend herdr.Backend
|
||||
// herdr is retained as a test/backward-compatibility alias. Production
|
||||
// workers set backend; executionBackend keeps older state-machine tests
|
||||
// from needing protocol-irrelevant rewrites.
|
||||
herdr *herdr.Client
|
||||
harnessID, harness, repo, root, remote string
|
||||
projects map[string]projectConfig
|
||||
@@ -46,6 +57,16 @@ type worker struct {
|
||||
window int64
|
||||
}
|
||||
|
||||
func (w *worker) executionBackend() herdr.Backend {
|
||||
if w.backend != nil {
|
||||
return w.backend
|
||||
}
|
||||
if w.herdr != nil {
|
||||
return w.herdr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *worker) recordError(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
@@ -56,6 +77,9 @@ func (w *worker) recordError(err error) {
|
||||
|
||||
func (w *worker) health(ctx context.Context) federation.WorkerHealth {
|
||||
h := federation.WorkerHealth{HerdrStatus: "unknown"}
|
||||
if backend := w.executionBackend(); backend != nil {
|
||||
h.Backend = backend.Kind()
|
||||
}
|
||||
for taskID, session := range w.sessions {
|
||||
// Workers currently advertise capacity one. Pick deterministically so a
|
||||
// recovered legacy state with more sessions remains intelligible.
|
||||
@@ -63,16 +87,16 @@ func (w *worker) health(ctx context.Context) federation.WorkerHealth {
|
||||
h.ActiveTask, h.ActivePane = taskID, session.PaneID
|
||||
}
|
||||
}
|
||||
if w.herdr != nil {
|
||||
if backend := w.executionBackend(); backend != nil {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
err := w.herdr.CheckProtocol(checkCtx, "17")
|
||||
err := backend.Check(checkCtx)
|
||||
cancel()
|
||||
h.CheckedAt = time.Now().UTC()
|
||||
if err == nil {
|
||||
h.HerdrStatus = "reachable"
|
||||
} else {
|
||||
h.HerdrStatus = "unreachable"
|
||||
w.recordError(fmt.Errorf("local herdr: %w", err))
|
||||
w.recordError(fmt.Errorf("local %s backend: %w", backend.Kind(), err))
|
||||
}
|
||||
}
|
||||
h.LastError, h.ErrorAt = w.lastError, w.lastErrorAt
|
||||
@@ -200,11 +224,11 @@ func (w *worker) save() error {
|
||||
// A failed close remains durable and is retried; it is never treated as a
|
||||
// harmless cleanup error while the old harness could still be working.
|
||||
func (w *worker) quarantine(ctx context.Context, taskID string, s herdr.Session) {
|
||||
if w.herdr == nil {
|
||||
if w.executionBackend() == nil {
|
||||
w.quarantined[taskID] = true
|
||||
return
|
||||
}
|
||||
if err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).Kill(ctx, s); err != nil {
|
||||
if err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).Kill(ctx, s); err != nil {
|
||||
w.quarantined[taskID] = true
|
||||
w.recordError(fmt.Errorf("quarantine %s: %w", taskID, err))
|
||||
return
|
||||
@@ -316,28 +340,98 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = w.herdr.Worktree(ctx, p.Repo, wt, "orchestra/"+t.ID); err != nil {
|
||||
backend := w.executionBackend()
|
||||
if backend == nil {
|
||||
return fmt.Errorf("execution backend is not configured")
|
||||
}
|
||||
if _, err = backend.Worktree(ctx, p.Repo, wt, "orchestra/"+t.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := w.herdr.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID)
|
||||
s, err := backend.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.TaskFileSHA = taskHash(t)
|
||||
prompt := "Read TASK.md at the worktree root and execute it."
|
||||
if len(p.SafeOperations) > 0 {
|
||||
prompt += " This project's audited no-grant policy permits only worktree-local " + strings.Join(p.SafeOperations, ", ") + ". Network, secrets, destructive actions, and paths outside this worktree still require an explicit operator approval."
|
||||
if w.harness == "claude" {
|
||||
s.ContextHandoffSHA, _ = fileSHA256(filepath.Join(wt, "HANDOFF.md"))
|
||||
}
|
||||
// One renderer, on both machines. The worker fetches the reduced
|
||||
// authority as data and renders it with agentctx, so a decision the human
|
||||
// recorded before this session existed is visible from its first turn.
|
||||
intent, err := w.api.Intent(ctx, t.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("effective intent: %w", err)
|
||||
}
|
||||
in := agentctx.Input{
|
||||
Task: t, Intent: intent, Phase: t.WorkPhase, DecisionRequest: t.DecisionRequest,
|
||||
Git: agentctx.GitState{Worktree: wt, Branch: "orchestra/" + t.ID},
|
||||
RepoRules: agentctx.DiscoverRepoRules(wt),
|
||||
}
|
||||
if sha, shaErr := herdr.HeadSHA(wt); shaErr == nil {
|
||||
in.Git.HeadSHA = sha
|
||||
}
|
||||
if ref != "" {
|
||||
prompt += " A validated handoff exists; inspect local Git history and the recorded checkpoint before continuing."
|
||||
in.Handoff = &h
|
||||
}
|
||||
if len(p.SafeOperations) > 0 {
|
||||
in.Policy = []string{
|
||||
"Permitted without an operator grant, inside this worktree only: " + strings.Join(p.SafeOperations, ", ") + ".",
|
||||
"Network access, secrets, destructive actions, and paths outside this worktree require an explicit operator approval.",
|
||||
}
|
||||
}
|
||||
if t.ResearchRef != "" {
|
||||
b, artErr := w.api.Artifact(ctx, t.ResearchRef)
|
||||
if artErr != nil {
|
||||
return fmt.Errorf("research artifact: %w", artErr)
|
||||
}
|
||||
r, decErr := workphase.DecodeResearch(b)
|
||||
if decErr != nil {
|
||||
return fmt.Errorf("research artifact: %w", decErr)
|
||||
}
|
||||
in.Research = &r
|
||||
}
|
||||
if t.PlanRef != "" {
|
||||
b, artErr := w.api.Artifact(ctx, t.PlanRef)
|
||||
if artErr != nil {
|
||||
return fmt.Errorf("plan artifact: %w", artErr)
|
||||
}
|
||||
pl, decErr := workphase.DecodePlan(b)
|
||||
if decErr != nil {
|
||||
return fmt.Errorf("plan artifact: %w", decErr)
|
||||
}
|
||||
in.Plan = &pl
|
||||
}
|
||||
if t.Review != nil {
|
||||
b, artErr := w.api.Artifact(ctx, t.Review.ArtifactRef)
|
||||
if artErr != nil {
|
||||
return fmt.Errorf("review artifact: %w", artErr)
|
||||
}
|
||||
r, decErr := review.Decode(b)
|
||||
if decErr != nil {
|
||||
return fmt.Errorf("review artifact: %w", decErr)
|
||||
}
|
||||
in.Review = &r
|
||||
}
|
||||
built, err := agentctx.Build(in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build context: %w", err)
|
||||
}
|
||||
prompt := built.System + "\n\n" + built.Task
|
||||
if writeErr := herdr.WriteLaunchContext(s.Worktree, prompt); writeErr != nil {
|
||||
w.recordError(fmt.Errorf("launch context %s: %w", t.ID, writeErr))
|
||||
}
|
||||
// The launch instruction carried these, so the first turn boundary must
|
||||
// not re-announce them as news.
|
||||
for _, d := range intent.Decisions {
|
||||
s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID)
|
||||
}
|
||||
w.sessions[t.ID] = s
|
||||
if err := w.save(); err != nil {
|
||||
return err
|
||||
}
|
||||
// A prompt response can be lost after herdr accepted it. Persist the
|
||||
// A prompt response can be lost after the backend accepted it. Persist the
|
||||
// session first so the worker can reconcile/release it after restart.
|
||||
if err := w.herdr.Prompt(ctx, s.PaneID, prompt, 0); err != nil {
|
||||
if err := backend.Prompt(ctx, s.PaneID, prompt, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
if l, ok := w.leases[t.ID]; ok {
|
||||
@@ -371,6 +465,14 @@ func classifyLaunchError(err error, sessionStarted bool) string {
|
||||
|
||||
func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) }
|
||||
|
||||
func fileSHA256(path string) (string, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return domain.Hash(b), nil
|
||||
}
|
||||
|
||||
func (w *worker) releaseReady(ctx context.Context) {
|
||||
for id, s := range w.sessions {
|
||||
if w.quarantined[id] {
|
||||
@@ -385,7 +487,7 @@ func (w *worker) releaseReady(ctx context.Context) {
|
||||
if _, err := os.Stat(filepath.Join(s.Worktree, ".orchestra", "done")); err == nil {
|
||||
// A done marker is an intent, not enough on its own: do not race a
|
||||
// still-running native harness into committing half-written work.
|
||||
status, statusErr := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).AgentStatus(ctx, s)
|
||||
status, statusErr := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).AgentStatus(ctx, s)
|
||||
if statusErr != nil {
|
||||
w.recordError(fmt.Errorf("completion identity %s: %w", id, statusErr))
|
||||
continue
|
||||
@@ -414,7 +516,7 @@ func (w *worker) releaseReady(ctx context.Context) {
|
||||
}
|
||||
// Completion is durable before closing the exact pane. If close
|
||||
// fails, retain the session mapping for a later explicit cleanup.
|
||||
a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}
|
||||
a := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}
|
||||
if err := a.Kill(ctx, s); err != nil {
|
||||
w.recordError(fmt.Errorf("close completed pane %s: %w", id, err))
|
||||
log.Printf("close completed pane %s: %v", id, err)
|
||||
@@ -436,7 +538,7 @@ func (w *worker) releaseReady(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (w *worker) adapter(s herdr.Session, remote string) herdr.CLIAdapter {
|
||||
a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness, Window: w.window, CAS: artifactCAS{w.api}, Remote: remote}
|
||||
a := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness, Window: w.window, CAS: artifactCAS{w.api}, Remote: remote}
|
||||
switch w.harness {
|
||||
case "claude":
|
||||
a.Usage = herdr.ClaudeUsage
|
||||
@@ -452,6 +554,18 @@ func (w *worker) adapter(s herdr.Session, remote string) herdr.CLIAdapter {
|
||||
// and pane status are all read from the persisted harness session identity;
|
||||
// any unknown source is recorded and never treated as zero usage.
|
||||
func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
|
||||
// Claude Code owns its context threshold through the installed
|
||||
// context-handoff hook. A changed HANDOFF.md means that hook has landed a
|
||||
// durable local continuation. Resume in the same process with Claude's
|
||||
// native context reset instead of manufacturing Orchestra's cross-worker
|
||||
// release artifact. Codex and OpenCode continue through the existing
|
||||
// occupancy/release state machine below.
|
||||
if w.harness == "claude" {
|
||||
if err := w.advanceClaudeContextReset(ctx, id, s); err != nil {
|
||||
w.recordError(fmt.Errorf("Claude context reset %s: %w", id, err))
|
||||
}
|
||||
return
|
||||
}
|
||||
t, ok := w.tasks[id]
|
||||
if !ok {
|
||||
w.recordError(fmt.Errorf("rotation %s: task cache missing", id))
|
||||
@@ -468,7 +582,9 @@ func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
|
||||
w.recordError(fmt.Errorf("rotation %s occupancy degraded: %w", id, err))
|
||||
return
|
||||
}
|
||||
if resolved != s {
|
||||
// DeepEqual, not !=: Session carries a slice since decisions are tracked
|
||||
// per session, so it is no longer comparable with ==.
|
||||
if !reflect.DeepEqual(resolved, s) {
|
||||
w.sessions[id] = resolved
|
||||
s = resolved
|
||||
_ = w.save()
|
||||
@@ -483,7 +599,11 @@ func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if d.Action == orchestrator.TurnContinue || d.Action == orchestrator.TurnRefuse || s.HandoffRequested {
|
||||
if d.Action == orchestrator.TurnContinue || d.Action == "" {
|
||||
w.federatedTurn(ctx, id, a, orchestrator.TurnContinue)
|
||||
return
|
||||
}
|
||||
if d.Action == orchestrator.TurnRefuse || s.HandoffRequested {
|
||||
return
|
||||
}
|
||||
if d.Reason == "milestone" || d.Reason == "thrash" {
|
||||
@@ -500,6 +620,86 @@ func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
|
||||
_ = w.save()
|
||||
}
|
||||
|
||||
func (w *worker) sendLine(ctx context.Context, s herdr.Session, line string) error {
|
||||
backend := w.executionBackend()
|
||||
if backend == nil {
|
||||
return fmt.Errorf("execution backend is not configured")
|
||||
}
|
||||
if err := backend.SendText(ctx, s, line); err != nil {
|
||||
return fmt.Errorf("send %q: %w", line, err)
|
||||
}
|
||||
if err := backend.SendKeys(ctx, s, []string{"ENTER"}); err != nil {
|
||||
return fmt.Errorf("submit %q: %w", line, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *worker) advanceClaudeContextReset(ctx context.Context, id string, s herdr.Session) error {
|
||||
backend := w.executionBackend()
|
||||
if backend == nil {
|
||||
return fmt.Errorf("execution backend is not configured")
|
||||
}
|
||||
if s.ContextResetSHA == "" {
|
||||
sha, err := fileSHA256(filepath.Join(s.Worktree, "HANDOFF.md"))
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read HANDOFF.md: %w", err)
|
||||
}
|
||||
if sha == s.ContextHandoffSHA {
|
||||
return nil
|
||||
}
|
||||
status, err := backend.AgentStatus(ctx, s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("confirm Claude stopped after handoff: %w", err)
|
||||
}
|
||||
if status != "idle" {
|
||||
return nil
|
||||
}
|
||||
s.ContextResetSHA = sha
|
||||
s.ContextResetPhase = "clear"
|
||||
w.sessions[id] = s
|
||||
if err := w.save(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
switch s.ContextResetPhase {
|
||||
case "clear":
|
||||
if err := w.sendLine(ctx, s, "/clear"); err != nil {
|
||||
return err
|
||||
}
|
||||
s.ContextResetPhase = "handoff"
|
||||
w.sessions[id] = s
|
||||
if err := w.save(); err != nil {
|
||||
return err
|
||||
}
|
||||
// /clear redraws Claude's input UI asynchronously. Give it a small,
|
||||
// bounded interval before submitting the new-session file mention.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
fallthrough
|
||||
case "handoff":
|
||||
if err := w.sendLine(ctx, s, "@HANDOFF.md"); err != nil {
|
||||
return err
|
||||
}
|
||||
s.ContextHandoffSHA = s.ContextResetSHA
|
||||
s.ContextResetSHA = ""
|
||||
s.ContextResetPhase = ""
|
||||
// Claude normally opens a fresh transcript for /clear. Force the next
|
||||
// observation to discover it instead of retaining the exhausted path.
|
||||
s.SessionFile = ""
|
||||
w.sessions[id] = s
|
||||
return w.save()
|
||||
default:
|
||||
return fmt.Errorf("unknown persisted context-reset phase %q", s.ContextResetPhase)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *worker) advanceRelease(ctx context.Context, id string, s herdr.Session) {
|
||||
t, ok := w.tasks[id]
|
||||
if !ok {
|
||||
@@ -604,7 +804,7 @@ func (w *worker) ackPickup(ctx context.Context, id string, s herdr.Session) erro
|
||||
|
||||
func (w *worker) sessionEvidence(ctx context.Context, taskID string, s herdr.Session) domain.SessionEvidence {
|
||||
e := domain.SessionEvidence{PaneID: s.PaneID, HarnessID: w.harnessID, PaneState: "open", Source: "worker", CheckedAt: time.Now().UTC()}
|
||||
text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent")
|
||||
text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, s, "recent")
|
||||
if err != nil {
|
||||
e.PaneState = "unreachable"
|
||||
return e
|
||||
@@ -710,7 +910,7 @@ func (w *worker) finalize(ctx context.Context, id string, s herdr.Session) (comp
|
||||
}
|
||||
|
||||
func (w *worker) renewLeases(ctx context.Context) {
|
||||
if w.herdr == nil {
|
||||
if w.executionBackend() == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
@@ -719,7 +919,7 @@ func (w *worker) renewLeases(ctx context.Context) {
|
||||
if !ok || l.Version == 0 || l.Until.After(now.Add(10*time.Minute)) {
|
||||
continue
|
||||
}
|
||||
if _, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent"); err != nil {
|
||||
if _, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, s, "recent"); err != nil {
|
||||
w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err))
|
||||
continue
|
||||
}
|
||||
@@ -741,7 +941,7 @@ func (w *worker) renewLeases(ctx context.Context) {
|
||||
// coordinator to touch their unix herdr socket.
|
||||
func (w *worker) publishCaptures(ctx context.Context) {
|
||||
for taskID, session := range w.sessions {
|
||||
text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, session, "recent")
|
||||
text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, session, "recent")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -787,7 +987,7 @@ func (w *worker) runCommands(ctx context.Context) {
|
||||
_ = w.api.ResolveCommand(ctx, command.ID, "stale", "session or pane changed")
|
||||
continue
|
||||
}
|
||||
text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, session, "recent")
|
||||
text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, session, "recent")
|
||||
if err != nil {
|
||||
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "capture unavailable: "+err.Error())
|
||||
continue
|
||||
@@ -806,12 +1006,15 @@ func (w *worker) runCommands(ctx context.Context) {
|
||||
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "prompt does not expose an executable approval control")
|
||||
continue
|
||||
}
|
||||
method, params := "pane.send_text", map[string]any{"pane_id": session.PaneID, "text": input.Text}
|
||||
backend := w.executionBackend()
|
||||
var inputErr error
|
||||
if len(input.Keys) > 0 {
|
||||
method, params = "pane.send_keys", map[string]any{"pane_id": session.PaneID, "keys": input.Keys}
|
||||
inputErr = backend.SendKeys(ctx, session, input.Keys)
|
||||
} else {
|
||||
inputErr = backend.SendText(ctx, session, input.Text)
|
||||
}
|
||||
if err := w.herdr.Call(ctx, method, params, nil); err != nil {
|
||||
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "herdr did not acknowledge input: "+err.Error())
|
||||
if inputErr != nil {
|
||||
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", backend.Kind()+" backend did not acknowledge input: "+inputErr.Error())
|
||||
continue
|
||||
}
|
||||
if err := w.api.ResolveCommand(ctx, command.ID, "acknowledged", ""); err != nil {
|
||||
@@ -875,9 +1078,9 @@ func (w *worker) once(ctx context.Context) error {
|
||||
if e.Type == "TaskCompleted" {
|
||||
delete(w.leases, e.TaskID)
|
||||
if session, active := w.sessions[e.TaskID]; active {
|
||||
if w.herdr == nil {
|
||||
if w.executionBackend() == nil {
|
||||
delete(w.sessions, e.TaskID)
|
||||
} else if err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).Kill(ctx, session); err != nil {
|
||||
} else if err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).Kill(ctx, session); err != nil {
|
||||
w.quarantined[e.TaskID] = true
|
||||
w.recordError(fmt.Errorf("close completed pane %s: %w", e.TaskID, err))
|
||||
continue
|
||||
@@ -979,10 +1182,10 @@ func (w *worker) once(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Unit/replay-only workers intentionally have no herdr connection. A
|
||||
// Unit/replay-only workers intentionally have no execution backend. A
|
||||
// production worker always does, and only then participates in the live
|
||||
// capture/control protocol.
|
||||
if w.herdr != nil {
|
||||
if w.executionBackend() != nil {
|
||||
w.publishCaptures(ctx)
|
||||
w.runCommands(ctx)
|
||||
w.renewLeases(ctx)
|
||||
@@ -1039,8 +1242,134 @@ func required(k string) string {
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// harnessSpec is one harness identity this process serves. Each spec becomes a
|
||||
// separate federation identity because the coordinator authorizes lease calls by
|
||||
// comparing the URL's worker id against the lease's harness id
|
||||
// (cmd/orchestra/main.go). One process may therefore hold several identities,
|
||||
// but it may never present one identity for several harnesses.
|
||||
type harnessSpec struct {
|
||||
ID string `json:"id"`
|
||||
Harness string `json:"harness"`
|
||||
Token string `json:"token,omitempty"`
|
||||
Backend string `json:"backend,omitempty"`
|
||||
// Herdr is the JSON-RPC address for Backend "herdr".
|
||||
Herdr string `json:"herdr,omitempty"`
|
||||
// TmuxSocket and Command configure Backend "tmux".
|
||||
TmuxSocket string `json:"tmux_socket,omitempty"`
|
||||
Command string `json:"command,omitempty"`
|
||||
State string `json:"state,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
}
|
||||
|
||||
// tokenEnvKey maps a harness id onto a per-identity token variable, so a
|
||||
// multi-harness deployment keeps its tokens in the protected environment file
|
||||
// rather than in the harness config file.
|
||||
func tokenEnvKey(id string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("ORCHESTRA_WORKER_TOKEN_")
|
||||
for _, r := range strings.ToUpper(id) {
|
||||
if (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
b.WriteRune('_')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// harnessSpecs reads the multi-harness declaration, falling back to the legacy
|
||||
// single-harness environment so an existing deployment upgrades unchanged.
|
||||
func harnessSpecs() []harnessSpec {
|
||||
path := os.Getenv("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE")
|
||||
if path == "" {
|
||||
return []harnessSpec{{
|
||||
ID: required("ORCHESTRA_WORKER_HERDR_ID"),
|
||||
Harness: required("ORCHESTRA_WORKER_HARNESS"),
|
||||
Token: required("ORCHESTRA_WORKER_TOKEN"),
|
||||
Backend: os.Getenv("ORCHESTRA_WORKER_BACKEND"),
|
||||
Herdr: os.Getenv("ORCHESTRA_WORKER_HERDR"),
|
||||
TmuxSocket: os.Getenv("ORCHESTRA_WORKER_TMUX_SOCKET"),
|
||||
Command: os.Getenv("ORCHESTRA_WORKER_HARNESS_COMMAND"),
|
||||
State: os.Getenv("ORCHESTRA_WORKER_STATE"),
|
||||
Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"),
|
||||
}}
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
log.Fatalf("read ORCHESTRA_WORKER_HARNESS_CONFIG_FILE: %v", err)
|
||||
}
|
||||
var specs []harnessSpec
|
||||
if err := json.Unmarshal(b, &specs); err != nil {
|
||||
log.Fatalf("parse ORCHESTRA_WORKER_HARNESS_CONFIG_FILE: %v", err)
|
||||
}
|
||||
if len(specs) == 0 {
|
||||
log.Fatal("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE declares no harnesses")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for i, spec := range specs {
|
||||
if spec.ID == "" || spec.Harness == "" {
|
||||
log.Fatalf("harness %d requires id and harness", i)
|
||||
}
|
||||
if seen[spec.ID] {
|
||||
log.Fatalf("harness id %q is declared twice", spec.ID)
|
||||
}
|
||||
seen[spec.ID] = true
|
||||
if spec.Token == "" {
|
||||
key := tokenEnvKey(spec.ID)
|
||||
if spec.Token = os.Getenv(key); spec.Token == "" {
|
||||
log.Fatalf("harness %s has no token: set %s or its config-file token", spec.ID, key)
|
||||
}
|
||||
specs[i] = spec
|
||||
}
|
||||
}
|
||||
return specs
|
||||
}
|
||||
|
||||
// statePathFor gives every identity its own state file. Sharing one across
|
||||
// harnesses would let a persisted session and its lease cross backends, handing
|
||||
// a tmux backend herdr pane ids it cannot act on. Only the single-harness form
|
||||
// keeps the historical default path, so an existing deployment recovers its
|
||||
// sessions after the upgrade instead of orphaning them.
|
||||
func statePathFor(spec harnessSpec, root, stateDir string, single bool) string {
|
||||
switch {
|
||||
case spec.State != "":
|
||||
return spec.State
|
||||
case stateDir != "":
|
||||
return filepath.Join(stateDir, "state-"+spec.ID+".json")
|
||||
case single:
|
||||
return filepath.Join(root, ".orchestra-worker-state.json")
|
||||
default:
|
||||
return filepath.Join(root, ".orchestra-worker-state-"+spec.ID+".json")
|
||||
}
|
||||
}
|
||||
|
||||
// backendFor builds the machine-local execution backend for one harness.
|
||||
func backendFor(spec harnessSpec) herdr.Backend {
|
||||
name := strings.ToLower(strings.TrimSpace(spec.Backend))
|
||||
if name == "" {
|
||||
name = "herdr"
|
||||
}
|
||||
switch name {
|
||||
case "herdr":
|
||||
address := spec.Herdr
|
||||
if address == "" {
|
||||
log.Fatalf("harness %s: backend herdr requires an address", spec.ID)
|
||||
}
|
||||
return herdr.New(address)
|
||||
case "tmux":
|
||||
if spec.Harness != "claude" {
|
||||
log.Fatalf("harness %s: backend tmux currently supports only harness claude, got %q", spec.ID, spec.Harness)
|
||||
}
|
||||
return herdr.NewTmuxBackend(spec.TmuxSocket, spec.Command)
|
||||
default:
|
||||
log.Fatalf("harness %s: unsupported backend %q (want herdr or tmux)", spec.ID, name)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
id, token := required("ORCHESTRA_WORKER_ID"), required("ORCHESTRA_WORKER_TOKEN")
|
||||
workerID := required("ORCHESTRA_WORKER_ID")
|
||||
hard := .75
|
||||
if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); err == nil && v > 0 && v < 1 {
|
||||
hard = v
|
||||
@@ -1091,36 +1420,72 @@ func main() {
|
||||
if v, err := strconv.ParseInt(os.Getenv("ORCHESTRA_CONTEXT_WINDOW"), 10, 64); err == nil && v > 0 {
|
||||
window = v
|
||||
}
|
||||
w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: repo, root: root, remote: remote, projects: projects, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, releases: map[string]releaseTransaction{}, quarantined: map[string]bool{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, soft: soft, window: window, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()}}
|
||||
if w.statePath == "" {
|
||||
w.statePath = filepath.Join(w.root, ".orchestra-worker-state.json")
|
||||
}
|
||||
if err := w.load(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
w.herdr = herdr.New(required("ORCHESTRA_WORKER_HERDR"))
|
||||
if id != w.harnessID {
|
||||
url, admit := required("ORCHESTRA_URL"), os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")
|
||||
specs := harnessSpecs()
|
||||
if len(specs) == 1 && specs[0].ID != workerID {
|
||||
// A single-harness deployment keeps the historical invariant: its lease
|
||||
// owner and its process identity are the same name.
|
||||
log.Fatal("ORCHESTRA_WORKER_ID must equal ORCHESTRA_WORKER_HERDR_ID so leases and offline recovery have one owner")
|
||||
}
|
||||
if err := w.api.Register(context.Background(), w.registration); err != nil {
|
||||
log.Fatal(err)
|
||||
stateDir := os.Getenv("ORCHESTRA_WORKER_STATE_DIR")
|
||||
workers := make([]*worker, 0, len(specs))
|
||||
for _, spec := range specs {
|
||||
w := &worker{
|
||||
api: federation.Client{BaseURL: url, WorkerID: spec.ID, Token: spec.Token, AdmitToken: admit},
|
||||
harnessID: spec.ID,
|
||||
harness: spec.Harness,
|
||||
repo: repo,
|
||||
root: root,
|
||||
remote: remote,
|
||||
projects: projects,
|
||||
tasks: map[string]domain.Task{},
|
||||
sessions: map[string]herdr.Session{},
|
||||
leases: map[string]lease{},
|
||||
releases: map[string]releaseTransaction{},
|
||||
quarantined: map[string]bool{},
|
||||
statePath: spec.State,
|
||||
hard: hard,
|
||||
soft: soft,
|
||||
window: window,
|
||||
// Capacity stays one per identity because a herdr's declared
|
||||
// concurrency is one. Serving N harnesses gives the process N slots.
|
||||
registration: federation.Worker{ID: spec.ID, Address: spec.Address, Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()},
|
||||
}
|
||||
if w.registration.Address == "" {
|
||||
w.registration.Address = os.Getenv("ORCHESTRA_WORKER_ADDRESS")
|
||||
}
|
||||
w.statePath = statePathFor(spec, root, stateDir, len(specs) == 1)
|
||||
if err := w.load(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
w.backend = backendFor(spec)
|
||||
if err := w.api.Register(context.Background(), w.registration); err != nil {
|
||||
log.Fatalf("register %s: %v", spec.ID, err)
|
||||
}
|
||||
log.Printf("serving harness %s (%s) on %s backend, state %s", spec.ID, spec.Harness, w.backend.Kind(), w.statePath)
|
||||
workers = append(workers, w)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil {
|
||||
w.recordError(fmt.Errorf("heartbeat: %w", err))
|
||||
log.Printf("heartbeat: %v", err)
|
||||
if w.reRegisterAfterCoordinatorRestart(ctx, err) {
|
||||
continue
|
||||
// Identities are served sequentially. Their checkouts and Git remotes are
|
||||
// shared, so concurrent ticks would race two fetch/worktree operations on
|
||||
// one repository for no useful latency gain at this fan-out.
|
||||
for _, w := range workers {
|
||||
if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil {
|
||||
w.recordError(fmt.Errorf("heartbeat: %w", err))
|
||||
log.Printf("[%s] heartbeat: %v", w.harnessID, err)
|
||||
if w.reRegisterAfterCoordinatorRestart(ctx, err) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := w.once(ctx); err != nil {
|
||||
w.recordError(fmt.Errorf("poll: %w", err))
|
||||
log.Printf("[%s] poll: %v", w.harnessID, err)
|
||||
w.reRegisterAfterCoordinatorRestart(ctx, err)
|
||||
}
|
||||
}
|
||||
if err := w.once(ctx); err != nil {
|
||||
w.recordError(fmt.Errorf("poll: %w", err))
|
||||
log.Printf("poll: %v", err)
|
||||
w.reRegisterAfterCoordinatorRestart(ctx, err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -1129,3 +1494,78 @@ func main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// federatedTurn is the worker half of a turn boundary. The coordinator
|
||||
// reconciles human input and answers with the decisions this session has not
|
||||
// been shown; the worker delivers them into its own pane.
|
||||
//
|
||||
// Nothing is preempted. The boundary is confirmed against the live pane
|
||||
// first, so a correction never lands mid tool call.
|
||||
func (w *worker) federatedTurn(ctx context.Context, id string, a herdr.Adapter, verdict string) {
|
||||
l, ok := w.leases[id]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
s, ok := w.sessions[id]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
boundary, ok := a.(herdr.TurnBoundary)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
at, err := boundary.AtTurnBoundary(ctx, s)
|
||||
if err != nil {
|
||||
w.recordError(fmt.Errorf("turn boundary %s: %w", id, err))
|
||||
return
|
||||
}
|
||||
if !at {
|
||||
return
|
||||
}
|
||||
answer, err := w.api.Turn(ctx, id, l.Epoch, verdict, s.DeliveredDecisions)
|
||||
if err != nil {
|
||||
// Observable, not fatal. A coordinator that cannot be reached does not
|
||||
// make this session's current intent any more stale than it already is.
|
||||
w.recordError(fmt.Errorf("federated turn %s: %w", id, err))
|
||||
return
|
||||
}
|
||||
if answer.Verdict == orchestrator.TurnPrepareHandoff && !s.HandoffRequested {
|
||||
// The coordinator has lost the ability to refresh this task's intent.
|
||||
// Ask for a handoff; the release loop takes over as soon as the agent
|
||||
// writes the report, exactly as it does for a local session.
|
||||
requester, ok := a.(herdr.ReasonedHandoffRequester)
|
||||
if !ok {
|
||||
w.recordError(fmt.Errorf("reconcile failure handoff %s: adapter cannot state a reason", id))
|
||||
return
|
||||
}
|
||||
if err := requester.RequestHandoffReason(ctx, s, "reconcile_failure", nil); err != nil {
|
||||
w.recordError(fmt.Errorf("reconcile failure handoff %s: %w", id, err))
|
||||
return
|
||||
}
|
||||
s.HandoffRequested, s.HandoffReason = true, "reconcile_failure"
|
||||
w.sessions[id] = s
|
||||
_ = w.save()
|
||||
return
|
||||
}
|
||||
if len(answer.Decisions) == 0 {
|
||||
return
|
||||
}
|
||||
if err := w.sendPrompt(ctx, s, agentctx.DecisionNotice(answer.Decisions)); err != nil {
|
||||
// Not recorded as delivered, so the next boundary retries.
|
||||
w.recordError(fmt.Errorf("deliver decisions %s: %w", id, err))
|
||||
return
|
||||
}
|
||||
for _, d := range answer.Decisions {
|
||||
s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID)
|
||||
}
|
||||
w.sessions[id] = s
|
||||
_ = w.save()
|
||||
}
|
||||
|
||||
func (w *worker) sendPrompt(ctx context.Context, s herdr.Session, text string) error {
|
||||
backend := w.executionBackend()
|
||||
if backend == nil {
|
||||
return fmt.Errorf("execution backend is not configured")
|
||||
}
|
||||
return backend.Prompt(ctx, s.PaneID, text, time.Minute)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user