v3 workflow: intent, phases, review, submission, enforcement, burn-in

The v3 stack, previously an uncommitted working tree, plus this session's two
units and the burn-in instrument. This commit is the burn-in build identity:
coordinator and worker must both report this revision before a task is created.

Workflow (earlier sessions, uncommitted until now): human decision events and
reduction, source cursors and reconcile-before-launch, turn-boundary
reconciliation, internal/agentctx as the single renderer, ace-fca phases with
sealed artifacts, the trajectory gate, bounded grilling, independent review,
task pr enforcement, and human review reflection.

Capability restrictions at the agent boundary: an authz.Agent surface at
GatedWrite may ask and may not act. It also fixes two bugs the unit exposed --
gated surfaces could not reach the two endpoints written for them, and
RequestHumanDecision would block an unowned task while rejecting a question
from the session that did own it.

Turn-boundary reconcile-failure escalation: a streak of consecutive failures
asks the session to hand off, fenced on the lease epoch, with reconcile_failure
as a real handoff reason. The worker was dropping the coordinator's verdict on
the floor; it now acts on it.

Burn-in: herdr.WriteLaunchContext dumps the exact agentctx.Build result to
<worktree>/.orchestra/launch.md at every launch, local and federated. BURNIN.md
is the runbook. deploy/build.sh stamps both binaries from one commit.

go build, go vet and go test ./... pass, 20 packages.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-26 18:31:20 +04:00
parent 97a9c65302
commit 7f12c7fc37
78 changed files with 16417 additions and 352 deletions
+497 -57
View File
@@ -1,6 +1,9 @@
// orchestra-worker consumes router-issued leases for one local herdr. Homesrv
// remains the scheduler and CAS authority; this process owns only local Git
// and pane operations.
// orchestra-worker consumes router-issued leases for one or more local
// execution backends. Each declared harness is a separate federation identity
// with its own token, cursor, backend, and state file, because the coordinator
// authorizes a lease call by comparing the URL's worker id against the lease's
// harness id. Homesrv remains the scheduler and CAS authority. This process
// owns only local Git and pane operations.
package main
import (
@@ -9,16 +12,20 @@ import (
"errors"
"fmt"
"log"
"orchestra/internal/agentctx"
"orchestra/internal/buildinfo"
"orchestra/internal/continuity"
"orchestra/internal/domain"
"orchestra/internal/federation"
"orchestra/internal/herdr"
"orchestra/internal/orchestrator"
"orchestra/internal/review"
"orchestra/internal/workphase"
"os"
"os/exec"
"os/signal"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
@@ -27,7 +34,11 @@ import (
)
type worker struct {
api federation.Client
api federation.Client
backend herdr.Backend
// herdr is retained as a test/backward-compatibility alias. Production
// workers set backend; executionBackend keeps older state-machine tests
// from needing protocol-irrelevant rewrites.
herdr *herdr.Client
harnessID, harness, repo, root, remote string
projects map[string]projectConfig
@@ -46,6 +57,16 @@ type worker struct {
window int64
}
func (w *worker) executionBackend() herdr.Backend {
if w.backend != nil {
return w.backend
}
if w.herdr != nil {
return w.herdr
}
return nil
}
func (w *worker) recordError(err error) {
if err == nil {
return
@@ -56,6 +77,9 @@ func (w *worker) recordError(err error) {
func (w *worker) health(ctx context.Context) federation.WorkerHealth {
h := federation.WorkerHealth{HerdrStatus: "unknown"}
if backend := w.executionBackend(); backend != nil {
h.Backend = backend.Kind()
}
for taskID, session := range w.sessions {
// Workers currently advertise capacity one. Pick deterministically so a
// recovered legacy state with more sessions remains intelligible.
@@ -63,16 +87,16 @@ func (w *worker) health(ctx context.Context) federation.WorkerHealth {
h.ActiveTask, h.ActivePane = taskID, session.PaneID
}
}
if w.herdr != nil {
if backend := w.executionBackend(); backend != nil {
checkCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
err := w.herdr.CheckProtocol(checkCtx, "17")
err := backend.Check(checkCtx)
cancel()
h.CheckedAt = time.Now().UTC()
if err == nil {
h.HerdrStatus = "reachable"
} else {
h.HerdrStatus = "unreachable"
w.recordError(fmt.Errorf("local herdr: %w", err))
w.recordError(fmt.Errorf("local %s backend: %w", backend.Kind(), err))
}
}
h.LastError, h.ErrorAt = w.lastError, w.lastErrorAt
@@ -200,11 +224,11 @@ func (w *worker) save() error {
// A failed close remains durable and is retried; it is never treated as a
// harmless cleanup error while the old harness could still be working.
func (w *worker) quarantine(ctx context.Context, taskID string, s herdr.Session) {
if w.herdr == nil {
if w.executionBackend() == nil {
w.quarantined[taskID] = true
return
}
if err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).Kill(ctx, s); err != nil {
if err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).Kill(ctx, s); err != nil {
w.quarantined[taskID] = true
w.recordError(fmt.Errorf("quarantine %s: %w", taskID, err))
return
@@ -316,28 +340,98 @@ func (w *worker) start(ctx context.Context, t domain.Task, ref string) error {
return err
}
}
if _, err = w.herdr.Worktree(ctx, p.Repo, wt, "orchestra/"+t.ID); err != nil {
backend := w.executionBackend()
if backend == nil {
return fmt.Errorf("execution backend is not configured")
}
if _, err = backend.Worktree(ctx, p.Repo, wt, "orchestra/"+t.ID); err != nil {
return err
}
s, err := w.herdr.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID)
s, err := backend.StartAgent(ctx, wt, wt, "orchestra/"+t.ID, w.harness, t.ID)
if err != nil {
return err
}
s.TaskFileSHA = taskHash(t)
prompt := "Read TASK.md at the worktree root and execute it."
if len(p.SafeOperations) > 0 {
prompt += " This project's audited no-grant policy permits only worktree-local " + strings.Join(p.SafeOperations, ", ") + ". Network, secrets, destructive actions, and paths outside this worktree still require an explicit operator approval."
if w.harness == "claude" {
s.ContextHandoffSHA, _ = fileSHA256(filepath.Join(wt, "HANDOFF.md"))
}
// One renderer, on both machines. The worker fetches the reduced
// authority as data and renders it with agentctx, so a decision the human
// recorded before this session existed is visible from its first turn.
intent, err := w.api.Intent(ctx, t.ID)
if err != nil {
return fmt.Errorf("effective intent: %w", err)
}
in := agentctx.Input{
Task: t, Intent: intent, Phase: t.WorkPhase, DecisionRequest: t.DecisionRequest,
Git: agentctx.GitState{Worktree: wt, Branch: "orchestra/" + t.ID},
RepoRules: agentctx.DiscoverRepoRules(wt),
}
if sha, shaErr := herdr.HeadSHA(wt); shaErr == nil {
in.Git.HeadSHA = sha
}
if ref != "" {
prompt += " A validated handoff exists; inspect local Git history and the recorded checkpoint before continuing."
in.Handoff = &h
}
if len(p.SafeOperations) > 0 {
in.Policy = []string{
"Permitted without an operator grant, inside this worktree only: " + strings.Join(p.SafeOperations, ", ") + ".",
"Network access, secrets, destructive actions, and paths outside this worktree require an explicit operator approval.",
}
}
if t.ResearchRef != "" {
b, artErr := w.api.Artifact(ctx, t.ResearchRef)
if artErr != nil {
return fmt.Errorf("research artifact: %w", artErr)
}
r, decErr := workphase.DecodeResearch(b)
if decErr != nil {
return fmt.Errorf("research artifact: %w", decErr)
}
in.Research = &r
}
if t.PlanRef != "" {
b, artErr := w.api.Artifact(ctx, t.PlanRef)
if artErr != nil {
return fmt.Errorf("plan artifact: %w", artErr)
}
pl, decErr := workphase.DecodePlan(b)
if decErr != nil {
return fmt.Errorf("plan artifact: %w", decErr)
}
in.Plan = &pl
}
if t.Review != nil {
b, artErr := w.api.Artifact(ctx, t.Review.ArtifactRef)
if artErr != nil {
return fmt.Errorf("review artifact: %w", artErr)
}
r, decErr := review.Decode(b)
if decErr != nil {
return fmt.Errorf("review artifact: %w", decErr)
}
in.Review = &r
}
built, err := agentctx.Build(in)
if err != nil {
return fmt.Errorf("build context: %w", err)
}
prompt := built.System + "\n\n" + built.Task
if writeErr := herdr.WriteLaunchContext(s.Worktree, prompt); writeErr != nil {
w.recordError(fmt.Errorf("launch context %s: %w", t.ID, writeErr))
}
// The launch instruction carried these, so the first turn boundary must
// not re-announce them as news.
for _, d := range intent.Decisions {
s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID)
}
w.sessions[t.ID] = s
if err := w.save(); err != nil {
return err
}
// A prompt response can be lost after herdr accepted it. Persist the
// A prompt response can be lost after the backend accepted it. Persist the
// session first so the worker can reconcile/release it after restart.
if err := w.herdr.Prompt(ctx, s.PaneID, prompt, 0); err != nil {
if err := backend.Prompt(ctx, s.PaneID, prompt, 0); err != nil {
return err
}
if l, ok := w.leases[t.ID]; ok {
@@ -371,6 +465,14 @@ func classifyLaunchError(err error, sessionStarted bool) string {
func taskHash(t domain.Task) string { b := continuity.RenderTaskFile(t); return domain.Hash(b) }
func fileSHA256(path string) (string, error) {
b, err := os.ReadFile(path)
if err != nil {
return "", err
}
return domain.Hash(b), nil
}
func (w *worker) releaseReady(ctx context.Context) {
for id, s := range w.sessions {
if w.quarantined[id] {
@@ -385,7 +487,7 @@ func (w *worker) releaseReady(ctx context.Context) {
if _, err := os.Stat(filepath.Join(s.Worktree, ".orchestra", "done")); err == nil {
// A done marker is an intent, not enough on its own: do not race a
// still-running native harness into committing half-written work.
status, statusErr := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).AgentStatus(ctx, s)
status, statusErr := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).AgentStatus(ctx, s)
if statusErr != nil {
w.recordError(fmt.Errorf("completion identity %s: %w", id, statusErr))
continue
@@ -414,7 +516,7 @@ func (w *worker) releaseReady(ctx context.Context) {
}
// Completion is durable before closing the exact pane. If close
// fails, retain the session mapping for a later explicit cleanup.
a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}
a := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}
if err := a.Kill(ctx, s); err != nil {
w.recordError(fmt.Errorf("close completed pane %s: %w", id, err))
log.Printf("close completed pane %s: %v", id, err)
@@ -436,7 +538,7 @@ func (w *worker) releaseReady(ctx context.Context) {
}
func (w *worker) adapter(s herdr.Session, remote string) herdr.CLIAdapter {
a := herdr.CLIAdapter{Client: w.herdr, Harness: w.harness, Window: w.window, CAS: artifactCAS{w.api}, Remote: remote}
a := herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness, Window: w.window, CAS: artifactCAS{w.api}, Remote: remote}
switch w.harness {
case "claude":
a.Usage = herdr.ClaudeUsage
@@ -452,6 +554,18 @@ func (w *worker) adapter(s herdr.Session, remote string) herdr.CLIAdapter {
// and pane status are all read from the persisted harness session identity;
// any unknown source is recorded and never treated as zero usage.
func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
// Claude Code owns its context threshold through the installed
// context-handoff hook. A changed HANDOFF.md means that hook has landed a
// durable local continuation. Resume in the same process with Claude's
// native context reset instead of manufacturing Orchestra's cross-worker
// release artifact. Codex and OpenCode continue through the existing
// occupancy/release state machine below.
if w.harness == "claude" {
if err := w.advanceClaudeContextReset(ctx, id, s); err != nil {
w.recordError(fmt.Errorf("Claude context reset %s: %w", id, err))
}
return
}
t, ok := w.tasks[id]
if !ok {
w.recordError(fmt.Errorf("rotation %s: task cache missing", id))
@@ -468,7 +582,9 @@ func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
w.recordError(fmt.Errorf("rotation %s occupancy degraded: %w", id, err))
return
}
if resolved != s {
// DeepEqual, not !=: Session carries a slice since decisions are tracked
// per session, so it is no longer comparable with ==.
if !reflect.DeepEqual(resolved, s) {
w.sessions[id] = resolved
s = resolved
_ = w.save()
@@ -483,7 +599,11 @@ func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
return
}
}
if d.Action == orchestrator.TurnContinue || d.Action == orchestrator.TurnRefuse || s.HandoffRequested {
if d.Action == orchestrator.TurnContinue || d.Action == "" {
w.federatedTurn(ctx, id, a, orchestrator.TurnContinue)
return
}
if d.Action == orchestrator.TurnRefuse || s.HandoffRequested {
return
}
if d.Reason == "milestone" || d.Reason == "thrash" {
@@ -500,6 +620,86 @@ func (w *worker) rotationTick(ctx context.Context, id string, s herdr.Session) {
_ = w.save()
}
func (w *worker) sendLine(ctx context.Context, s herdr.Session, line string) error {
backend := w.executionBackend()
if backend == nil {
return fmt.Errorf("execution backend is not configured")
}
if err := backend.SendText(ctx, s, line); err != nil {
return fmt.Errorf("send %q: %w", line, err)
}
if err := backend.SendKeys(ctx, s, []string{"ENTER"}); err != nil {
return fmt.Errorf("submit %q: %w", line, err)
}
return nil
}
func (w *worker) advanceClaudeContextReset(ctx context.Context, id string, s herdr.Session) error {
backend := w.executionBackend()
if backend == nil {
return fmt.Errorf("execution backend is not configured")
}
if s.ContextResetSHA == "" {
sha, err := fileSHA256(filepath.Join(s.Worktree, "HANDOFF.md"))
if os.IsNotExist(err) {
return nil
}
if err != nil {
return fmt.Errorf("read HANDOFF.md: %w", err)
}
if sha == s.ContextHandoffSHA {
return nil
}
status, err := backend.AgentStatus(ctx, s)
if err != nil {
return fmt.Errorf("confirm Claude stopped after handoff: %w", err)
}
if status != "idle" {
return nil
}
s.ContextResetSHA = sha
s.ContextResetPhase = "clear"
w.sessions[id] = s
if err := w.save(); err != nil {
return err
}
}
switch s.ContextResetPhase {
case "clear":
if err := w.sendLine(ctx, s, "/clear"); err != nil {
return err
}
s.ContextResetPhase = "handoff"
w.sessions[id] = s
if err := w.save(); err != nil {
return err
}
// /clear redraws Claude's input UI asynchronously. Give it a small,
// bounded interval before submitting the new-session file mention.
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(500 * time.Millisecond):
}
fallthrough
case "handoff":
if err := w.sendLine(ctx, s, "@HANDOFF.md"); err != nil {
return err
}
s.ContextHandoffSHA = s.ContextResetSHA
s.ContextResetSHA = ""
s.ContextResetPhase = ""
// Claude normally opens a fresh transcript for /clear. Force the next
// observation to discover it instead of retaining the exhausted path.
s.SessionFile = ""
w.sessions[id] = s
return w.save()
default:
return fmt.Errorf("unknown persisted context-reset phase %q", s.ContextResetPhase)
}
}
func (w *worker) advanceRelease(ctx context.Context, id string, s herdr.Session) {
t, ok := w.tasks[id]
if !ok {
@@ -604,7 +804,7 @@ func (w *worker) ackPickup(ctx context.Context, id string, s herdr.Session) erro
func (w *worker) sessionEvidence(ctx context.Context, taskID string, s herdr.Session) domain.SessionEvidence {
e := domain.SessionEvidence{PaneID: s.PaneID, HarnessID: w.harnessID, PaneState: "open", Source: "worker", CheckedAt: time.Now().UTC()}
text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent")
text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, s, "recent")
if err != nil {
e.PaneState = "unreachable"
return e
@@ -710,7 +910,7 @@ func (w *worker) finalize(ctx context.Context, id string, s herdr.Session) (comp
}
func (w *worker) renewLeases(ctx context.Context) {
if w.herdr == nil {
if w.executionBackend() == nil {
return
}
now := time.Now()
@@ -719,7 +919,7 @@ func (w *worker) renewLeases(ctx context.Context) {
if !ok || l.Version == 0 || l.Until.After(now.Add(10*time.Minute)) {
continue
}
if _, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, s, "recent"); err != nil {
if _, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, s, "recent"); err != nil {
w.recordError(fmt.Errorf("validate lease %s: %w", taskID, err))
continue
}
@@ -741,7 +941,7 @@ func (w *worker) renewLeases(ctx context.Context) {
// coordinator to touch their unix herdr socket.
func (w *worker) publishCaptures(ctx context.Context) {
for taskID, session := range w.sessions {
text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, session, "recent")
text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, session, "recent")
if err != nil {
continue
}
@@ -787,7 +987,7 @@ func (w *worker) runCommands(ctx context.Context) {
_ = w.api.ResolveCommand(ctx, command.ID, "stale", "session or pane changed")
continue
}
text, err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).PaneCapture(ctx, session, "recent")
text, err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).PaneCapture(ctx, session, "recent")
if err != nil {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "capture unavailable: "+err.Error())
continue
@@ -806,12 +1006,15 @@ func (w *worker) runCommands(ctx context.Context) {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "prompt does not expose an executable approval control")
continue
}
method, params := "pane.send_text", map[string]any{"pane_id": session.PaneID, "text": input.Text}
backend := w.executionBackend()
var inputErr error
if len(input.Keys) > 0 {
method, params = "pane.send_keys", map[string]any{"pane_id": session.PaneID, "keys": input.Keys}
inputErr = backend.SendKeys(ctx, session, input.Keys)
} else {
inputErr = backend.SendText(ctx, session, input.Text)
}
if err := w.herdr.Call(ctx, method, params, nil); err != nil {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", "herdr did not acknowledge input: "+err.Error())
if inputErr != nil {
_ = w.api.ResolveCommand(ctx, command.ID, "rejected", backend.Kind()+" backend did not acknowledge input: "+inputErr.Error())
continue
}
if err := w.api.ResolveCommand(ctx, command.ID, "acknowledged", ""); err != nil {
@@ -875,9 +1078,9 @@ func (w *worker) once(ctx context.Context) error {
if e.Type == "TaskCompleted" {
delete(w.leases, e.TaskID)
if session, active := w.sessions[e.TaskID]; active {
if w.herdr == nil {
if w.executionBackend() == nil {
delete(w.sessions, e.TaskID)
} else if err := (herdr.CLIAdapter{Client: w.herdr, Harness: w.harness}).Kill(ctx, session); err != nil {
} else if err := (herdr.CLIAdapter{Backend: w.executionBackend(), Harness: w.harness}).Kill(ctx, session); err != nil {
w.quarantined[e.TaskID] = true
w.recordError(fmt.Errorf("close completed pane %s: %w", e.TaskID, err))
continue
@@ -979,10 +1182,10 @@ func (w *worker) once(ctx context.Context) error {
}
}
}
// Unit/replay-only workers intentionally have no herdr connection. A
// Unit/replay-only workers intentionally have no execution backend. A
// production worker always does, and only then participates in the live
// capture/control protocol.
if w.herdr != nil {
if w.executionBackend() != nil {
w.publishCaptures(ctx)
w.runCommands(ctx)
w.renewLeases(ctx)
@@ -1039,8 +1242,134 @@ func required(k string) string {
}
return v
}
// harnessSpec is one harness identity this process serves. Each spec becomes a
// separate federation identity because the coordinator authorizes lease calls by
// comparing the URL's worker id against the lease's harness id
// (cmd/orchestra/main.go). One process may therefore hold several identities,
// but it may never present one identity for several harnesses.
type harnessSpec struct {
ID string `json:"id"`
Harness string `json:"harness"`
Token string `json:"token,omitempty"`
Backend string `json:"backend,omitempty"`
// Herdr is the JSON-RPC address for Backend "herdr".
Herdr string `json:"herdr,omitempty"`
// TmuxSocket and Command configure Backend "tmux".
TmuxSocket string `json:"tmux_socket,omitempty"`
Command string `json:"command,omitempty"`
State string `json:"state,omitempty"`
Address string `json:"address,omitempty"`
}
// tokenEnvKey maps a harness id onto a per-identity token variable, so a
// multi-harness deployment keeps its tokens in the protected environment file
// rather than in the harness config file.
func tokenEnvKey(id string) string {
var b strings.Builder
b.WriteString("ORCHESTRA_WORKER_TOKEN_")
for _, r := range strings.ToUpper(id) {
if (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
continue
}
b.WriteRune('_')
}
return b.String()
}
// harnessSpecs reads the multi-harness declaration, falling back to the legacy
// single-harness environment so an existing deployment upgrades unchanged.
func harnessSpecs() []harnessSpec {
path := os.Getenv("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE")
if path == "" {
return []harnessSpec{{
ID: required("ORCHESTRA_WORKER_HERDR_ID"),
Harness: required("ORCHESTRA_WORKER_HARNESS"),
Token: required("ORCHESTRA_WORKER_TOKEN"),
Backend: os.Getenv("ORCHESTRA_WORKER_BACKEND"),
Herdr: os.Getenv("ORCHESTRA_WORKER_HERDR"),
TmuxSocket: os.Getenv("ORCHESTRA_WORKER_TMUX_SOCKET"),
Command: os.Getenv("ORCHESTRA_WORKER_HARNESS_COMMAND"),
State: os.Getenv("ORCHESTRA_WORKER_STATE"),
Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"),
}}
}
b, err := os.ReadFile(path)
if err != nil {
log.Fatalf("read ORCHESTRA_WORKER_HARNESS_CONFIG_FILE: %v", err)
}
var specs []harnessSpec
if err := json.Unmarshal(b, &specs); err != nil {
log.Fatalf("parse ORCHESTRA_WORKER_HARNESS_CONFIG_FILE: %v", err)
}
if len(specs) == 0 {
log.Fatal("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE declares no harnesses")
}
seen := map[string]bool{}
for i, spec := range specs {
if spec.ID == "" || spec.Harness == "" {
log.Fatalf("harness %d requires id and harness", i)
}
if seen[spec.ID] {
log.Fatalf("harness id %q is declared twice", spec.ID)
}
seen[spec.ID] = true
if spec.Token == "" {
key := tokenEnvKey(spec.ID)
if spec.Token = os.Getenv(key); spec.Token == "" {
log.Fatalf("harness %s has no token: set %s or its config-file token", spec.ID, key)
}
specs[i] = spec
}
}
return specs
}
// statePathFor gives every identity its own state file. Sharing one across
// harnesses would let a persisted session and its lease cross backends, handing
// a tmux backend herdr pane ids it cannot act on. Only the single-harness form
// keeps the historical default path, so an existing deployment recovers its
// sessions after the upgrade instead of orphaning them.
func statePathFor(spec harnessSpec, root, stateDir string, single bool) string {
switch {
case spec.State != "":
return spec.State
case stateDir != "":
return filepath.Join(stateDir, "state-"+spec.ID+".json")
case single:
return filepath.Join(root, ".orchestra-worker-state.json")
default:
return filepath.Join(root, ".orchestra-worker-state-"+spec.ID+".json")
}
}
// backendFor builds the machine-local execution backend for one harness.
func backendFor(spec harnessSpec) herdr.Backend {
name := strings.ToLower(strings.TrimSpace(spec.Backend))
if name == "" {
name = "herdr"
}
switch name {
case "herdr":
address := spec.Herdr
if address == "" {
log.Fatalf("harness %s: backend herdr requires an address", spec.ID)
}
return herdr.New(address)
case "tmux":
if spec.Harness != "claude" {
log.Fatalf("harness %s: backend tmux currently supports only harness claude, got %q", spec.ID, spec.Harness)
}
return herdr.NewTmuxBackend(spec.TmuxSocket, spec.Command)
default:
log.Fatalf("harness %s: unsupported backend %q (want herdr or tmux)", spec.ID, name)
return nil
}
}
func main() {
id, token := required("ORCHESTRA_WORKER_ID"), required("ORCHESTRA_WORKER_TOKEN")
workerID := required("ORCHESTRA_WORKER_ID")
hard := .75
if v, err := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); err == nil && v > 0 && v < 1 {
hard = v
@@ -1091,36 +1420,72 @@ func main() {
if v, err := strconv.ParseInt(os.Getenv("ORCHESTRA_CONTEXT_WINDOW"), 10, 64); err == nil && v > 0 {
window = v
}
w := &worker{api: federation.Client{BaseURL: required("ORCHESTRA_URL"), WorkerID: id, Token: token, AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")}, harnessID: required("ORCHESTRA_WORKER_HERDR_ID"), harness: required("ORCHESTRA_WORKER_HARNESS"), repo: repo, root: root, remote: remote, projects: projects, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, releases: map[string]releaseTransaction{}, quarantined: map[string]bool{}, statePath: os.Getenv("ORCHESTRA_WORKER_STATE"), hard: hard, soft: soft, window: window, registration: federation.Worker{ID: id, Address: os.Getenv("ORCHESTRA_WORKER_ADDRESS"), Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()}}
if w.statePath == "" {
w.statePath = filepath.Join(w.root, ".orchestra-worker-state.json")
}
if err := w.load(); err != nil {
log.Fatal(err)
}
w.herdr = herdr.New(required("ORCHESTRA_WORKER_HERDR"))
if id != w.harnessID {
url, admit := required("ORCHESTRA_URL"), os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN")
specs := harnessSpecs()
if len(specs) == 1 && specs[0].ID != workerID {
// A single-harness deployment keeps the historical invariant: its lease
// owner and its process identity are the same name.
log.Fatal("ORCHESTRA_WORKER_ID must equal ORCHESTRA_WORKER_HERDR_ID so leases and offline recovery have one owner")
}
if err := w.api.Register(context.Background(), w.registration); err != nil {
log.Fatal(err)
stateDir := os.Getenv("ORCHESTRA_WORKER_STATE_DIR")
workers := make([]*worker, 0, len(specs))
for _, spec := range specs {
w := &worker{
api: federation.Client{BaseURL: url, WorkerID: spec.ID, Token: spec.Token, AdmitToken: admit},
harnessID: spec.ID,
harness: spec.Harness,
repo: repo,
root: root,
remote: remote,
projects: projects,
tasks: map[string]domain.Task{},
sessions: map[string]herdr.Session{},
leases: map[string]lease{},
releases: map[string]releaseTransaction{},
quarantined: map[string]bool{},
statePath: spec.State,
hard: hard,
soft: soft,
window: window,
// Capacity stays one per identity because a herdr's declared
// concurrency is one. Serving N harnesses gives the process N slots.
registration: federation.Worker{ID: spec.ID, Address: spec.Address, Capacity: 1, SupportedProjects: supported, Build: buildinfo.Current()},
}
if w.registration.Address == "" {
w.registration.Address = os.Getenv("ORCHESTRA_WORKER_ADDRESS")
}
w.statePath = statePathFor(spec, root, stateDir, len(specs) == 1)
if err := w.load(); err != nil {
log.Fatal(err)
}
w.backend = backendFor(spec)
if err := w.api.Register(context.Background(), w.registration); err != nil {
log.Fatalf("register %s: %v", spec.ID, err)
}
log.Printf("serving harness %s (%s) on %s backend, state %s", spec.ID, spec.Harness, w.backend.Kind(), w.statePath)
workers = append(workers, w)
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil {
w.recordError(fmt.Errorf("heartbeat: %w", err))
log.Printf("heartbeat: %v", err)
if w.reRegisterAfterCoordinatorRestart(ctx, err) {
continue
// Identities are served sequentially. Their checkouts and Git remotes are
// shared, so concurrent ticks would race two fetch/worktree operations on
// one repository for no useful latency gain at this fan-out.
for _, w := range workers {
if err := w.api.Heartbeat(ctx, w.health(ctx)); err != nil {
w.recordError(fmt.Errorf("heartbeat: %w", err))
log.Printf("[%s] heartbeat: %v", w.harnessID, err)
if w.reRegisterAfterCoordinatorRestart(ctx, err) {
continue
}
}
if err := w.once(ctx); err != nil {
w.recordError(fmt.Errorf("poll: %w", err))
log.Printf("[%s] poll: %v", w.harnessID, err)
w.reRegisterAfterCoordinatorRestart(ctx, err)
}
}
if err := w.once(ctx); err != nil {
w.recordError(fmt.Errorf("poll: %w", err))
log.Printf("poll: %v", err)
w.reRegisterAfterCoordinatorRestart(ctx, err)
}
select {
case <-ctx.Done():
@@ -1129,3 +1494,78 @@ func main() {
}
}
}
// federatedTurn is the worker half of a turn boundary. The coordinator
// reconciles human input and answers with the decisions this session has not
// been shown; the worker delivers them into its own pane.
//
// Nothing is preempted. The boundary is confirmed against the live pane
// first, so a correction never lands mid tool call.
func (w *worker) federatedTurn(ctx context.Context, id string, a herdr.Adapter, verdict string) {
l, ok := w.leases[id]
if !ok {
return
}
s, ok := w.sessions[id]
if !ok {
return
}
boundary, ok := a.(herdr.TurnBoundary)
if !ok {
return
}
at, err := boundary.AtTurnBoundary(ctx, s)
if err != nil {
w.recordError(fmt.Errorf("turn boundary %s: %w", id, err))
return
}
if !at {
return
}
answer, err := w.api.Turn(ctx, id, l.Epoch, verdict, s.DeliveredDecisions)
if err != nil {
// Observable, not fatal. A coordinator that cannot be reached does not
// make this session's current intent any more stale than it already is.
w.recordError(fmt.Errorf("federated turn %s: %w", id, err))
return
}
if answer.Verdict == orchestrator.TurnPrepareHandoff && !s.HandoffRequested {
// The coordinator has lost the ability to refresh this task's intent.
// Ask for a handoff; the release loop takes over as soon as the agent
// writes the report, exactly as it does for a local session.
requester, ok := a.(herdr.ReasonedHandoffRequester)
if !ok {
w.recordError(fmt.Errorf("reconcile failure handoff %s: adapter cannot state a reason", id))
return
}
if err := requester.RequestHandoffReason(ctx, s, "reconcile_failure", nil); err != nil {
w.recordError(fmt.Errorf("reconcile failure handoff %s: %w", id, err))
return
}
s.HandoffRequested, s.HandoffReason = true, "reconcile_failure"
w.sessions[id] = s
_ = w.save()
return
}
if len(answer.Decisions) == 0 {
return
}
if err := w.sendPrompt(ctx, s, agentctx.DecisionNotice(answer.Decisions)); err != nil {
// Not recorded as delivered, so the next boundary retries.
w.recordError(fmt.Errorf("deliver decisions %s: %w", id, err))
return
}
for _, d := range answer.Decisions {
s.DeliveredDecisions = append(s.DeliveredDecisions, d.ID)
}
w.sessions[id] = s
_ = w.save()
}
func (w *worker) sendPrompt(ctx context.Context, s herdr.Session, text string) error {
backend := w.executionBackend()
if backend == nil {
return fmt.Errorf("execution backend is not configured")
}
return backend.Prompt(ctx, s.PaneID, text, time.Minute)
}
+384 -1
View File
@@ -17,6 +17,7 @@ import (
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"time"
)
@@ -153,6 +154,8 @@ func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
t.Fatal(err)
}
defer ln.Close()
var promptMu sync.Mutex
var prompts []string
go func() {
for {
c, e := ln.Accept()
@@ -176,6 +179,12 @@ func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
case "pane.read":
result = `{"read":{"text":""}}`
case "agent.prompt":
if m, ok := r.Params.(map[string]any); ok {
text, _ := m["text"].(string)
promptMu.Lock()
prompts = append(prompts, text)
promptMu.Unlock()
}
result = `{}`
default:
result = `{}`
@@ -185,7 +194,23 @@ func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
}
}()
root := filepath.Join(filepath.Dir(repo), "worktrees")
w := &worker{herdr: &herdr.Client{Path: ln.Addr().String()}, repo: repo, root: root, remote: "origin", harness: "opencode", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: filepath.Join(t.TempDir(), "state.json"), hard: .75}
// The worker renders its own launch instruction from the coordinator's
// reduced authority, so the fake API must serve it.
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/intent") {
json.NewEncoder(w).Encode(domain.EffectiveIntent{
Task: domain.Task{ID: "task", Title: "test", Description: "do work"},
Decisions: []domain.HumanDecision{{
ID: "d1", TaskID: "task", Kind: domain.HumanDecisionCorrection,
Subject: "strategy", Value: "no, use b",
}},
})
return
}
w.WriteHeader(200)
}))
defer api.Close()
w := &worker{api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, herdr: &herdr.Client{Path: ln.Addr().String()}, repo: repo, root: root, remote: "origin", harness: "opencode", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: filepath.Join(t.TempDir(), "state.json"), hard: .75}
// New() is needed for its pane map; override the address for the fake.
w.herdr = herdr.New(ln.Addr().String())
task := domain.Task{ID: "task", Source: "s", ExternalID: "x", Project: "p", Title: "test", Description: "do work"}
@@ -198,6 +223,16 @@ func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
if _, err := os.Stat(filepath.Join(root, "task", "TASK.md")); err != nil {
t.Fatalf("TASK.md: %v", err)
}
// The deployed worker path renders through agentctx, so a decision the
// human recorded before this session existed reaches the agent.
promptMu.Lock()
launched := strings.Join(prompts, "\n")
promptMu.Unlock()
for _, want := range []string{"## Current human decisions", "no, use b", "Authority order"} {
if !strings.Contains(launched, want) {
t.Fatalf("worker launch instruction missing %q:\n%s", want, launched)
}
}
}
func TestFinalizeRunsGateCommitsPushesAndVerifiesRemote(t *testing.T) {
@@ -332,3 +367,351 @@ func TestApprovalResponseOpenCodeAllowOnce(t *testing.T) {
}
func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }
type recordingBackend struct {
status string
calls []string
prompts []string
}
func (b *recordingBackend) Kind() string { return "recording" }
func (b *recordingBackend) Check(context.Context) error { return nil }
func (b *recordingBackend) Worktree(_ context.Context, _, path, _ string) (string, error) {
return path, nil
}
func (b *recordingBackend) StartAgent(_ context.Context, _, path, _, harness, _ string) (herdr.Session, error) {
return herdr.Session{PaneID: "pane", Worktree: path, Harness: harness}, nil
}
func (b *recordingBackend) Prompt(_ context.Context, _, text string, _ time.Duration) error {
b.prompts = append(b.prompts, text)
return nil
}
func (b *recordingBackend) Kill(context.Context, herdr.Session) error { return nil }
func (b *recordingBackend) AgentStatus(context.Context, herdr.Session) (string, error) {
if b.status == "" {
return "idle", nil
}
return b.status, nil
}
func (b *recordingBackend) PaneCapture(context.Context, herdr.Session, string) (string, error) {
return "", nil
}
func (b *recordingBackend) SendText(_ context.Context, _ herdr.Session, text string) error {
b.calls = append(b.calls, "text:"+text)
return nil
}
func (b *recordingBackend) SendKeys(_ context.Context, _ herdr.Session, keys []string) error {
b.calls = append(b.calls, "keys:"+strings.Join(keys, ","))
return nil
}
func (b *recordingBackend) ReleaseAgent(context.Context, herdr.Session, string) error { return nil }
func TestClaudeContextHookRolloverSendsClearThenHandoff(t *testing.T) {
worktree := t.TempDir()
handoff := filepath.Join(worktree, "HANDOFF.md")
if err := os.WriteFile(handoff, []byte("old handoff"), 0o644); err != nil {
t.Fatal(err)
}
oldSHA, err := fileSHA256(handoff)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(handoff, []byte("new durable handoff"), 0o644); err != nil {
t.Fatal(err)
}
backend := &recordingBackend{}
session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude", SessionFile: "exhausted.jsonl", ContextHandoffSHA: oldSHA}
w := &worker{
backend: backend,
harness: "claude",
sessions: map[string]herdr.Session{"task": session},
tasks: map[string]domain.Task{},
leases: map[string]lease{},
releases: map[string]releaseTransaction{},
quarantined: map[string]bool{},
statePath: filepath.Join(t.TempDir(), "state.json"),
}
if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil {
t.Fatal(err)
}
want := []string{"text:/clear", "keys:ENTER", "text:@HANDOFF.md", "keys:ENTER"}
if !reflect.DeepEqual(backend.calls, want) {
t.Fatalf("rollover calls=%v want %v", backend.calls, want)
}
got := w.sessions["task"]
newSHA, _ := fileSHA256(handoff)
if got.ContextHandoffSHA != newSHA || got.ContextResetSHA != "" || got.ContextResetPhase != "" || got.SessionFile != "" {
t.Fatalf("rollover state was not finalized: %+v", got)
}
if err := w.advanceClaudeContextReset(context.Background(), "task", got); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(backend.calls, want) {
t.Fatalf("unchanged handoff retriggered rollover: %v", backend.calls)
}
}
func TestClaudeContextHookRolloverWaitsForIdle(t *testing.T) {
worktree := t.TempDir()
if err := os.WriteFile(filepath.Join(worktree, "HANDOFF.md"), []byte("handoff"), 0o644); err != nil {
t.Fatal(err)
}
backend := &recordingBackend{status: "busy"}
session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude"}
w := &worker{backend: backend, harness: "claude", sessions: map[string]herdr.Session{"task": session}, statePath: filepath.Join(t.TempDir(), "state.json")}
if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil {
t.Fatal(err)
}
if len(backend.calls) != 0 {
t.Fatalf("busy Claude received rollover input: %v", backend.calls)
}
}
func TestHarnessSpecsFallsBackToSingleHarnessEnvironment(t *testing.T) {
t.Setenv("ORCHESTRA_WORKER_HERDR_ID", "workpc-opencode")
t.Setenv("ORCHESTRA_WORKER_HARNESS", "opencode")
t.Setenv("ORCHESTRA_WORKER_TOKEN", "secret")
t.Setenv("ORCHESTRA_WORKER_HERDR", "127.0.0.1:9247")
specs := harnessSpecs()
if len(specs) != 1 {
t.Fatalf("want one legacy spec, got %d", len(specs))
}
if specs[0].ID != "workpc-opencode" || specs[0].Harness != "opencode" || specs[0].Token != "secret" || specs[0].Herdr != "127.0.0.1:9247" {
t.Fatalf("legacy environment was not carried through: %+v", specs[0])
}
}
func TestHarnessSpecsReadsMultipleHarnessesAndPerIdentityTokens(t *testing.T) {
path := filepath.Join(t.TempDir(), "harnesses.json")
if err := os.WriteFile(path, []byte(`[
{"id":"workpc-claude","harness":"claude","backend":"tmux","tmux_socket":"orchestra","command":"/usr/bin/true"},
{"id":"workpc-opencode","harness":"opencode","backend":"herdr","herdr":"127.0.0.1:9247","token":"inline"}
]`), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE", path)
t.Setenv("ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE", "from-env")
specs := harnessSpecs()
if len(specs) != 2 {
t.Fatalf("want two specs, got %d", len(specs))
}
if specs[0].Token != "from-env" {
t.Fatalf("per-identity token env was not consulted: %q", specs[0].Token)
}
if specs[1].Token != "inline" {
t.Fatalf("config-file token was overridden: %q", specs[1].Token)
}
// Distinct backends in one process is the point of the multi-harness form.
if got := backendFor(specs[0]).Kind(); got != "tmux" {
t.Fatalf("first harness backend = %q, want tmux", got)
}
if got := backendFor(specs[1]).Kind(); got != "herdr" {
t.Fatalf("second harness backend = %q, want herdr", got)
}
}
func TestTokenEnvKeySanitizesHarnessID(t *testing.T) {
if got := tokenEnvKey("workpc-claude"); got != "ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE" {
t.Fatalf("tokenEnvKey = %q", got)
}
if got := tokenEnvKey("box.1-opencode"); got != "ORCHESTRA_WORKER_TOKEN_BOX_1_OPENCODE" {
t.Fatalf("tokenEnvKey = %q", got)
}
}
func TestStatePathIsolatesEveryHarnessIdentity(t *testing.T) {
claude := harnessSpec{ID: "workpc-claude"}
opencode := harnessSpec{ID: "workpc-opencode"}
// The single-harness form must keep the historical path so an upgraded
// deployment recovers its sessions rather than orphaning their panes.
if got := statePathFor(opencode, "/wt", "", true); got != "/wt/.orchestra-worker-state.json" {
t.Fatalf("single-harness state path changed: %q", got)
}
a := statePathFor(claude, "/wt", "", false)
b := statePathFor(opencode, "/wt", "", false)
if a == b {
t.Fatalf("two harnesses share one state file: %q", a)
}
if got := statePathFor(claude, "/wt", "/var/lib/orchestra-worker", false); got != "/var/lib/orchestra-worker/state-workpc-claude.json" {
t.Fatalf("state dir ignored: %q", got)
}
explicit := harnessSpec{ID: "workpc-claude", State: "/srv/claude.json"}
if got := statePathFor(explicit, "/wt", "/var/lib/orchestra-worker", false); got != "/srv/claude.json" {
t.Fatalf("explicit per-entry state path ignored: %q", got)
}
}
// boundaryAdapter reports a verified turn boundary without a live pane.
type boundaryAdapter struct {
at bool
err error
}
func (b boundaryAdapter) Lease(context.Context, string, string) (herdr.Session, error) {
return herdr.Session{}, nil
}
func (b boundaryAdapter) Release(context.Context, herdr.Session) (string, error) { return "", nil }
func (b boundaryAdapter) Kill(context.Context, herdr.Session) error { return nil }
func (b boundaryAdapter) Occupancy(herdr.Session) (float64, error) { return 0, nil }
func (b boundaryAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
return b.at, b.err
}
// The federated half of the authority model: at a verified boundary the worker
// asks the coordinator, delivers the correction into its own pane once, and
// records it so the next boundary stays quiet.
func TestFederatedTurnDeliversCorrectionOnce(t *testing.T) {
prompts := make(chan string, 4)
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go func() {
for {
c, e := ln.Accept()
if e != nil {
return
}
go func() {
defer c.Close()
var request herdr.Request
if json.NewDecoder(c).Decode(&request) != nil {
return
}
result := `{}`
switch request.Method {
case "pane.get":
result = `{"pane":{"agent":"opencode","agent_status":"idle"}}`
case "pane.read":
result = `{"read":{"text":""}}`
case "agent.prompt":
if m, ok := request.Params.(map[string]any); ok {
text, _ := m["text"].(string)
prompts <- text
}
}
_ = json.NewEncoder(c).Encode(herdr.Response{ID: request.ID, Result: json.RawMessage(result)})
}()
}
}()
var turnCalls int
var lastDelivered []string
fail := false
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasSuffix(r.URL.Path, "/federation/turn") {
w.WriteHeader(200)
return
}
if fail {
http.Error(w, "coordinator down", 503)
return
}
turnCalls++
var body struct {
Delivered []string `json:"delivered_decisions"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
lastDelivered = body.Delivered
out := federation.TurnDecision{Verdict: "continue"}
if len(body.Delivered) == 0 {
out.Decisions = []domain.HumanDecision{{
ID: "d1", Kind: domain.HumanDecisionCorrection, Subject: "strategy", Value: "no, use b",
}}
}
json.NewEncoder(w).Encode(out)
}))
defer api.Close()
w := &worker{
api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
herdr: herdr.New(ln.Addr().String()), harness: "opencode",
sessions: map[string]herdr.Session{"task": {PaneID: "pane", Harness: "opencode"}},
leases: map[string]lease{"task": {Epoch: "e1", Version: 2}},
statePath: t.TempDir() + "/state.json",
}
// Not at a boundary: nothing is asked and nothing is delivered.
w.federatedTurn(context.Background(), "task", boundaryAdapter{at: false}, "continue")
if turnCalls != 0 {
t.Fatal("asked the coordinator without a verified boundary")
}
w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
select {
case got := <-prompts:
if !strings.Contains(got, "no, use b") || !strings.Contains(got, "outrank") {
t.Fatalf("delivered prompt = %q", got)
}
default:
t.Fatal("correction was not delivered to the pane")
}
if ids := w.sessions["task"].DeliveredDecisions; len(ids) != 1 || ids[0] != "d1" {
t.Fatalf("delivered ids = %v", ids)
}
// Second boundary: the worker reports what it has shown, so nothing repeats.
w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
if len(lastDelivered) != 1 || lastDelivered[0] != "d1" {
t.Fatalf("worker did not report delivered ids: %v", lastDelivered)
}
select {
case got := <-prompts:
t.Fatalf("correction delivered twice: %q", got)
default:
}
// Coordinator unreachable: observable, and the session keeps running.
fail = true
w.lastError = ""
w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
if !strings.Contains(w.lastError, "federated turn") {
t.Fatalf("transport failure not observable: %q", w.lastError)
}
}
// A worker must act on the coordinator's verdict, not only on its decisions.
// When the coordinator can no longer reconcile human input for this task, it
// answers prepare_handoff, and the worker asks its own agent to hand off. The
// release loop then takes over on the next tick, because the handoff report is
// what it watches for.
func TestFederatedTurnActsOnPrepareHandoffVerdict(t *testing.T) {
api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/federation/turn" {
rw.WriteHeader(http.StatusNotFound)
return
}
json.NewEncoder(rw).Encode(federation.TurnDecision{Verdict: orchestrator.TurnPrepareHandoff})
}))
defer api.Close()
backend := &recordingBackend{}
session := herdr.Session{PaneID: "pane", Worktree: t.TempDir(), Harness: "opencode"}
w := &worker{
api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
backend: backend,
harness: "opencode",
sessions: map[string]herdr.Session{"task": session},
leases: map[string]lease{"task": {Epoch: "e1", Version: 2}},
tasks: map[string]domain.Task{},
statePath: filepath.Join(t.TempDir(), "state.json"),
}
a := herdr.CLIAdapter{Backend: backend, Harness: "opencode"}
w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue)
got := w.sessions["task"]
if !got.HandoffRequested || got.HandoffReason != "reconcile_failure" {
t.Fatalf("session did not record the requested handoff: %+v", got)
}
if len(backend.prompts) != 1 || !strings.Contains(backend.prompts[0], "reconcile_failure") {
t.Fatalf("agent was not asked to hand off: %v", backend.prompts)
}
// Idempotent: a second boundary must not re-prompt a session that is
// already preparing its handoff.
w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue)
if len(backend.prompts) != 1 {
t.Fatalf("handoff re-requested: %v", backend.prompts)
}
}
+371 -11
View File
@@ -17,10 +17,12 @@ import (
"orchestra/internal/domain"
"orchestra/internal/federation"
"orchestra/internal/herdr"
"orchestra/internal/human"
"orchestra/internal/operations"
"orchestra/internal/orchestrator"
"orchestra/internal/provider"
"orchestra/internal/registry"
"orchestra/internal/review"
"orchestra/internal/router"
"orchestra/internal/store"
"orchestra/internal/ui"
@@ -37,6 +39,9 @@ func id() string { return domain.NewID() }
const defaultHerdrPort = "9245"
func herdrAddress(rr registry.Registry, h registry.Herdr) string {
if h.Backend == "tmux" {
return rr.Endpoint(h)
}
if h.Address != "" {
return h.Address
}
@@ -57,9 +62,9 @@ type federatedAvailability struct {
localMachine string
}
// federatedReachability keeps the legacy TCP probe for local herdrs, while
// avoiding a coordinator-side probe of a remote worker's herdr socket. A
// remote harness is reachable precisely when its worker is registered (as
// federatedReachability keeps the legacy TCP probe for coordinator-owned
// herdrs, while avoiding a coordinator-side probe of a worker-owned backend.
// A worker-owned harness is reachable precisely when its worker is registered (as
// enforced by federatedAvailability); probing its raw herdr endpoint here
// would reintroduce the cross-machine Design A dependency.
type federatedReachability struct {
@@ -77,7 +82,7 @@ func (r federatedReachability) Reachable(address string, timeout time.Duration)
func remoteHerdrAddresses(rr registry.Registry, localMachine string) map[string]bool {
remote := map[string]bool{}
for _, h := range rr.Herdrs() {
if h.MachineID != localMachine {
if !coordinatorOwnsHerdr(h, localMachine) {
remote[herdrAddress(rr, h)] = true
}
}
@@ -89,6 +94,9 @@ func remoteHerdrAddresses(rr registry.Registry, localMachine string) map[string]
// reaching into that machine would turn a worker-owned health signal back
// into a misleading coordinator TCP result.
func coordinatorOwnsHerdr(h registry.Herdr, localMachine string) bool {
if h.Backend == "tmux" {
return false
}
return localMachine == "" || h.MachineID == localMachine
}
@@ -96,7 +104,7 @@ func (a federatedAvailability) Available(h registry.Herdr) bool {
if a.base != nil && !a.base.Available(h) {
return false
}
if a.localMachine == "" || h.MachineID == a.localMachine {
if coordinatorOwnsHerdr(h, a.localMachine) {
return true
}
return a.workers.Available(h.ID)
@@ -106,7 +114,7 @@ func (a federatedAvailability) Supports(h registry.Herdr, project string) bool {
// Locally-owned herdrs keep their static registry/project affinity. A
// remote worker must additionally prove it has a local checkout for the
// project before the router can offer it a lease.
if a.localMachine == "" || h.MachineID == a.localMachine {
if coordinatorOwnsHerdr(h, a.localMachine) {
return true
}
return a.workers.Supports(h.ID, project)
@@ -114,11 +122,18 @@ func (a federatedAvailability) Supports(h registry.Herdr, project string) bool {
func validateLocalMachine(rr registry.Registry, localMachine string) error {
machines := rr.Machines()
if len(machines) <= 1 {
requiresWorkerOwnership := false
for _, h := range rr.Herdrs() {
if h.Backend == "tmux" {
requiresWorkerOwnership = true
break
}
}
if len(machines) <= 1 && !requiresWorkerOwnership {
return nil
}
if localMachine == "" {
return fmt.Errorf("ORCHESTRA_MACHINE_ID is required for a multi-machine registry; refusing unsafe remote-herdr coordination")
return fmt.Errorf("ORCHESTRA_MACHINE_ID is required when the registry is multi-machine or has worker-owned backends")
}
if _, ok := rr.Machine(localMachine); !ok {
return fmt.Errorf("ORCHESTRA_MACHINE_ID %q is not in the registry", localMachine)
@@ -138,6 +153,15 @@ func main() {
var rr registry.Registry
var rt *router.Router
var coordinator *orchestrator.Coordinator
// submissionPublisher is nil until a forge is configured. Submission then
// returns the verified plan instead of performing it, which keeps `task
// pr` the only path without inventing a fake success.
var submissionPublisher func(operations.SubmissionPlan) operations.Publisher
// Worktree root per project, so a submission can find the checkout that
// holds the commit it is publishing.
projectRoots := map[string]string{}
// Pull-request readers by source name, for reflecting submitted work.
pullRequests := map[string]human.PullRequestSource{}
localMachine := os.Getenv("ORCHESTRA_MACHINE_ID")
workers := &federation.Registry{AdmitToken: os.Getenv("ORCHESTRA_FEDERATION_ADMIT_TOKEN"), StatePath: filepath.Join(dir, "federation-state.json")}
if err := workers.Load(); err != nil {
@@ -208,6 +232,7 @@ func main() {
for _, p := range rr.Projects() {
if p.Repo != "" && p.WorktreeRoot != "" {
projectRepos[p.ID] = orchestrator.ProjectRepo{Repo: p.Repo, WorktreeRoot: p.WorktreeRoot}
projectRoots[p.ID] = p.WorktreeRoot
}
}
worktrees := orchestrator.PerProjectGitWorktrees{
@@ -219,15 +244,15 @@ func main() {
return ok && coordinatorOwnsHerdr(h, localMachine)
}}
rt.OnLease = func(e domain.Event) error {
// In federated mode the coordinator must never inspect a remote
// In federated mode the coordinator must never inspect a worker-owned
// checkout. Its worker consumes the router-issued lease event and
// performs all Git/herdr operations on that machine (§2.1).
// performs all Git/backend operations on that machine (§2.1).
if localMachine != "" {
var p struct {
HarnessID string `json:"harness_id"`
}
if json.Unmarshal(e.Payload, &p) == nil {
if h, ok := rr.Herdr(p.HarnessID); ok && h.MachineID != localMachine {
if h, ok := rr.Herdr(p.HarnessID); ok && !coordinatorOwnsHerdr(h, localMachine) {
return nil
}
}
@@ -628,6 +653,18 @@ func main() {
mux.HandleFunc("/v1/admin/diagnostics", adminServer.Diagnostics)
mux.HandleFunc("/v1/tasks/", func(w http.ResponseWriter, r *http.Request) {
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
if r.Method == http.MethodGet && len(parts) == 4 && parts[3] == "intent" {
// The reduced authority for one task. Federation workers read this
// and render their own launch instruction with agentctx, so there
// is one renderer in the codebase rather than one per machine.
intent, err := s.EffectiveIntent(parts[2])
if err != nil {
http.Error(w, err.Error(), 404)
return
}
json.NewEncoder(w).Encode(intent)
return
}
if r.Method == http.MethodGet && coordinator != nil && len(parts) == 4 {
taskID, view := parts[2], parts[3]
if view == "health" {
@@ -653,6 +690,199 @@ func main() {
return
}
taskID, action := parts[2], parts[3]
if (action == "decision-request" || action == "deferred") && len(parts) == 4 {
// An agent surface may state a bounded question or a deferred
// finding. Neither moves the lifecycle: Orchestra decides what a
// question does to the task.
if err := authz.AuthorizeEvent(surface(r), "ApprovalRequested"); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
project, _ := rr.Project(t.Project)
r.Body = http.MaxBytesReader(w, r.Body, 64<<10)
if action == "deferred" {
var f domain.DeferredFinding
if json.NewDecoder(r.Body).Decode(&f) != nil {
http.Error(w, "invalid deferred finding", http.StatusBadRequest)
return
}
e, err := operations.RecordDeferredFinding(s, taskID, f)
if err != nil {
http.Error(w, err.Error(), 409)
return
}
json.NewEncoder(w).Encode(e)
return
}
var req domain.DecisionRequest
if json.NewDecoder(r.Body).Decode(&req) != nil {
http.Error(w, "invalid decision request", http.StatusBadRequest)
return
}
e, err := operations.RequestHumanDecision(s, project, taskID, req)
if errors.Is(err, operations.ErrDecisionBudgetSpent) {
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(map[string]string{"status": "operator_required", "detail": err.Error()})
return
}
if err != nil {
http.Error(w, err.Error(), 409)
return
}
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(e)
return
}
if action == "submission" && len(parts) == 4 {
// The only path from a reviewed implementation to the human. It
// verifies first and returns the plan, or performs the submission
// when the caller supplies a publisher-backed request.
if err := authz.AuthorizeEvent(surface(r), domain.EventTaskSubmitted); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
project, ok := rr.Project(t.Project)
if !ok {
http.Error(w, "unknown project "+t.Project, 409)
return
}
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
var body struct {
HeadSHA string `json:"head_sha"`
Gate domain.GateResult `json:"gate"`
Notes operations.Notes `json:"notes"`
DryRun bool `json:"dry_run"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || body.HeadSHA == "" {
http.Error(w, "head_sha and gate are required", http.StatusBadRequest)
return
}
check := domain.CheckSubmission(t, body.HeadSHA, body.Gate)
check.Reasons = append(check.Reasons, t.RequirePhaseArtifacts(project.Phases())...)
check.Eligible = len(check.Reasons) == 0
if body.DryRun || !check.Eligible {
code := http.StatusOK
if !check.Eligible {
code = http.StatusConflict
}
w.WriteHeader(code)
json.NewEncoder(w).Encode(check)
return
}
plan, err := operations.PrepareSubmission(s, project, taskID, body.HeadSHA, body.Gate, body.Notes)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if submissionPublisher == nil {
// No forge configured: hand back the verified plan so an
// operator can perform the submission themselves.
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(map[string]any{"status": "no_publisher", "plan": plan})
return
}
e, err := operations.ExecuteSubmission(r.Context(), s, plan, submissionPublisher(plan), nil)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(e)
return
}
if action == "review" && len(parts) == 4 {
// Entering review and sealing a review are both Orchestra's, not
// the reviewing session's. The session only supplies findings.
if err := authz.AuthorizeEvent(surface(r), domain.EventReviewRecorded); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
project, ok := rr.Project(t.Project)
if !ok {
http.Error(w, "unknown project "+t.Project, 409)
return
}
r.Body = http.MaxBytesReader(w, r.Body, int64(review.MaxDiffBytes)+(1<<20))
var body struct {
Evidence *review.Evidence `json:"evidence"`
Result *review.Result `json:"result"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || (body.Evidence == nil) == (body.Result == nil) {
http.Error(w, "send either evidence (to enter review) or result (to seal one)", http.StatusBadRequest)
return
}
var e domain.Event
var err error
if body.Evidence != nil {
e, err = operations.EnterReview(s, project, taskID, *body.Evidence)
} else {
e, err = operations.RecordReview(s, project, taskID, *body.Result)
}
if errors.Is(err, operations.ErrReviewNotEligible) || errors.Is(err, domain.ErrInvalid) {
http.Error(w, err.Error(), http.StatusConflict)
return
}
if err != nil {
http.Error(w, err.Error(), 409)
return
}
json.NewEncoder(w).Encode(e)
return
}
if action == "phase" && len(parts) == 4 {
// Orchestra owns the phase. An agent asks for a change through the
// approval surface; this endpoint is how the decision is applied.
if err := authz.AuthorizeEvent(surface(r), domain.EventWorkPhaseChanged); err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
t, ok := s.Task(taskID)
if !ok {
http.Error(w, "task not found", 404)
return
}
project, ok := rr.Project(t.Project)
if !ok {
http.Error(w, "unknown project "+t.Project, 409)
return
}
// The body, when present, is the artifact the finished phase
// produced. Bounded like every other artifact upload.
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
artifact, readErr := io.ReadAll(r.Body)
if readErr != nil {
http.Error(w, "artifact unreadable", http.StatusRequestEntityTooLarge)
return
}
e, err := operations.AdvanceWorkPhase(s, project, taskID, artifact)
if errors.Is(err, operations.ErrTrajectoryGate) {
// Not a failure: the task is blocked on the human, and the
// packet is on the block event the notification surfaces read.
w.WriteHeader(http.StatusAccepted)
json.NewEncoder(w).Encode(map[string]string{"status": "trajectory_gate", "detail": err.Error()})
return
}
if err != nil {
http.Error(w, err.Error(), 409)
return
}
json.NewEncoder(w).Encode(e)
return
}
if action == "approval" {
if err := authz.AuthorizeEvent(surface(r), map[bool]string{true: "ApprovalRequested", false: "ApprovalGranted"}[len(parts) == 4]); err != nil && len(parts) == 4 {
http.Error(w, err.Error(), http.StatusForbidden)
@@ -791,6 +1021,12 @@ func main() {
// past their lease TTL). Calling ExpireLeases here too would
// just resurrect that race, so leave reclaim to the
// coordinator and only keep retrying pending assignment.
// A blocked task is invisible to the router, so answered
// blockers are returned to the queue here, immediately
// upstream of assignment.
if _, err := operations.ResumeAnsweredBlockers(s); err != nil {
log.Printf("resume answered blockers: %v", err)
}
if coordinator != nil {
if _, err := rt.AssignPending(); err != nil {
log.Printf("route expired task: %v", err)
@@ -805,6 +1041,45 @@ func main() {
}
}()
}
if len(pullRequests) > 0 {
// Submitted work is reconciled on its own loop, not behind
// Store.PreLease: an in-review task cannot be leased, so a pre-lease
// hook could never see the feedback that should make it leasable.
trust := human.Trust{
Accepted: splitList(os.Getenv("ORCHESTRA_REVIEW_ACTORS")),
Ignored: splitList(os.Getenv("ORCHESTRA_REVIEW_IGNORE_ACTORS")),
}
go func() {
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for range ticker.C {
for _, t := range s.Tasks() {
if t.Submission == nil || (t.State != domain.StateInReview && t.State != domain.StateQueued) {
continue
}
source, ok := pullRequests[t.Submission.PR.Provider]
if !ok {
continue
}
project, ok := rr.Project(t.Project)
if !ok {
continue
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
state, err := source.PullRequest(ctx, t)
cancel()
if err != nil {
// Observable, and the task stays exactly where it was.
log.Printf("reflect submission %s: %v", t.ID, err)
continue
}
if _, err := operations.ReflectSubmission(s, project, t.ID, state, trust); err != nil {
log.Printf("reflect submission %s: %v", t.ID, err)
}
}
}
}()
}
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok\n")) })
mux.HandleFunc("/readyz", adminServer.Readiness)
mux.HandleFunc("/v1/providers/health", func(w http.ResponseWriter, r *http.Request) {
@@ -869,6 +1144,42 @@ func main() {
}
return wid, nil
}
mux.HandleFunc("/v1/federation/turn", func(w http.ResponseWriter, r *http.Request) {
if _, err := workerAuth(r); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
if coordinator == nil {
http.Error(w, "coordinator not configured", http.StatusServiceUnavailable)
return
}
var body struct {
TaskID string `json:"task_id"`
LeaseEpoch string `json:"lease_epoch"`
Verdict string `json:"verdict"`
Delivered []string `json:"delivered_decisions"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || body.TaskID == "" || body.LeaseEpoch == "" {
http.Error(w, "task_id and lease_epoch are required", http.StatusBadRequest)
return
}
switch body.Verdict {
case orchestrator.TurnContinue, orchestrator.TurnPrepareHandoff, orchestrator.TurnRotateNow, orchestrator.TurnRefuse:
default:
http.Error(w, "unknown verdict "+body.Verdict, http.StatusBadRequest)
return
}
verdict, decisions, err := coordinator.RemoteTurn(r.Context(), body.TaskID, body.LeaseEpoch, body.Verdict, body.Delivered)
if err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
json.NewEncoder(w).Encode(federation.TurnDecision{Verdict: verdict, Decisions: decisions})
})
mux.HandleFunc("/v1/federation/commands", func(w http.ResponseWriter, r *http.Request) {
wid, err := workerAuth(r)
if err != nil {
@@ -1218,6 +1529,7 @@ func main() {
Token: os.Getenv("ORCHESTRA_GITEA_TOKEN"), WebhookSecret: os.Getenv("ORCHESTRA_GITEA_WEBHOOK_SECRET"),
}}
}
humanSources := map[string]human.Source{}
if len(giteaSources) > 0 {
reflectors := map[string]provider.Gitea{}
for _, c := range giteaSources {
@@ -1252,6 +1564,38 @@ func main() {
}}
sup.Start(context.Background())
providerHealth[name] = sup
humanSources[g.SourceName()] = provider.GiteaComments{Gitea: g}
if publisher := (provider.GiteaPublisher{Gitea: g, Base: os.Getenv("ORCHESTRA_PR_BASE")}); publisher.BaseURL != "" {
root := projectRoots[c.Project]
submissionPublisher = func(plan operations.SubmissionPlan) operations.Publisher {
p := publisher
p.Root = filepath.Join(root, plan.TaskID)
return p
}
pullRequests[g.SourceName()] = publisher
}
}
}
// Reconciliation runs immediately before every lease, which is where
// ownership of a task begins. A configured source that cannot be read
// refuses the lease rather than letting a successor resume from an older
// intent. Set ORCHESTRA_HUMAN_RECONCILE=off to disable it for an
// operator who needs to run while a source is down.
if len(humanSources) > 0 && !strings.EqualFold(os.Getenv("ORCHESTRA_HUMAN_RECONCILE"), "off") {
reconciler := &human.Reconciler{Store: s, Sources: humanSources, Timeout: 30 * time.Second}
s.PreLease = func(taskID string) error {
return reconciler.Reconcile(context.Background(), taskID)
}
if coordinator != nil {
// The second reconciliation point: a verified turn boundary on a
// lease that is already running. Nothing here preempts the agent.
coordinator.ReconcileHumanInput = reconciler.Reconcile
// After this many consecutive failures at that boundary, the
// session is asked to hand off rather than keep running on intent
// Orchestra can no longer refresh.
if v, parseErr := strconv.Atoi(os.Getenv("ORCHESTRA_RECONCILE_FAILURE_HANDOFF")); parseErr == nil && v > 0 {
coordinator.ReconcileFailureHandoff = v
}
}
}
if path := os.Getenv("ORCHESTRA_JSONL"); path != "" {
@@ -1321,6 +1665,11 @@ func main() {
tokens := map[authz.Surface]string{
authz.TUI: os.Getenv("ORCHESTRA_TUI_TOKEN"),
authz.MCP: os.Getenv("ORCHESTRA_MCP_TOKEN"), authz.Maven: os.Getenv("ORCHESTRA_MAVEN_TOKEN"),
// The credential an in-pane coding session presents. It buys the two
// request endpoints and read access — never a lifecycle mutation. The
// forge, Vikunja and operator tokens must never reach an agent pane;
// this one is the only Orchestra credential an agent may hold.
authz.Agent: os.Getenv("ORCHESTRA_AGENT_TOKEN"),
// S12: this is the credential callers present *to* Orchestra on the
// ntfy surface. ORCHESTRA_NTFY_TOKEN is a different secret entirely —
// it is handed out to the third-party ntfy server (see the sender
@@ -1329,3 +1678,14 @@ func main() {
}
log.Fatal(http.ListenAndServe(":"+port, authz.HTTPWithSessions(tokens, sessions, mux)))
}
// splitList reads a comma-separated env list, ignoring blanks.
func splitList(v string) []string {
var out []string
for _, item := range strings.Split(v, ",") {
if s := strings.TrimSpace(item); s != "" {
out = append(out, s)
}
}
return out
}
+20
View File
@@ -36,6 +36,7 @@ func TestFederatedReachabilityDefersRemoteHerdrToWorkerHeartbeat(t *testing.T) {
func TestCoordinatorOwnsOnlyLocalHerdrInFederationMode(t *testing.T) {
local := registry.Herdr{ID: "homesrv-opencode", MachineID: "homesrv"}
localTmux := registry.Herdr{ID: "homesrv-claude", MachineID: "homesrv", Backend: "tmux", Harness: "claude"}
remote := registry.Herdr{ID: "workpc-opencode", MachineID: "workpc"}
if !coordinatorOwnsHerdr(local, "homesrv") {
t.Fatal("coordinator does not own its local herdr")
@@ -43,6 +44,9 @@ func TestCoordinatorOwnsOnlyLocalHerdrInFederationMode(t *testing.T) {
if coordinatorOwnsHerdr(remote, "homesrv") {
t.Fatal("coordinator claimed a worker-owned remote herdr")
}
if coordinatorOwnsHerdr(localTmux, "homesrv") {
t.Fatal("coordinator claimed a local worker-owned tmux backend")
}
if !coordinatorOwnsHerdr(remote, "") {
t.Fatal("single-machine mode should retain legacy local ownership")
}
@@ -69,3 +73,19 @@ func TestMultiMachineRegistryRequiresKnownLocalMachine(t *testing.T) {
t.Fatalf("known local machine rejected: %v", err)
}
}
func TestTmuxRegistryRequiresMachineIdentityEvenOnOneMachine(t *testing.T) {
r, err := registry.New(registry.Config{
Machines: []registry.Machine{{ID: "homesrv", Address: "homesrv:9145"}},
Herdrs: []registry.Herdr{{ID: "homesrv-claude", MachineID: "homesrv", Backend: "tmux", Harness: "claude"}},
})
if err != nil {
t.Fatal(err)
}
if err := validateLocalMachine(r, ""); err == nil {
t.Fatal("worker-owned tmux backend accepted without machine identity")
}
if err := validateLocalMachine(r, "homesrv"); err != nil {
t.Fatal(err)
}
}