v3 workflow: intent, phases, review, submission, enforcement, burn-in

The v3 stack, previously an uncommitted working tree, plus this session's two units and the burn-in instrument. This commit is the burn-in build identity: coordinator and worker must both report this revision before a task is created. Workflow (earlier sessions, uncommitted until now): human decision events and reduction, source cursors and reconcile-before-launch, turn-boundary reconciliation, internal/agentctx as the single renderer, ace-fca phases with sealed artifacts, the trajectory gate, bounded grilling, independent review, task pr enforcement, and human review reflection. Capability restrictions at the agent boundary: an authz.Agent surface at GatedWrite may ask and may not act. It also fixes two bugs the unit exposed -- gated surfaces could not reach the two endpoints written for them, and RequestHumanDecision would block an unowned task while rejecting a question from the session that did own it. Turn-boundary reconcile-failure escalation: a streak of consecutive failures asks the session to hand off, fenced on the lease epoch, with reconcile_failure as a real handoff reason. The worker was dropping the coordinator's verdict on the floor; it now acts on it. Burn-in: herdr.WriteLaunchContext dumps the exact agentctx.Build result to <worktree>/.orchestra/launch.md at every launch, local and federated. BURNIN.md is the runbook. deploy/build.sh stamps both binaries from one commit. go build, go vet and go test ./... pass, 20 packages. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-26 18:31:20 +04:00
parent 97a9c65302
commit 7f12c7fc37
78 changed files with 16417 additions and 352 deletions
@@ -17,6 +17,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 )
@@ -153,6 +154,8 @@ func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer ln.Close()
+	var promptMu sync.Mutex
+	var prompts []string
 	go func() {
 		for {
 			c, e := ln.Accept()
@@ -176,6 +179,12 @@ func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
 				case "pane.read":
 					result = `{"read":{"text":""}}`
 				case "agent.prompt":
+					if m, ok := r.Params.(map[string]any); ok {
+						text, _ := m["text"].(string)
+						promptMu.Lock()
+						prompts = append(prompts, text)
+						promptMu.Unlock()
+					}
 					result = `{}`
 				default:
 					result = `{}`
@@ -185,7 +194,23 @@ func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
 		}
 	}()
 	root := filepath.Join(filepath.Dir(repo), "worktrees")
-	w := &worker{herdr: &herdr.Client{Path: ln.Addr().String()}, repo: repo, root: root, remote: "origin", harness: "opencode", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: filepath.Join(t.TempDir(), "state.json"), hard: .75}
+	// The worker renders its own launch instruction from the coordinator's
+	// reduced authority, so the fake API must serve it.
+	api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/intent") {
+			json.NewEncoder(w).Encode(domain.EffectiveIntent{
+				Task: domain.Task{ID: "task", Title: "test", Description: "do work"},
+				Decisions: []domain.HumanDecision{{
+					ID: "d1", TaskID: "task", Kind: domain.HumanDecisionCorrection,
+					Subject: "strategy", Value: "no, use b",
+				}},
+			})
+			return
+		}
+		w.WriteHeader(200)
+	}))
+	defer api.Close()
+	w := &worker{api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, herdr: &herdr.Client{Path: ln.Addr().String()}, repo: repo, root: root, remote: "origin", harness: "opencode", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: filepath.Join(t.TempDir(), "state.json"), hard: .75}
 	// New() is needed for its pane map; override the address for the fake.
 	w.herdr = herdr.New(ln.Addr().String())
 	task := domain.Task{ID: "task", Source: "s", ExternalID: "x", Project: "p", Title: "test", Description: "do work"}
@@ -198,6 +223,16 @@ func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
 	if _, err := os.Stat(filepath.Join(root, "task", "TASK.md")); err != nil {
 		t.Fatalf("TASK.md: %v", err)
 	}
+	// The deployed worker path renders through agentctx, so a decision the
+	// human recorded before this session existed reaches the agent.
+	promptMu.Lock()
+	launched := strings.Join(prompts, "\n")
+	promptMu.Unlock()
+	for _, want := range []string{"## Current human decisions", "no, use b", "Authority order"} {
+		if !strings.Contains(launched, want) {
+			t.Fatalf("worker launch instruction missing %q:\n%s", want, launched)
+		}
+	}
 }

 func TestFinalizeRunsGateCommitsPushesAndVerifiesRemote(t *testing.T) {
@@ -332,3 +367,351 @@ func TestApprovalResponseOpenCodeAllowOnce(t *testing.T) {
 }

 func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }
+
+type recordingBackend struct {
+	status  string
+	calls   []string
+	prompts []string
+}
+
+func (b *recordingBackend) Kind() string                { return "recording" }
+func (b *recordingBackend) Check(context.Context) error { return nil }
+func (b *recordingBackend) Worktree(_ context.Context, _, path, _ string) (string, error) {
+	return path, nil
+}
+func (b *recordingBackend) StartAgent(_ context.Context, _, path, _, harness, _ string) (herdr.Session, error) {
+	return herdr.Session{PaneID: "pane", Worktree: path, Harness: harness}, nil
+}
+func (b *recordingBackend) Prompt(_ context.Context, _, text string, _ time.Duration) error {
+	b.prompts = append(b.prompts, text)
+	return nil
+}
+func (b *recordingBackend) Kill(context.Context, herdr.Session) error { return nil }
+func (b *recordingBackend) AgentStatus(context.Context, herdr.Session) (string, error) {
+	if b.status == "" {
+		return "idle", nil
+	}
+	return b.status, nil
+}
+func (b *recordingBackend) PaneCapture(context.Context, herdr.Session, string) (string, error) {
+	return "", nil
+}
+func (b *recordingBackend) SendText(_ context.Context, _ herdr.Session, text string) error {
+	b.calls = append(b.calls, "text:"+text)
+	return nil
+}
+func (b *recordingBackend) SendKeys(_ context.Context, _ herdr.Session, keys []string) error {
+	b.calls = append(b.calls, "keys:"+strings.Join(keys, ","))
+	return nil
+}
+func (b *recordingBackend) ReleaseAgent(context.Context, herdr.Session, string) error { return nil }
+
+func TestClaudeContextHookRolloverSendsClearThenHandoff(t *testing.T) {
+	worktree := t.TempDir()
+	handoff := filepath.Join(worktree, "HANDOFF.md")
+	if err := os.WriteFile(handoff, []byte("old handoff"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	oldSHA, err := fileSHA256(handoff)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(handoff, []byte("new durable handoff"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	backend := &recordingBackend{}
+	session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude", SessionFile: "exhausted.jsonl", ContextHandoffSHA: oldSHA}
+	w := &worker{
+		backend:     backend,
+		harness:     "claude",
+		sessions:    map[string]herdr.Session{"task": session},
+		tasks:       map[string]domain.Task{},
+		leases:      map[string]lease{},
+		releases:    map[string]releaseTransaction{},
+		quarantined: map[string]bool{},
+		statePath:   filepath.Join(t.TempDir(), "state.json"),
+	}
+	if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil {
+		t.Fatal(err)
+	}
+	want := []string{"text:/clear", "keys:ENTER", "text:@HANDOFF.md", "keys:ENTER"}
+	if !reflect.DeepEqual(backend.calls, want) {
+		t.Fatalf("rollover calls=%v want %v", backend.calls, want)
+	}
+	got := w.sessions["task"]
+	newSHA, _ := fileSHA256(handoff)
+	if got.ContextHandoffSHA != newSHA || got.ContextResetSHA != "" || got.ContextResetPhase != "" || got.SessionFile != "" {
+		t.Fatalf("rollover state was not finalized: %+v", got)
+	}
+	if err := w.advanceClaudeContextReset(context.Background(), "task", got); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(backend.calls, want) {
+		t.Fatalf("unchanged handoff retriggered rollover: %v", backend.calls)
+	}
+}
+
+func TestClaudeContextHookRolloverWaitsForIdle(t *testing.T) {
+	worktree := t.TempDir()
+	if err := os.WriteFile(filepath.Join(worktree, "HANDOFF.md"), []byte("handoff"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	backend := &recordingBackend{status: "busy"}
+	session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude"}
+	w := &worker{backend: backend, harness: "claude", sessions: map[string]herdr.Session{"task": session}, statePath: filepath.Join(t.TempDir(), "state.json")}
+	if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil {
+		t.Fatal(err)
+	}
+	if len(backend.calls) != 0 {
+		t.Fatalf("busy Claude received rollover input: %v", backend.calls)
+	}
+}
+
+func TestHarnessSpecsFallsBackToSingleHarnessEnvironment(t *testing.T) {
+	t.Setenv("ORCHESTRA_WORKER_HERDR_ID", "workpc-opencode")
+	t.Setenv("ORCHESTRA_WORKER_HARNESS", "opencode")
+	t.Setenv("ORCHESTRA_WORKER_TOKEN", "secret")
+	t.Setenv("ORCHESTRA_WORKER_HERDR", "127.0.0.1:9247")
+	specs := harnessSpecs()
+	if len(specs) != 1 {
+		t.Fatalf("want one legacy spec, got %d", len(specs))
+	}
+	if specs[0].ID != "workpc-opencode" || specs[0].Harness != "opencode" || specs[0].Token != "secret" || specs[0].Herdr != "127.0.0.1:9247" {
+		t.Fatalf("legacy environment was not carried through: %+v", specs[0])
+	}
+}
+
+func TestHarnessSpecsReadsMultipleHarnessesAndPerIdentityTokens(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "harnesses.json")
+	if err := os.WriteFile(path, []byte(`[
+		{"id":"workpc-claude","harness":"claude","backend":"tmux","tmux_socket":"orchestra","command":"/usr/bin/true"},
+		{"id":"workpc-opencode","harness":"opencode","backend":"herdr","herdr":"127.0.0.1:9247","token":"inline"}
+	]`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE", path)
+	t.Setenv("ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE", "from-env")
+	specs := harnessSpecs()
+	if len(specs) != 2 {
+		t.Fatalf("want two specs, got %d", len(specs))
+	}
+	if specs[0].Token != "from-env" {
+		t.Fatalf("per-identity token env was not consulted: %q", specs[0].Token)
+	}
+	if specs[1].Token != "inline" {
+		t.Fatalf("config-file token was overridden: %q", specs[1].Token)
+	}
+	// Distinct backends in one process is the point of the multi-harness form.
+	if got := backendFor(specs[0]).Kind(); got != "tmux" {
+		t.Fatalf("first harness backend = %q, want tmux", got)
+	}
+	if got := backendFor(specs[1]).Kind(); got != "herdr" {
+		t.Fatalf("second harness backend = %q, want herdr", got)
+	}
+}
+
+func TestTokenEnvKeySanitizesHarnessID(t *testing.T) {
+	if got := tokenEnvKey("workpc-claude"); got != "ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE" {
+		t.Fatalf("tokenEnvKey = %q", got)
+	}
+	if got := tokenEnvKey("box.1-opencode"); got != "ORCHESTRA_WORKER_TOKEN_BOX_1_OPENCODE" {
+		t.Fatalf("tokenEnvKey = %q", got)
+	}
+}
+
+func TestStatePathIsolatesEveryHarnessIdentity(t *testing.T) {
+	claude := harnessSpec{ID: "workpc-claude"}
+	opencode := harnessSpec{ID: "workpc-opencode"}
+	// The single-harness form must keep the historical path so an upgraded
+	// deployment recovers its sessions rather than orphaning their panes.
+	if got := statePathFor(opencode, "/wt", "", true); got != "/wt/.orchestra-worker-state.json" {
+		t.Fatalf("single-harness state path changed: %q", got)
+	}
+	a := statePathFor(claude, "/wt", "", false)
+	b := statePathFor(opencode, "/wt", "", false)
+	if a == b {
+		t.Fatalf("two harnesses share one state file: %q", a)
+	}
+	if got := statePathFor(claude, "/wt", "/var/lib/orchestra-worker", false); got != "/var/lib/orchestra-worker/state-workpc-claude.json" {
+		t.Fatalf("state dir ignored: %q", got)
+	}
+	explicit := harnessSpec{ID: "workpc-claude", State: "/srv/claude.json"}
+	if got := statePathFor(explicit, "/wt", "/var/lib/orchestra-worker", false); got != "/srv/claude.json" {
+		t.Fatalf("explicit per-entry state path ignored: %q", got)
+	}
+}
+
+// boundaryAdapter reports a verified turn boundary without a live pane.
+type boundaryAdapter struct {
+	at  bool
+	err error
+}
+
+func (b boundaryAdapter) Lease(context.Context, string, string) (herdr.Session, error) {
+	return herdr.Session{}, nil
+}
+func (b boundaryAdapter) Release(context.Context, herdr.Session) (string, error) { return "", nil }
+func (b boundaryAdapter) Kill(context.Context, herdr.Session) error              { return nil }
+func (b boundaryAdapter) Occupancy(herdr.Session) (float64, error)               { return 0, nil }
+func (b boundaryAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
+	return b.at, b.err
+}
+
+// The federated half of the authority model: at a verified boundary the worker
+// asks the coordinator, delivers the correction into its own pane once, and
+// records it so the next boundary stays quiet.
+func TestFederatedTurnDeliversCorrectionOnce(t *testing.T) {
+	prompts := make(chan string, 4)
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, e := ln.Accept()
+			if e != nil {
+				return
+			}
+			go func() {
+				defer c.Close()
+				var request herdr.Request
+				if json.NewDecoder(c).Decode(&request) != nil {
+					return
+				}
+				result := `{}`
+				switch request.Method {
+				case "pane.get":
+					result = `{"pane":{"agent":"opencode","agent_status":"idle"}}`
+				case "pane.read":
+					result = `{"read":{"text":""}}`
+				case "agent.prompt":
+					if m, ok := request.Params.(map[string]any); ok {
+						text, _ := m["text"].(string)
+						prompts <- text
+					}
+				}
+				_ = json.NewEncoder(c).Encode(herdr.Response{ID: request.ID, Result: json.RawMessage(result)})
+			}()
+		}
+	}()
+
+	var turnCalls int
+	var lastDelivered []string
+	fail := false
+	api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasSuffix(r.URL.Path, "/federation/turn") {
+			w.WriteHeader(200)
+			return
+		}
+		if fail {
+			http.Error(w, "coordinator down", 503)
+			return
+		}
+		turnCalls++
+		var body struct {
+			Delivered []string `json:"delivered_decisions"`
+		}
+		_ = json.NewDecoder(r.Body).Decode(&body)
+		lastDelivered = body.Delivered
+		out := federation.TurnDecision{Verdict: "continue"}
+		if len(body.Delivered) == 0 {
+			out.Decisions = []domain.HumanDecision{{
+				ID: "d1", Kind: domain.HumanDecisionCorrection, Subject: "strategy", Value: "no, use b",
+			}}
+		}
+		json.NewEncoder(w).Encode(out)
+	}))
+	defer api.Close()
+
+	w := &worker{
+		api:   federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
+		herdr: herdr.New(ln.Addr().String()), harness: "opencode",
+		sessions:  map[string]herdr.Session{"task": {PaneID: "pane", Harness: "opencode"}},
+		leases:    map[string]lease{"task": {Epoch: "e1", Version: 2}},
+		statePath: t.TempDir() + "/state.json",
+	}
+
+	// Not at a boundary: nothing is asked and nothing is delivered.
+	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: false}, "continue")
+	if turnCalls != 0 {
+		t.Fatal("asked the coordinator without a verified boundary")
+	}
+
+	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
+	select {
+	case got := <-prompts:
+		if !strings.Contains(got, "no, use b") || !strings.Contains(got, "outrank") {
+			t.Fatalf("delivered prompt = %q", got)
+		}
+	default:
+		t.Fatal("correction was not delivered to the pane")
+	}
+	if ids := w.sessions["task"].DeliveredDecisions; len(ids) != 1 || ids[0] != "d1" {
+		t.Fatalf("delivered ids = %v", ids)
+	}
+
+	// Second boundary: the worker reports what it has shown, so nothing repeats.
+	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
+	if len(lastDelivered) != 1 || lastDelivered[0] != "d1" {
+		t.Fatalf("worker did not report delivered ids: %v", lastDelivered)
+	}
+	select {
+	case got := <-prompts:
+		t.Fatalf("correction delivered twice: %q", got)
+	default:
+	}
+
+	// Coordinator unreachable: observable, and the session keeps running.
+	fail = true
+	w.lastError = ""
+	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
+	if !strings.Contains(w.lastError, "federated turn") {
+		t.Fatalf("transport failure not observable: %q", w.lastError)
+	}
+}
+
+// A worker must act on the coordinator's verdict, not only on its decisions.
+// When the coordinator can no longer reconcile human input for this task, it
+// answers prepare_handoff, and the worker asks its own agent to hand off. The
+// release loop then takes over on the next tick, because the handoff report is
+// what it watches for.
+func TestFederatedTurnActsOnPrepareHandoffVerdict(t *testing.T) {
+	api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/federation/turn" {
+			rw.WriteHeader(http.StatusNotFound)
+			return
+		}
+		json.NewEncoder(rw).Encode(federation.TurnDecision{Verdict: orchestrator.TurnPrepareHandoff})
+	}))
+	defer api.Close()
+
+	backend := &recordingBackend{}
+	session := herdr.Session{PaneID: "pane", Worktree: t.TempDir(), Harness: "opencode"}
+	w := &worker{
+		api:       federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
+		backend:   backend,
+		harness:   "opencode",
+		sessions:  map[string]herdr.Session{"task": session},
+		leases:    map[string]lease{"task": {Epoch: "e1", Version: 2}},
+		tasks:     map[string]domain.Task{},
+		statePath: filepath.Join(t.TempDir(), "state.json"),
+	}
+	a := herdr.CLIAdapter{Backend: backend, Harness: "opencode"}
+	w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue)
+
+	got := w.sessions["task"]
+	if !got.HandoffRequested || got.HandoffReason != "reconcile_failure" {
+		t.Fatalf("session did not record the requested handoff: %+v", got)
+	}
+	if len(backend.prompts) != 1 || !strings.Contains(backend.prompts[0], "reconcile_failure") {
+		t.Fatalf("agent was not asked to hand off: %v", backend.prompts)
+	}
+
+	// Idempotent: a second boundary must not re-prompt a session that is
+	// already preparing its handoff.
+	w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue)
+	if len(backend.prompts) != 1 {
+		t.Fatalf("handoff re-requested: %v", backend.prompts)
+	}
+}