package main import ( "bufio" "context" "encoding/json" "errors" "net" "net/http" "net/http/httptest" "orchestra/internal/domain" "orchestra/internal/federation" "orchestra/internal/herdr" "orchestra/internal/orchestrator" "os" "os/exec" "path/filepath" "reflect" "strings" "sync" "testing" "time" ) func TestWorkerReregistersAfterCoordinatorForgetsIt(t *testing.T) { registered := false s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost || r.URL.Path != "/v1/federation/workers" { t.Fatalf("unexpected %s %s", r.Method, r.URL.Path) } registered = true w.WriteHeader(http.StatusCreated) })) defer s.Close() w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, registration: federation.Worker{ID: "h", Capacity: 1}} if !w.reRegisterAfterCoordinatorRestart(context.Background(), errors.New("federation: 401 Unauthorized: unknown worker")) { t.Fatal("worker did not identify a coordinator restart") } if !registered { t.Fatal("worker did not register") } } func TestWorkerRefusesCorruptDurableState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") if err := os.WriteFile(path, []byte(`{"cursor":`), 0600); err != nil { t.Fatal(err) } w := &worker{statePath: path} if err := w.load(); err == nil { t.Fatal("corrupt worker state was accepted") } } func TestClassifyLaunchErrorPreservesUncertainLivePane(t *testing.T) { if got := classifyLaunchError(errors.New("prompt response lost"), true); got != "launch_uncertain" { t.Fatalf("live pane class=%q", got) } if got := classifyLaunchError(errors.New("pickup anchor mismatch"), false); got != "invalid_handoff" { t.Fatalf("bad handoff class=%q", got) } if got := classifyLaunchError(errors.New("temporary herdr outage"), false); got != "launch_transient" { t.Fatalf("transient class=%q", got) } } func TestInitialReplayDoesNotResurrectReleasedLease(t *testing.T) { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v1/federation/events": _, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"c","type":"TaskCreated","task_id":"t","version":1,"payload":{"source":"s","external_id":"x","project":"p"},"surface":"system"},{"seq":2,"id":"l","type":"TaskLeased","task_id":"t","version":2,"payload":{"harness_id":"h"},"surface":"system"},{"seq":3,"id":"r","type":"TaskReleased","task_id":"t","version":3,"payload":{"reason":"expired"},"surface":"system"}]}`)) case "/v1/federation/events/ack": w.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected %s", r.URL.Path) } })) defer s.Close() w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, harnessID: "h", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: t.TempDir() + "/state.json", hard: .75} if err := w.once(context.Background()); err != nil { t.Fatal(err) } if len(w.leases) != 0 || len(w.sessions) != 0 { t.Fatalf("replayed lease survived: leases=%v sessions=%v", w.leases, w.sessions) } if w.cursor != 3 { t.Fatalf("cursor=%d want 3", w.cursor) } } func TestWorkerHydratesLeaseMissingTaskCache(t *testing.T) { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v1/federation/events": _, _ = w.Write([]byte(`{"cursor":2,"events":[{"seq":2,"id":"l","type":"TaskLeased","task_id":"t","version":2,"payload":{"harness_id":"h"},"surface":"system"}]}`)) case "/v1/tasks": _, _ = w.Write([]byte(`[{"id":"t","source":"s","external_id":"x","project":"p","state":"leased"}]`)) case "/v1/federation/events/ack": w.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected %s", r.URL.Path) } })) defer s.Close() w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, harnessID: "h", cursor: 1, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{"t": {}}, leases: map[string]lease{}, statePath: t.TempDir() + "/state.json", hard: .75} if err := w.once(context.Background()); err != nil { t.Fatal(err) } if _, ok := w.tasks["t"]; !ok || w.cursor != 2 { t.Fatalf("task was not hydrated: tasks=%v cursor=%d", w.tasks, w.cursor) } } func TestWorkerReconcilesLeaseAfterEmptyEventReplay(t *testing.T) { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v1/federation/events": _, _ = w.Write([]byte(`{"cursor":42,"events":[]}`)) case "/v1/tasks": _, _ = w.Write([]byte(`[{"id":"t","source":"s","external_id":"x","project":"p","state":"leased","lease":{"harness_id":"h"}}]`)) case "/v1/federation/events/ack": w.WriteHeader(http.StatusNoContent) default: t.Fatalf("unexpected %s", r.URL.Path) } })) defer s.Close() w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, harnessID: "h", cursor: 42, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{"t": {}}, leases: map[string]lease{}, statePath: t.TempDir() + "/state.json", hard: .75} if err := w.once(context.Background()); err != nil { t.Fatal(err) } if _, ok := w.leases["t"]; !ok { t.Fatal("empty event replay did not recover current lease") } } func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) { remote := filepath.Join(t.TempDir(), "remote.git") if out, err := exec.Command("git", "init", "--bare", remote).CombinedOutput(); err != nil { t.Fatalf("remote: %v %s", err, out) } seed := t.TempDir() for _, a := range [][]string{{"init", seed}, {"-C", seed, "config", "user.email", "t@t"}, {"-C", seed, "config", "user.name", "t"}, {"-C", seed, "commit", "--allow-empty", "-m", "init"}, {"-C", seed, "remote", "add", "origin", remote}, {"-C", seed, "push", "-u", "origin", "HEAD:master"}} { if out, err := exec.Command("git", a...).CombinedOutput(); err != nil { t.Fatalf("git %v: %v %s", a, err, out) } } repo := filepath.Join(t.TempDir(), "repo") if out, err := exec.Command("git", "clone", remote, repo).CombinedOutput(); err != nil { t.Fatal(string(out)) } ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer ln.Close() var promptMu sync.Mutex var prompts []string go func() { for { c, e := ln.Accept() if e != nil { return } go func() { defer c.Close() var r herdr.Request if json.NewDecoder(bufio.NewReader(c)).Decode(&r) != nil { return } var result string switch r.Method { case "worktree.create": result = `{"path":"` + filepath.Join(filepath.Dir(repo), "worktrees", "task") + `","root_pane":{"pane_id":"p1"}}` case "agent.start": result = `{}` case "pane.get": result = `{"pane":{"agent":"opencode","agent_status":"idle"}}` case "pane.read": result = `{"read":{"text":""}}` case "agent.prompt": if m, ok := r.Params.(map[string]any); ok { text, _ := m["text"].(string) promptMu.Lock() prompts = append(prompts, text) promptMu.Unlock() } result = `{}` default: result = `{}` } _ = json.NewEncoder(c).Encode(herdr.Response{ID: r.ID, Result: json.RawMessage(result)}) }() } }() root := filepath.Join(filepath.Dir(repo), "worktrees") // The worker renders its own launch instruction from the coordinator's // reduced authority, so the fake API must serve it. api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/intent") { json.NewEncoder(w).Encode(domain.EffectiveIntent{ Task: domain.Task{ID: "task", Title: "test", Description: "do work"}, Decisions: []domain.HumanDecision{{ ID: "d1", TaskID: "task", Kind: domain.HumanDecisionCorrection, Subject: "strategy", Value: "no, use b", }}, }) return } w.WriteHeader(200) })) defer api.Close() w := &worker{api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, herdr: &herdr.Client{Path: ln.Addr().String()}, repo: repo, root: root, remote: "origin", harness: "opencode", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: filepath.Join(t.TempDir(), "state.json"), hard: .75} // New() is needed for its pane map; override the address for the fake. w.herdr = herdr.New(ln.Addr().String()) task := domain.Task{ID: "task", Source: "s", ExternalID: "x", Project: "p", Title: "test", Description: "do work"} if err := w.start(context.Background(), task, ""); err != nil { t.Fatal(err) } if _, ok := w.sessions[task.ID]; !ok { t.Fatal("session not persisted") } if _, err := os.Stat(filepath.Join(root, "task", "TASK.md")); err != nil { t.Fatalf("TASK.md: %v", err) } // The deployed worker path renders through agentctx, so a decision the // human recorded before this session existed reaches the agent. promptMu.Lock() launched := strings.Join(prompts, "\n") promptMu.Unlock() for _, want := range []string{"## Current human decisions", "no, use b", "Authority order"} { if !strings.Contains(launched, want) { t.Fatalf("worker launch instruction missing %q:\n%s", want, launched) } } } func TestFinalizeRunsGateCommitsPushesAndVerifiesRemote(t *testing.T) { remote := filepath.Join(t.TempDir(), "remote.git") if out, err := exec.Command("git", "init", "--bare", remote).CombinedOutput(); err != nil { t.Fatalf("remote: %v %s", err, out) } seed := t.TempDir() for _, a := range [][]string{{"init", seed}, {"-C", seed, "config", "user.email", "t@t"}, {"-C", seed, "config", "user.name", "t"}, {"-C", seed, "commit", "--allow-empty", "-m", "init"}, {"-C", seed, "remote", "add", "origin", remote}, {"-C", seed, "push", "-u", "origin", "HEAD:master"}} { if out, err := exec.Command("git", a...).CombinedOutput(); err != nil { t.Fatalf("git %v: %v %s", a, err, out) } } repo := filepath.Join(t.TempDir(), "repo") if out, err := exec.Command("git", "clone", remote, repo).CombinedOutput(); err != nil { t.Fatalf("clone: %v %s", err, out) } for _, a := range [][]string{{"-C", repo, "config", "user.email", "t@t"}, {"-C", repo, "config", "user.name", "t"}} { if out, err := exec.Command("git", a...).CombinedOutput(); err != nil { t.Fatalf("git %v: %v %s", a, err, out) } } root := filepath.Join(t.TempDir(), "worktrees") task := domain.Task{ID: "task", Source: "s", ExternalID: "delivery", Project: "p", QualityGate: "test -f result.txt"} wt, err := (orchestrator.GitWorktrees{Repo: repo, Root: root}).Create(context.Background(), task) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(wt, "result.txt"), []byte("delivered"), 0644); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(wt, ".orchestra"), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(wt, ".orchestra", "done"), nil, 0644); err != nil { t.Fatal(err) } w := &worker{harnessID: "worker", harness: "opencode", repo: repo, root: root, remote: "origin", tasks: map[string]domain.Task{task.ID: task}} evidence, err := w.finalize(context.Background(), task.ID, herdr.Session{PaneID: "pane", Worktree: wt, TaskFileSHA: taskHash(task)}) if err != nil { t.Fatal(err) } if evidence.ResultSHA == evidence.BaseSHA || evidence.Branch != "orchestra/task" || evidence.QualityGate != task.QualityGate { t.Fatalf("unexpected evidence: %+v", evidence) } out, err := exec.Command("git", "-C", repo, "ls-remote", "origin", "refs/heads/orchestra/task").CombinedOutput() if err != nil || !strings.HasPrefix(string(out), evidence.ResultSHA+"\t") { t.Fatalf("remote result=%q err=%v want %s", out, err, evidence.ResultSHA) } } func TestWorkerApprovalCommandIsRevisionBoundAndAcknowledged(t *testing.T) { ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer ln.Close() sent := make(chan string, 1) go func() { for { c, err := ln.Accept() if err != nil { return } go func() { defer c.Close() var request herdr.Request if json.NewDecoder(c).Decode(&request) != nil { return } result := `{}` switch request.Method { case "pane.read": result = `{"read":{"text":"Permission required\n$ git status\nProceed? [y/n]"}}` case "pane.send_text": var p struct { Text string `json:"text"` } _ = json.Unmarshal(mustJSON(request.Params), &p) sent <- p.Text } _ = json.NewEncoder(c).Encode(herdr.Response{ID: request.ID, Result: json.RawMessage(result)}) }() } }() resolved := make(chan map[string]string, 1) api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && r.URL.Path == "/v1/federation/commands": _ = json.NewEncoder(rw).Encode([]federation.Command{{ID: "c1", TaskID: "task", Kind: "grant_approval", PaneID: "pane", CaptureRevision: 7}}) case r.Method == http.MethodPost && r.URL.Path == "/v1/federation/workers/h/captures": _ = json.NewEncoder(rw).Encode(federation.Capture{TaskID: "task", PaneID: "pane", Revision: 7}) case r.Method == http.MethodPost && r.URL.Path == "/v1/federation/commands/c1": var body map[string]string _ = json.NewDecoder(r.Body).Decode(&body) resolved <- body rw.WriteHeader(http.StatusNoContent) default: t.Errorf("unexpected %s %s", r.Method, r.URL.Path) rw.WriteHeader(http.StatusNotFound) } })) defer api.Close() w := &worker{api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, herdr: herdr.New(ln.Addr().String()), harness: "opencode", sessions: map[string]herdr.Session{"task": {PaneID: "pane"}}} w.runCommands(context.Background()) select { case got := <-sent: if got != "y\n" { t.Fatalf("approval input=%q", got) } case <-time.After(time.Second): t.Fatal("worker did not send approval") } select { case got := <-resolved: if got["status"] != "acknowledged" { t.Fatalf("resolution=%v", got) } case <-time.After(time.Second): t.Fatal("worker did not resolve command") } } func TestApprovalResponseOpenCodeAllowOnce(t *testing.T) { text := "Permission required\nAllow once Allow always Reject\n⇆ select enter confirm" if got, ok := approvalResponse(text, "grant_approval"); !ok || !reflect.DeepEqual(got.Keys, []string{"ENTER"}) || got.Text != "" { t.Fatalf("grant response = %+v, %v", got, ok) } if got, ok := approvalResponse(text, "deny_approval"); ok || got.Text != "" || len(got.Keys) != 0 { t.Fatalf("deny response = %+v, %v; reject must not guess selector navigation", got, ok) } } func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b } type recordingBackend struct { status string calls []string prompts []string } func (b *recordingBackend) Kind() string { return "recording" } func (b *recordingBackend) Check(context.Context) error { return nil } func (b *recordingBackend) Worktree(_ context.Context, _, path, _ string) (string, error) { return path, nil } func (b *recordingBackend) StartAgent(_ context.Context, _, path, _, harness, _ string) (herdr.Session, error) { return herdr.Session{PaneID: "pane", Worktree: path, Harness: harness}, nil } func (b *recordingBackend) Prompt(_ context.Context, _, text string, _ time.Duration) error { b.prompts = append(b.prompts, text) return nil } func (b *recordingBackend) Kill(context.Context, herdr.Session) error { return nil } func (b *recordingBackend) AgentStatus(context.Context, herdr.Session) (string, error) { if b.status == "" { return "idle", nil } return b.status, nil } func (b *recordingBackend) PaneCapture(context.Context, herdr.Session, string) (string, error) { return "", nil } func (b *recordingBackend) SendText(_ context.Context, _ herdr.Session, text string) error { b.calls = append(b.calls, "text:"+text) return nil } func (b *recordingBackend) SendKeys(_ context.Context, _ herdr.Session, keys []string) error { b.calls = append(b.calls, "keys:"+strings.Join(keys, ",")) return nil } func (b *recordingBackend) ReleaseAgent(context.Context, herdr.Session, string) error { return nil } func TestClaudeContextHookRolloverSendsClearThenHandoff(t *testing.T) { worktree := t.TempDir() handoff := filepath.Join(worktree, "HANDOFF.md") if err := os.WriteFile(handoff, []byte("old handoff"), 0o644); err != nil { t.Fatal(err) } oldSHA, err := fileSHA256(handoff) if err != nil { t.Fatal(err) } if err := os.WriteFile(handoff, []byte("new durable handoff"), 0o644); err != nil { t.Fatal(err) } backend := &recordingBackend{} session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude", SessionFile: "exhausted.jsonl", ContextHandoffSHA: oldSHA} w := &worker{ backend: backend, harness: "claude", sessions: map[string]herdr.Session{"task": session}, tasks: map[string]domain.Task{}, leases: map[string]lease{}, releases: map[string]releaseTransaction{}, quarantined: map[string]bool{}, statePath: filepath.Join(t.TempDir(), "state.json"), } if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil { t.Fatal(err) } want := []string{"text:/clear", "keys:ENTER", "text:@HANDOFF.md", "keys:ENTER"} if !reflect.DeepEqual(backend.calls, want) { t.Fatalf("rollover calls=%v want %v", backend.calls, want) } got := w.sessions["task"] newSHA, _ := fileSHA256(handoff) if got.ContextHandoffSHA != newSHA || got.ContextResetSHA != "" || got.ContextResetPhase != "" || got.SessionFile != "" { t.Fatalf("rollover state was not finalized: %+v", got) } if err := w.advanceClaudeContextReset(context.Background(), "task", got); err != nil { t.Fatal(err) } if !reflect.DeepEqual(backend.calls, want) { t.Fatalf("unchanged handoff retriggered rollover: %v", backend.calls) } } func TestClaudeContextHookRolloverWaitsForIdle(t *testing.T) { worktree := t.TempDir() if err := os.WriteFile(filepath.Join(worktree, "HANDOFF.md"), []byte("handoff"), 0o644); err != nil { t.Fatal(err) } backend := &recordingBackend{status: "busy"} session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude"} w := &worker{backend: backend, harness: "claude", sessions: map[string]herdr.Session{"task": session}, statePath: filepath.Join(t.TempDir(), "state.json")} if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil { t.Fatal(err) } if len(backend.calls) != 0 { t.Fatalf("busy Claude received rollover input: %v", backend.calls) } } func TestHarnessSpecsFallsBackToSingleHarnessEnvironment(t *testing.T) { t.Setenv("ORCHESTRA_WORKER_HERDR_ID", "workpc-opencode") t.Setenv("ORCHESTRA_WORKER_HARNESS", "opencode") t.Setenv("ORCHESTRA_WORKER_TOKEN", "secret") t.Setenv("ORCHESTRA_WORKER_HERDR", "127.0.0.1:9247") specs := harnessSpecs() if len(specs) != 1 { t.Fatalf("want one legacy spec, got %d", len(specs)) } if specs[0].ID != "workpc-opencode" || specs[0].Harness != "opencode" || specs[0].Token != "secret" || specs[0].Herdr != "127.0.0.1:9247" { t.Fatalf("legacy environment was not carried through: %+v", specs[0]) } } func TestHarnessSpecsReadsMultipleHarnessesAndPerIdentityTokens(t *testing.T) { path := filepath.Join(t.TempDir(), "harnesses.json") if err := os.WriteFile(path, []byte(`[ {"id":"workpc-claude","harness":"claude","backend":"tmux","tmux_socket":"orchestra","command":"/usr/bin/true"}, {"id":"workpc-opencode","harness":"opencode","backend":"herdr","herdr":"127.0.0.1:9247","token":"inline"} ]`), 0o600); err != nil { t.Fatal(err) } t.Setenv("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE", path) t.Setenv("ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE", "from-env") specs := harnessSpecs() if len(specs) != 2 { t.Fatalf("want two specs, got %d", len(specs)) } if specs[0].Token != "from-env" { t.Fatalf("per-identity token env was not consulted: %q", specs[0].Token) } if specs[1].Token != "inline" { t.Fatalf("config-file token was overridden: %q", specs[1].Token) } // Distinct backends in one process is the point of the multi-harness form. if got := backendFor(specs[0]).Kind(); got != "tmux" { t.Fatalf("first harness backend = %q, want tmux", got) } if got := backendFor(specs[1]).Kind(); got != "herdr" { t.Fatalf("second harness backend = %q, want herdr", got) } } func TestTokenEnvKeySanitizesHarnessID(t *testing.T) { if got := tokenEnvKey("workpc-claude"); got != "ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE" { t.Fatalf("tokenEnvKey = %q", got) } if got := tokenEnvKey("box.1-opencode"); got != "ORCHESTRA_WORKER_TOKEN_BOX_1_OPENCODE" { t.Fatalf("tokenEnvKey = %q", got) } } func TestStatePathIsolatesEveryHarnessIdentity(t *testing.T) { claude := harnessSpec{ID: "workpc-claude"} opencode := harnessSpec{ID: "workpc-opencode"} // The single-harness form must keep the historical path so an upgraded // deployment recovers its sessions rather than orphaning their panes. if got := statePathFor(opencode, "/wt", "", true); got != "/wt/.orchestra-worker-state.json" { t.Fatalf("single-harness state path changed: %q", got) } a := statePathFor(claude, "/wt", "", false) b := statePathFor(opencode, "/wt", "", false) if a == b { t.Fatalf("two harnesses share one state file: %q", a) } if got := statePathFor(claude, "/wt", "/var/lib/orchestra-worker", false); got != "/var/lib/orchestra-worker/state-workpc-claude.json" { t.Fatalf("state dir ignored: %q", got) } explicit := harnessSpec{ID: "workpc-claude", State: "/srv/claude.json"} if got := statePathFor(explicit, "/wt", "/var/lib/orchestra-worker", false); got != "/srv/claude.json" { t.Fatalf("explicit per-entry state path ignored: %q", got) } } // boundaryAdapter reports a verified turn boundary without a live pane. type boundaryAdapter struct { at bool err error } func (b boundaryAdapter) Lease(context.Context, string, string) (herdr.Session, error) { return herdr.Session{}, nil } func (b boundaryAdapter) Release(context.Context, herdr.Session) (string, error) { return "", nil } func (b boundaryAdapter) Kill(context.Context, herdr.Session) error { return nil } func (b boundaryAdapter) Occupancy(herdr.Session) (float64, error) { return 0, nil } func (b boundaryAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) { return b.at, b.err } // The federated half of the authority model: at a verified boundary the worker // asks the coordinator, delivers the correction into its own pane once, and // records it so the next boundary stays quiet. func TestFederatedTurnDeliversCorrectionOnce(t *testing.T) { prompts := make(chan string, 4) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer ln.Close() go func() { for { c, e := ln.Accept() if e != nil { return } go func() { defer c.Close() var request herdr.Request if json.NewDecoder(c).Decode(&request) != nil { return } result := `{}` switch request.Method { case "pane.get": result = `{"pane":{"agent":"opencode","agent_status":"idle"}}` case "pane.read": result = `{"read":{"text":""}}` case "agent.prompt": if m, ok := request.Params.(map[string]any); ok { text, _ := m["text"].(string) prompts <- text } } _ = json.NewEncoder(c).Encode(herdr.Response{ID: request.ID, Result: json.RawMessage(result)}) }() } }() var turnCalls int var lastDelivered []string fail := false api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !strings.HasSuffix(r.URL.Path, "/federation/turn") { w.WriteHeader(200) return } if fail { http.Error(w, "coordinator down", 503) return } turnCalls++ var body struct { Delivered []string `json:"delivered_decisions"` } _ = json.NewDecoder(r.Body).Decode(&body) lastDelivered = body.Delivered out := federation.TurnDecision{Verdict: "continue"} if len(body.Delivered) == 0 { out.Decisions = []domain.HumanDecision{{ ID: "d1", Kind: domain.HumanDecisionCorrection, Subject: "strategy", Value: "no, use b", }} } json.NewEncoder(w).Encode(out) })) defer api.Close() w := &worker{ api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, herdr: herdr.New(ln.Addr().String()), harness: "opencode", sessions: map[string]herdr.Session{"task": {PaneID: "pane", Harness: "opencode"}}, leases: map[string]lease{"task": {Epoch: "e1", Version: 2}}, statePath: t.TempDir() + "/state.json", } // Not at a boundary: nothing is asked and nothing is delivered. w.federatedTurn(context.Background(), "task", boundaryAdapter{at: false}, "continue") if turnCalls != 0 { t.Fatal("asked the coordinator without a verified boundary") } w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue") select { case got := <-prompts: if !strings.Contains(got, "no, use b") || !strings.Contains(got, "outrank") { t.Fatalf("delivered prompt = %q", got) } default: t.Fatal("correction was not delivered to the pane") } if ids := w.sessions["task"].DeliveredDecisions; len(ids) != 1 || ids[0] != "d1" { t.Fatalf("delivered ids = %v", ids) } // Second boundary: the worker reports what it has shown, so nothing repeats. w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue") if len(lastDelivered) != 1 || lastDelivered[0] != "d1" { t.Fatalf("worker did not report delivered ids: %v", lastDelivered) } select { case got := <-prompts: t.Fatalf("correction delivered twice: %q", got) default: } // Coordinator unreachable: observable, and the session keeps running. fail = true w.lastError = "" w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue") if !strings.Contains(w.lastError, "federated turn") { t.Fatalf("transport failure not observable: %q", w.lastError) } } // A worker must act on the coordinator's verdict, not only on its decisions. // When the coordinator can no longer reconcile human input for this task, it // answers prepare_handoff, and the worker asks its own agent to hand off. The // release loop then takes over on the next tick, because the handoff report is // what it watches for. func TestFederatedTurnActsOnPrepareHandoffVerdict(t *testing.T) { api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v1/federation/turn" { rw.WriteHeader(http.StatusNotFound) return } json.NewEncoder(rw).Encode(federation.TurnDecision{Verdict: orchestrator.TurnPrepareHandoff}) })) defer api.Close() backend := &recordingBackend{} session := herdr.Session{PaneID: "pane", Worktree: t.TempDir(), Harness: "opencode"} w := &worker{ api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, backend: backend, harness: "opencode", sessions: map[string]herdr.Session{"task": session}, leases: map[string]lease{"task": {Epoch: "e1", Version: 2}}, tasks: map[string]domain.Task{}, statePath: filepath.Join(t.TempDir(), "state.json"), } a := herdr.CLIAdapter{Backend: backend, Harness: "opencode"} w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue) got := w.sessions["task"] if !got.HandoffRequested || got.HandoffReason != "reconcile_failure" { t.Fatalf("session did not record the requested handoff: %+v", got) } if len(backend.prompts) != 1 || !strings.Contains(backend.prompts[0], "reconcile_failure") { t.Fatalf("agent was not asked to hand off: %v", backend.prompts) } // Idempotent: a second boundary must not re-prompt a session that is // already preparing its handoff. w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue) if len(backend.prompts) != 1 { t.Fatalf("handoff re-requested: %v", backend.prompts) } }