orchestra/cmd/orchestra-worker/main_test.go

package main

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"orchestra/internal/domain"
	"orchestra/internal/federation"
	"orchestra/internal/herdr"
	"orchestra/internal/orchestrator"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"
)

func TestWorkerReregistersAfterCoordinatorForgetsIt(t *testing.T) {
	registered := false
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodPost || r.URL.Path != "/v1/federation/workers" {
			t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
		}
		registered = true
		w.WriteHeader(http.StatusCreated)
	}))
	defer s.Close()
	w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, registration: federation.Worker{ID: "h", Capacity: 1}}
	if !w.reRegisterAfterCoordinatorRestart(context.Background(), errors.New("federation: 401 Unauthorized: unknown worker")) {
		t.Fatal("worker did not identify a coordinator restart")
	}
	if !registered {
		t.Fatal("worker did not register")
	}
}

func TestWorkerRefusesCorruptDurableState(t *testing.T) {
	path := filepath.Join(t.TempDir(), "state.json")
	if err := os.WriteFile(path, []byte(`{"cursor":`), 0600); err != nil {
		t.Fatal(err)
	}
	w := &worker{statePath: path}
	if err := w.load(); err == nil {
		t.Fatal("corrupt worker state was accepted")
	}
}

func TestClassifyLaunchErrorPreservesUncertainLivePane(t *testing.T) {
	if got := classifyLaunchError(errors.New("prompt response lost"), true); got != "launch_uncertain" {
		t.Fatalf("live pane class=%q", got)
	}
	if got := classifyLaunchError(errors.New("pickup anchor mismatch"), false); got != "invalid_handoff" {
		t.Fatalf("bad handoff class=%q", got)
	}
	if got := classifyLaunchError(errors.New("temporary herdr outage"), false); got != "launch_transient" {
		t.Fatalf("transient class=%q", got)
	}
}

func TestInitialReplayDoesNotResurrectReleasedLease(t *testing.T) {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/v1/federation/events":
			_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"c","type":"TaskCreated","task_id":"t","version":1,"payload":{"source":"s","external_id":"x","project":"p"},"surface":"system"},{"seq":2,"id":"l","type":"TaskLeased","task_id":"t","version":2,"payload":{"harness_id":"h"},"surface":"system"},{"seq":3,"id":"r","type":"TaskReleased","task_id":"t","version":3,"payload":{"reason":"expired"},"surface":"system"}]}`))
		case "/v1/federation/events/ack":
			w.WriteHeader(http.StatusNoContent)
		default:
			t.Fatalf("unexpected %s", r.URL.Path)
		}
	}))
	defer s.Close()
	w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, harnessID: "h", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: t.TempDir() + "/state.json", hard: .75}
	if err := w.once(context.Background()); err != nil {
		t.Fatal(err)
	}
	if len(w.leases) != 0 || len(w.sessions) != 0 {
		t.Fatalf("replayed lease survived: leases=%v sessions=%v", w.leases, w.sessions)
	}
	if w.cursor != 3 {
		t.Fatalf("cursor=%d want 3", w.cursor)
	}
}

func TestWorkerHydratesLeaseMissingTaskCache(t *testing.T) {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/v1/federation/events":
			_, _ = w.Write([]byte(`{"cursor":2,"events":[{"seq":2,"id":"l","type":"TaskLeased","task_id":"t","version":2,"payload":{"harness_id":"h"},"surface":"system"}]}`))
		case "/v1/tasks":
			_, _ = w.Write([]byte(`[{"id":"t","source":"s","external_id":"x","project":"p","state":"leased"}]`))
		case "/v1/federation/events/ack":
			w.WriteHeader(http.StatusNoContent)
		default:
			t.Fatalf("unexpected %s", r.URL.Path)
		}
	}))
	defer s.Close()
	w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, harnessID: "h", cursor: 1, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{"t": {}}, leases: map[string]lease{}, statePath: t.TempDir() + "/state.json", hard: .75}
	if err := w.once(context.Background()); err != nil {
		t.Fatal(err)
	}
	if _, ok := w.tasks["t"]; !ok || w.cursor != 2 {
		t.Fatalf("task was not hydrated: tasks=%v cursor=%d", w.tasks, w.cursor)
	}
}

func TestWorkerReconcilesLeaseAfterEmptyEventReplay(t *testing.T) {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/v1/federation/events":
			_, _ = w.Write([]byte(`{"cursor":42,"events":[]}`))
		case "/v1/tasks":
			_, _ = w.Write([]byte(`[{"id":"t","source":"s","external_id":"x","project":"p","state":"leased","lease":{"harness_id":"h"}}]`))
		case "/v1/federation/events/ack":
			w.WriteHeader(http.StatusNoContent)
		default:
			t.Fatalf("unexpected %s", r.URL.Path)
		}
	}))
	defer s.Close()
	w := &worker{api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, harnessID: "h", cursor: 42, tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{"t": {}}, leases: map[string]lease{}, statePath: t.TempDir() + "/state.json", hard: .75}
	if err := w.once(context.Background()); err != nil {
		t.Fatal(err)
	}
	if _, ok := w.leases["t"]; !ok {
		t.Fatal("empty event replay did not recover current lease")
	}
}

func TestWorkerStartsRouterIssuedLeaseInLocalGitWorktree(t *testing.T) {
	remote := filepath.Join(t.TempDir(), "remote.git")
	if out, err := exec.Command("git", "init", "--bare", remote).CombinedOutput(); err != nil {
		t.Fatalf("remote: %v %s", err, out)
	}
	seed := t.TempDir()
	for _, a := range [][]string{{"init", seed}, {"-C", seed, "config", "user.email", "t@t"}, {"-C", seed, "config", "user.name", "t"}, {"-C", seed, "commit", "--allow-empty", "-m", "init"}, {"-C", seed, "remote", "add", "origin", remote}, {"-C", seed, "push", "-u", "origin", "HEAD:master"}} {
		if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
			t.Fatalf("git %v: %v %s", a, err, out)
		}
	}
	repo := filepath.Join(t.TempDir(), "repo")
	if out, err := exec.Command("git", "clone", remote, repo).CombinedOutput(); err != nil {
		t.Fatal(string(out))
	}
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	var promptMu sync.Mutex
	var prompts []string
	go func() {
		for {
			c, e := ln.Accept()
			if e != nil {
				return
			}
			go func() {
				defer c.Close()
				var r herdr.Request
				if json.NewDecoder(bufio.NewReader(c)).Decode(&r) != nil {
					return
				}
				var result string
				switch r.Method {
				case "worktree.create":
					result = `{"path":"` + filepath.Join(filepath.Dir(repo), "worktrees", "task") + `","root_pane":{"pane_id":"p1"}}`
				case "agent.start":
					result = `{}`
				case "pane.get":
					result = `{"pane":{"agent":"opencode","agent_status":"idle"}}`
				case "pane.read":
					result = `{"read":{"text":""}}`
				case "agent.prompt":
					if m, ok := r.Params.(map[string]any); ok {
						text, _ := m["text"].(string)
						promptMu.Lock()
						prompts = append(prompts, text)
						promptMu.Unlock()
					}
					result = `{}`
				default:
					result = `{}`
				}
				_ = json.NewEncoder(c).Encode(herdr.Response{ID: r.ID, Result: json.RawMessage(result)})
			}()
		}
	}()
	root := filepath.Join(filepath.Dir(repo), "worktrees")
	// The worker renders its own launch instruction from the coordinator's
	// reduced authority, so the fake API must serve it.
	api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if strings.HasSuffix(r.URL.Path, "/intent") {
			json.NewEncoder(w).Encode(domain.EffectiveIntent{
				Task: domain.Task{ID: "task", Title: "test", Description: "do work"},
				Decisions: []domain.HumanDecision{{
					ID: "d1", TaskID: "task", Kind: domain.HumanDecisionCorrection,
					Subject: "strategy", Value: "no, use b",
				}},
			})
			return
		}
		w.WriteHeader(200)
	}))
	defer api.Close()
	w := &worker{api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, herdr: &herdr.Client{Path: ln.Addr().String()}, repo: repo, root: root, remote: "origin", harness: "opencode", tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{}, statePath: filepath.Join(t.TempDir(), "state.json"), hard: .75}
	// New() is needed for its pane map; override the address for the fake.
	w.herdr = herdr.New(ln.Addr().String())
	task := domain.Task{ID: "task", Source: "s", ExternalID: "x", Project: "p", Title: "test", Description: "do work"}
	if err := w.start(context.Background(), task, ""); err != nil {
		t.Fatal(err)
	}
	if _, ok := w.sessions[task.ID]; !ok {
		t.Fatal("session not persisted")
	}
	if _, err := os.Stat(filepath.Join(root, "task", "TASK.md")); err != nil {
		t.Fatalf("TASK.md: %v", err)
	}
	// The deployed worker path renders through agentctx, so a decision the
	// human recorded before this session existed reaches the agent.
	promptMu.Lock()
	launched := strings.Join(prompts, "\n")
	promptMu.Unlock()
	for _, want := range []string{"## Current human decisions", "no, use b", "Authority order"} {
		if !strings.Contains(launched, want) {
			t.Fatalf("worker launch instruction missing %q:\n%s", want, launched)
		}
	}
}

func TestFinalizeRunsGateCommitsPushesAndVerifiesRemote(t *testing.T) {
	remote := filepath.Join(t.TempDir(), "remote.git")
	if out, err := exec.Command("git", "init", "--bare", remote).CombinedOutput(); err != nil {
		t.Fatalf("remote: %v %s", err, out)
	}
	seed := t.TempDir()
	for _, a := range [][]string{{"init", seed}, {"-C", seed, "config", "user.email", "t@t"}, {"-C", seed, "config", "user.name", "t"}, {"-C", seed, "commit", "--allow-empty", "-m", "init"}, {"-C", seed, "remote", "add", "origin", remote}, {"-C", seed, "push", "-u", "origin", "HEAD:master"}} {
		if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
			t.Fatalf("git %v: %v %s", a, err, out)
		}
	}
	repo := filepath.Join(t.TempDir(), "repo")
	if out, err := exec.Command("git", "clone", remote, repo).CombinedOutput(); err != nil {
		t.Fatalf("clone: %v %s", err, out)
	}
	for _, a := range [][]string{{"-C", repo, "config", "user.email", "t@t"}, {"-C", repo, "config", "user.name", "t"}} {
		if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
			t.Fatalf("git %v: %v %s", a, err, out)
		}
	}
	root := filepath.Join(t.TempDir(), "worktrees")
	task := domain.Task{ID: "task", Source: "s", ExternalID: "delivery", Project: "p", QualityGate: "test -f result.txt"}
	wt, err := (orchestrator.GitWorktrees{Repo: repo, Root: root}).Create(context.Background(), task)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(wt, "result.txt"), []byte("delivered"), 0644); err != nil {
		t.Fatal(err)
	}
	if err := os.MkdirAll(filepath.Join(wt, ".orchestra"), 0755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(wt, ".orchestra", "done"), nil, 0644); err != nil {
		t.Fatal(err)
	}
	w := &worker{harnessID: "worker", harness: "opencode", repo: repo, root: root, remote: "origin", tasks: map[string]domain.Task{task.ID: task}}
	evidence, err := w.finalize(context.Background(), task.ID, herdr.Session{PaneID: "pane", Worktree: wt, TaskFileSHA: taskHash(task)})
	if err != nil {
		t.Fatal(err)
	}
	if evidence.ResultSHA == evidence.BaseSHA || evidence.Branch != "orchestra/task" || evidence.QualityGate != task.QualityGate {
		t.Fatalf("unexpected evidence: %+v", evidence)
	}
	out, err := exec.Command("git", "-C", repo, "ls-remote", "origin", "refs/heads/orchestra/task").CombinedOutput()
	if err != nil || !strings.HasPrefix(string(out), evidence.ResultSHA+"\t") {
		t.Fatalf("remote result=%q err=%v want %s", out, err, evidence.ResultSHA)
	}
}

func TestWorkerApprovalCommandIsRevisionBoundAndAcknowledged(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	sent := make(chan string, 1)
	go func() {
		for {
			c, err := ln.Accept()
			if err != nil {
				return
			}
			go func() {
				defer c.Close()
				var request herdr.Request
				if json.NewDecoder(c).Decode(&request) != nil {
					return
				}
				result := `{}`
				switch request.Method {
				case "pane.read":
					result = `{"read":{"text":"Permission required\n$ git status\nProceed? [y/n]"}}`
				case "pane.send_text":
					var p struct {
						Text string `json:"text"`
					}
					_ = json.Unmarshal(mustJSON(request.Params), &p)
					sent <- p.Text
				}
				_ = json.NewEncoder(c).Encode(herdr.Response{ID: request.ID, Result: json.RawMessage(result)})
			}()
		}
	}()
	resolved := make(chan map[string]string, 1)
	api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		switch {
		case r.Method == http.MethodGet && r.URL.Path == "/v1/federation/commands":
			_ = json.NewEncoder(rw).Encode([]federation.Command{{ID: "c1", TaskID: "task", Kind: "grant_approval", PaneID: "pane", CaptureRevision: 7}})
		case r.Method == http.MethodPost && r.URL.Path == "/v1/federation/workers/h/captures":
			_ = json.NewEncoder(rw).Encode(federation.Capture{TaskID: "task", PaneID: "pane", Revision: 7})
		case r.Method == http.MethodPost && r.URL.Path == "/v1/federation/commands/c1":
			var body map[string]string
			_ = json.NewDecoder(r.Body).Decode(&body)
			resolved <- body
			rw.WriteHeader(http.StatusNoContent)
		default:
			t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
			rw.WriteHeader(http.StatusNotFound)
		}
	}))
	defer api.Close()
	w := &worker{api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, herdr: herdr.New(ln.Addr().String()), harness: "opencode", sessions: map[string]herdr.Session{"task": {PaneID: "pane"}}}
	w.runCommands(context.Background())
	select {
	case got := <-sent:
		if got != "y\n" {
			t.Fatalf("approval input=%q", got)
		}
	case <-time.After(time.Second):
		t.Fatal("worker did not send approval")
	}
	select {
	case got := <-resolved:
		if got["status"] != "acknowledged" {
			t.Fatalf("resolution=%v", got)
		}
	case <-time.After(time.Second):
		t.Fatal("worker did not resolve command")
	}
}

func TestApprovalResponseOpenCodeAllowOnce(t *testing.T) {
	text := "Permission required\nAllow once  Allow always  Reject\n⇆ select  enter confirm"
	if got, ok := approvalResponse(text, "grant_approval"); !ok || !reflect.DeepEqual(got.Keys, []string{"ENTER"}) || got.Text != "" {
		t.Fatalf("grant response = %+v, %v", got, ok)
	}
	if got, ok := approvalResponse(text, "deny_approval"); ok || got.Text != "" || len(got.Keys) != 0 {
		t.Fatalf("deny response = %+v, %v; reject must not guess selector navigation", got, ok)
	}
}

func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }

type recordingBackend struct {
	status   string
	capture  string
	progress string
	calls    []string
	prompts  []string
}

// PaneProgress makes this stub the kind of backend that can separate harness
// output from input, which is what the worker must prefer over a raw capture.
func (b *recordingBackend) PaneProgress(context.Context, herdr.Session) (string, error) {
	return b.progress, nil
}

func (b *recordingBackend) Kind() string                { return "recording" }
func (b *recordingBackend) Check(context.Context) error { return nil }
func (b *recordingBackend) Worktree(_ context.Context, _, path, _ string) (string, error) {
	return path, nil
}
func (b *recordingBackend) StartAgent(_ context.Context, _, path, _, harness, _ string) (herdr.Session, error) {
	return herdr.Session{PaneID: "pane", Worktree: path, Harness: harness}, nil
}
func (b *recordingBackend) Prompt(_ context.Context, _, text string, _ time.Duration) error {
	b.prompts = append(b.prompts, text)
	return nil
}
func (b *recordingBackend) Kill(context.Context, herdr.Session) error { return nil }
func (b *recordingBackend) AgentStatus(context.Context, herdr.Session) (string, error) {
	if b.status == "" {
		return "idle", nil
	}
	return b.status, nil
}
func (b *recordingBackend) PaneCapture(context.Context, herdr.Session, string) (string, error) {
	return b.capture, nil
}

func TestRenewLeasesRequiresProgress(t *testing.T) {
	renewals := 0
	api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		renewals++
		rw.Write([]byte(`{}`))
	}))
	defer api.Close()
	backend := &recordingBackend{status: "idle", progress: "same screen"}
	w := &worker{
		api:         federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
		backend:     backend,
		harness:     "claude",
		sessions:    map[string]herdr.Session{"task": {PaneID: "pane"}},
		leases:      map[string]lease{"task": {Epoch: "e", Version: 1, Until: time.Now(), ProgressSHA: domain.Hash([]byte("same screen"))}},
		quarantined: map[string]bool{},
		statePath:   filepath.Join(t.TempDir(), "state.json"),
	}
	w.renewLeases(context.Background())
	if renewals != 0 {
		t.Fatalf("idle pane with an unchanged capture renewed its lease %d times", renewals)
	}
	backend.status = "busy"
	w.renewLeases(context.Background())
	if renewals != 1 {
		t.Fatalf("busy agent renewals=%d, want 1", renewals)
	}
	backend.status, backend.progress = "idle", "new output"
	l := w.leases["task"]
	l.Until = time.Now()
	w.leases["task"] = l
	w.renewLeases(context.Background())
	if renewals != 2 {
		t.Fatalf("changed capture renewals=%d, want 2", renewals)
	}
}
func (b *recordingBackend) SendText(_ context.Context, _ herdr.Session, text string) error {
	b.calls = append(b.calls, "text:"+text)
	return nil
}
func (b *recordingBackend) SendKeys(_ context.Context, _ herdr.Session, keys []string) error {
	b.calls = append(b.calls, "keys:"+strings.Join(keys, ","))
	return nil
}
func (b *recordingBackend) ReleaseAgent(context.Context, herdr.Session, string) error { return nil }

func TestClaudeContextHookRolloverSendsClearThenHandoff(t *testing.T) {
	worktree := t.TempDir()
	handoff := filepath.Join(worktree, "HANDOFF.md")
	if err := os.WriteFile(handoff, []byte("old handoff"), 0o644); err != nil {
		t.Fatal(err)
	}
	oldSHA, err := fileSHA256(handoff)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(handoff, []byte("new durable handoff"), 0o644); err != nil {
		t.Fatal(err)
	}
	backend := &recordingBackend{}
	session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude", SessionFile: "exhausted.jsonl", ContextHandoffSHA: oldSHA}
	w := &worker{
		backend:     backend,
		harness:     "claude",
		sessions:    map[string]herdr.Session{"task": session},
		tasks:       map[string]domain.Task{},
		leases:      map[string]lease{},
		releases:    map[string]releaseTransaction{},
		quarantined: map[string]bool{},
		statePath:   filepath.Join(t.TempDir(), "state.json"),
	}
	if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil {
		t.Fatal(err)
	}
	want := []string{"text:/clear", "keys:ENTER", "text:@HANDOFF.md", "keys:ENTER"}
	if !reflect.DeepEqual(backend.calls, want) {
		t.Fatalf("rollover calls=%v want %v", backend.calls, want)
	}
	got := w.sessions["task"]
	newSHA, _ := fileSHA256(handoff)
	if got.ContextHandoffSHA != newSHA || got.ContextResetSHA != "" || got.ContextResetPhase != "" || got.SessionFile != "" {
		t.Fatalf("rollover state was not finalized: %+v", got)
	}
	if err := w.advanceClaudeContextReset(context.Background(), "task", got); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(backend.calls, want) {
		t.Fatalf("unchanged handoff retriggered rollover: %v", backend.calls)
	}
}

func TestClaudeContextHookRolloverWaitsForIdle(t *testing.T) {
	worktree := t.TempDir()
	if err := os.WriteFile(filepath.Join(worktree, "HANDOFF.md"), []byte("handoff"), 0o644); err != nil {
		t.Fatal(err)
	}
	backend := &recordingBackend{status: "busy"}
	session := herdr.Session{PaneID: "pane", Worktree: worktree, Harness: "claude"}
	w := &worker{backend: backend, harness: "claude", sessions: map[string]herdr.Session{"task": session}, statePath: filepath.Join(t.TempDir(), "state.json")}
	if err := w.advanceClaudeContextReset(context.Background(), "task", session); err != nil {
		t.Fatal(err)
	}
	if len(backend.calls) != 0 {
		t.Fatalf("busy Claude received rollover input: %v", backend.calls)
	}
}

func TestHarnessSpecsFallsBackToSingleHarnessEnvironment(t *testing.T) {
	t.Setenv("ORCHESTRA_WORKER_HERDR_ID", "workpc-opencode")
	t.Setenv("ORCHESTRA_WORKER_HARNESS", "opencode")
	t.Setenv("ORCHESTRA_WORKER_TOKEN", "secret")
	t.Setenv("ORCHESTRA_WORKER_HERDR", "127.0.0.1:9247")
	specs := harnessSpecs()
	if len(specs) != 1 {
		t.Fatalf("want one legacy spec, got %d", len(specs))
	}
	if specs[0].ID != "workpc-opencode" || specs[0].Harness != "opencode" || specs[0].Token != "secret" || specs[0].Herdr != "127.0.0.1:9247" {
		t.Fatalf("legacy environment was not carried through: %+v", specs[0])
	}
}

func TestHarnessSpecsReadsMultipleHarnessesAndPerIdentityTokens(t *testing.T) {
	path := filepath.Join(t.TempDir(), "harnesses.json")
	if err := os.WriteFile(path, []byte(`[
		{"id":"workpc-claude","harness":"claude","backend":"tmux","tmux_socket":"orchestra","command":"/usr/bin/true"},
		{"id":"workpc-opencode","harness":"opencode","backend":"herdr","herdr":"127.0.0.1:9247","token":"inline"}
	]`), 0o600); err != nil {
		t.Fatal(err)
	}
	t.Setenv("ORCHESTRA_WORKER_HARNESS_CONFIG_FILE", path)
	t.Setenv("ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE", "from-env")
	specs := harnessSpecs()
	if len(specs) != 2 {
		t.Fatalf("want two specs, got %d", len(specs))
	}
	if specs[0].Token != "from-env" {
		t.Fatalf("per-identity token env was not consulted: %q", specs[0].Token)
	}
	if specs[1].Token != "inline" {
		t.Fatalf("config-file token was overridden: %q", specs[1].Token)
	}
	// Distinct backends in one process is the point of the multi-harness form.
	if got := backendFor(specs[0]).Kind(); got != "tmux" {
		t.Fatalf("first harness backend = %q, want tmux", got)
	}
	if got := backendFor(specs[1]).Kind(); got != "herdr" {
		t.Fatalf("second harness backend = %q, want herdr", got)
	}
}

func TestTokenEnvKeySanitizesHarnessID(t *testing.T) {
	if got := tokenEnvKey("workpc-claude"); got != "ORCHESTRA_WORKER_TOKEN_WORKPC_CLAUDE" {
		t.Fatalf("tokenEnvKey = %q", got)
	}
	if got := tokenEnvKey("box.1-opencode"); got != "ORCHESTRA_WORKER_TOKEN_BOX_1_OPENCODE" {
		t.Fatalf("tokenEnvKey = %q", got)
	}
}

func TestStatePathIsolatesEveryHarnessIdentity(t *testing.T) {
	claude := harnessSpec{ID: "workpc-claude"}
	opencode := harnessSpec{ID: "workpc-opencode"}
	// The single-harness form must keep the historical path so an upgraded
	// deployment recovers its sessions rather than orphaning their panes.
	if got := statePathFor(opencode, "/wt", "", true); got != "/wt/.orchestra-worker-state.json" {
		t.Fatalf("single-harness state path changed: %q", got)
	}
	a := statePathFor(claude, "/wt", "", false)
	b := statePathFor(opencode, "/wt", "", false)
	if a == b {
		t.Fatalf("two harnesses share one state file: %q", a)
	}
	if got := statePathFor(claude, "/wt", "/var/lib/orchestra-worker", false); got != "/var/lib/orchestra-worker/state-workpc-claude.json" {
		t.Fatalf("state dir ignored: %q", got)
	}
	explicit := harnessSpec{ID: "workpc-claude", State: "/srv/claude.json"}
	if got := statePathFor(explicit, "/wt", "/var/lib/orchestra-worker", false); got != "/srv/claude.json" {
		t.Fatalf("explicit per-entry state path ignored: %q", got)
	}
}

// boundaryAdapter reports a verified turn boundary without a live pane.
type boundaryAdapter struct {
	at  bool
	err error
}

func (b boundaryAdapter) Lease(context.Context, string, string) (herdr.Session, error) {
	return herdr.Session{}, nil
}
func (b boundaryAdapter) Release(context.Context, herdr.Session) (string, error) { return "", nil }
func (b boundaryAdapter) Kill(context.Context, herdr.Session) error              { return nil }
func (b boundaryAdapter) Occupancy(herdr.Session) (float64, error)               { return 0, nil }
func (b boundaryAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
	return b.at, b.err
}

// The federated half of the authority model: at a verified boundary the worker
// asks the coordinator, delivers the correction into its own pane once, and
// records it so the next boundary stays quiet.
func TestFederatedTurnDeliversCorrectionOnce(t *testing.T) {
	prompts := make(chan string, 4)
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	go func() {
		for {
			c, e := ln.Accept()
			if e != nil {
				return
			}
			go func() {
				defer c.Close()
				var request herdr.Request
				if json.NewDecoder(c).Decode(&request) != nil {
					return
				}
				result := `{}`
				switch request.Method {
				case "pane.get":
					result = `{"pane":{"agent":"opencode","agent_status":"idle"}}`
				case "pane.read":
					result = `{"read":{"text":""}}`
				case "agent.prompt":
					if m, ok := request.Params.(map[string]any); ok {
						text, _ := m["text"].(string)
						prompts <- text
					}
				}
				_ = json.NewEncoder(c).Encode(herdr.Response{ID: request.ID, Result: json.RawMessage(result)})
			}()
		}
	}()

	var turnCalls int
	var lastDelivered []string
	fail := false
	api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if !strings.HasSuffix(r.URL.Path, "/federation/turn") {
			w.WriteHeader(200)
			return
		}
		if fail {
			http.Error(w, "coordinator down", 503)
			return
		}
		turnCalls++
		var body struct {
			Delivered []string `json:"delivered_decisions"`
		}
		_ = json.NewDecoder(r.Body).Decode(&body)
		lastDelivered = body.Delivered
		out := federation.TurnDecision{Verdict: "continue"}
		if len(body.Delivered) == 0 {
			out.Decisions = []domain.HumanDecision{{
				ID: "d1", Kind: domain.HumanDecisionCorrection, Subject: "strategy", Value: "no, use b",
			}}
		}
		json.NewEncoder(w).Encode(out)
	}))
	defer api.Close()

	w := &worker{
		api:   federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
		herdr: herdr.New(ln.Addr().String()), harness: "opencode",
		sessions:  map[string]herdr.Session{"task": {PaneID: "pane", Harness: "opencode"}},
		leases:    map[string]lease{"task": {Epoch: "e1", Version: 2}},
		statePath: t.TempDir() + "/state.json",
	}

	// Not at a boundary: nothing is asked and nothing is delivered.
	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: false}, "continue")
	if turnCalls != 0 {
		t.Fatal("asked the coordinator without a verified boundary")
	}

	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
	select {
	case got := <-prompts:
		if !strings.Contains(got, "no, use b") || !strings.Contains(got, "outrank") {
			t.Fatalf("delivered prompt = %q", got)
		}
	default:
		t.Fatal("correction was not delivered to the pane")
	}
	if ids := w.sessions["task"].DeliveredDecisions; len(ids) != 1 || ids[0] != "d1" {
		t.Fatalf("delivered ids = %v", ids)
	}

	// Second boundary: the worker reports what it has shown, so nothing repeats.
	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
	if len(lastDelivered) != 1 || lastDelivered[0] != "d1" {
		t.Fatalf("worker did not report delivered ids: %v", lastDelivered)
	}
	select {
	case got := <-prompts:
		t.Fatalf("correction delivered twice: %q", got)
	default:
	}

	// Coordinator unreachable: observable, and the session keeps running.
	fail = true
	w.lastError = ""
	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
	if !strings.Contains(w.lastError, "federated turn") {
		t.Fatalf("transport failure not observable: %q", w.lastError)
	}
}

// A worker must act on the coordinator's verdict, not only on its decisions.
// When the coordinator can no longer reconcile human input for this task, it
// answers prepare_handoff, and the worker asks its own agent to hand off. The
// release loop then takes over on the next tick, because the handoff report is
// what it watches for.
func TestFederatedTurnActsOnPrepareHandoffVerdict(t *testing.T) {
	api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/v1/federation/turn" {
			rw.WriteHeader(http.StatusNotFound)
			return
		}
		json.NewEncoder(rw).Encode(federation.TurnDecision{Verdict: orchestrator.TurnPrepareHandoff})
	}))
	defer api.Close()

	backend := &recordingBackend{}
	session := herdr.Session{PaneID: "pane", Worktree: t.TempDir(), Harness: "opencode"}
	w := &worker{
		api:       federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
		backend:   backend,
		harness:   "opencode",
		sessions:  map[string]herdr.Session{"task": session},
		leases:    map[string]lease{"task": {Epoch: "e1", Version: 2}},
		tasks:     map[string]domain.Task{},
		statePath: filepath.Join(t.TempDir(), "state.json"),
	}
	a := herdr.CLIAdapter{Backend: backend, Harness: "opencode"}
	w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue)

	got := w.sessions["task"]
	if !got.HandoffRequested || got.HandoffReason != "reconcile_failure" {
		t.Fatalf("session did not record the requested handoff: %+v", got)
	}
	if len(backend.prompts) != 1 || !strings.Contains(backend.prompts[0], "reconcile_failure") {
		t.Fatalf("agent was not asked to hand off: %v", backend.prompts)
	}

	// Idempotent: a second boundary must not re-prompt a session that is
	// already preparing its handoff.
	w.federatedTurn(context.Background(), "task", a, orchestrator.TurnContinue)
	if len(backend.prompts) != 1 {
		t.Fatalf("handoff re-requested: %v", backend.prompts)
	}
}

// launchBackend is a recordingBackend that also declares a launch transport
// and a confirmation outcome, which is the seam F15 added.
type launchBackend struct {
	recordingBackend
	transport herdr.LaunchTransport
	confirmed bool
	killed    int
}

func (b *launchBackend) LaunchTransport(string) herdr.LaunchTransport { return b.transport }
func (b *launchBackend) ConfirmInput(context.Context, herdr.Session, string) (string, error) {
	if !b.confirmed {
		return "", fmt.Errorf("%w: editor still holds the prompt", herdr.ErrPromptNotSubmitted)
	}
	return "input editor cleared, agent busy", nil
}
func (b *launchBackend) Kill(context.Context, herdr.Session) error { b.killed++; return nil }

// startFixture builds the git remote, local clone and coordinator stub that
// worker.start needs, and returns a worker wired to the given backend.
func startFixture(t *testing.T, backend herdr.Backend, harness string) (*worker, domain.Task, string) {
	t.Helper()
	remote := filepath.Join(t.TempDir(), "remote.git")
	if out, err := exec.Command("git", "init", "--bare", remote).CombinedOutput(); err != nil {
		t.Fatalf("remote: %v %s", err, out)
	}
	seed := t.TempDir()
	for _, a := range [][]string{{"init", seed}, {"-C", seed, "config", "user.email", "t@t"}, {"-C", seed, "config", "user.name", "t"}, {"-C", seed, "commit", "--allow-empty", "-m", "init"}, {"-C", seed, "remote", "add", "origin", remote}, {"-C", seed, "push", "-u", "origin", "HEAD:master"}} {
		if out, err := exec.Command("git", a...).CombinedOutput(); err != nil {
			t.Fatalf("git %v: %v %s", a, err, out)
		}
	}
	repo := filepath.Join(t.TempDir(), "repo")
	if out, err := exec.Command("git", "clone", remote, repo).CombinedOutput(); err != nil {
		t.Fatal(string(out))
	}
	api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if strings.HasSuffix(r.URL.Path, "/intent") {
			json.NewEncoder(w).Encode(domain.EffectiveIntent{Task: domain.Task{ID: "task", Title: "test", Description: "do work"}})
			return
		}
		w.WriteHeader(200)
	}))
	t.Cleanup(api.Close)
	root := filepath.Join(filepath.Dir(repo), "worktrees")
	w := &worker{
		api: federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"}, harnessID: "h",
		backend: backend, repo: repo, root: root, remote: "origin", harness: harness,
		tasks: map[string]domain.Task{}, sessions: map[string]herdr.Session{}, leases: map[string]lease{},
		releases: map[string]releaseTransaction{}, quarantined: map[string]bool{},
		statePath: filepath.Join(t.TempDir(), "state.json"), hard: .75,
	}
	return w, domain.Task{ID: "task", Source: "s", ExternalID: "x", Project: "p", Title: "test", Description: "do work"}, root
}

// The claude/tmux launch submits one line, while the agent still receives the
// exact bytes agentctx rendered, through the file the line points at.
func TestClaudeLaunchSubmitsAFileReferenceAndConfirmsIt(t *testing.T) {
	backend := &launchBackend{transport: herdr.LaunchFileRef, confirmed: true}
	w, task, root := startFixture(t, backend, "claude")
	if err := w.start(context.Background(), task, ""); err != nil {
		t.Fatal(err)
	}
	if len(backend.prompts) != 1 || backend.prompts[0] != herdr.LaunchReference {
		t.Fatalf("submitted %q, want the one-line launch reference", backend.prompts)
	}
	b, err := os.ReadFile(filepath.Join(root, "task", herdr.LaunchContextFile))
	if err != nil {
		t.Fatalf("launch context: %v", err)
	}
	for _, want := range []string{"Authority order", "## Goal", "do work"} {
		if !strings.Contains(string(b), want) {
			t.Fatalf("launch context missing %q", want)
		}
	}
	if _, ok := w.sessions[task.ID]; !ok {
		t.Fatal("confirmed launch did not keep its session")
	}
}

// Prompt returning nil is not an acknowledgement. An unconfirmed launch must
// reclaim the pane, drop the session so the retry can start clean, and
// classify as prompt_not_submitted so the lease is released rather than held.
func TestUnconfirmedLaunchFailsAndReclaimsThePane(t *testing.T) {
	backend := &launchBackend{transport: herdr.LaunchFileRef, confirmed: false}
	w, task, _ := startFixture(t, backend, "claude")
	err := w.start(context.Background(), task, "")
	if !errors.Is(err, herdr.ErrPromptNotSubmitted) {
		t.Fatalf("start err=%v, want ErrPromptNotSubmitted", err)
	}
	if _, ok := w.sessions[task.ID]; ok {
		t.Fatal("a launch that never happened must not leave a session behind")
	}
	if backend.killed != 1 {
		t.Fatalf("killed %d panes, want 1", backend.killed)
	}
	if got := classifyLaunchError(err, false); got != "prompt_not_submitted" {
		t.Fatalf("class=%q, want prompt_not_submitted", got)
	}
	// Even with a session still recorded, positive evidence outranks the
	// uncertain classification that would otherwise hold the lease forever.
	if got := classifyLaunchError(err, true); got != "prompt_not_submitted" {
		t.Fatalf("class with session=%q, want prompt_not_submitted", got)
	}
}

// A backend that cannot confirm keeps the inline launch it was verified on.
func TestInlineTransportSubmitsTheWholeInstruction(t *testing.T) {
	backend := &launchBackend{transport: herdr.LaunchInline, confirmed: true}
	w, task, _ := startFixture(t, backend, "opencode")
	if err := w.start(context.Background(), task, ""); err != nil {
		t.Fatal(err)
	}
	if len(backend.prompts) != 1 || !strings.Contains(backend.prompts[0], "Authority order") {
		t.Fatalf("submitted %q, want the whole instruction", backend.prompts)
	}
}

// Found live during burn-in run 3: reconcileLeases rebuilt each lease from the
// coordinator's view every tick, which wiped ProgressSHA and made the renewal
// progress check renew unconditionally on its no-baseline branch. The fix in
// isolation is worthless if the live call path resets its input.
func TestReconcileLeasesKeepsWorkerLocalObservations(t *testing.T) {
	until := time.Now().Add(30 * time.Minute)
	epoch := "epoch-1"
	api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		rw.Write(mustJSON([]domain.Task{{
			ID: "task", State: domain.StateLeased, Version: 12,
			Lease: &domain.Lease{HarnessID: "h", Epoch: epoch, Until: until},
		}}))
	}))
	defer api.Close()
	w := &worker{
		api:       federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
		harnessID: "h",
		tasks:     map[string]domain.Task{},
		leases:    map[string]lease{"task": {Epoch: "epoch-1", Version: 11, ProgressSHA: "sha-1", UsageBaseline: 0.25, PickupAcknowledged: true}},
		statePath: filepath.Join(t.TempDir(), "state.json"),
	}
	if err := w.reconcileLeases(context.Background()); err != nil {
		t.Fatal(err)
	}
	got := w.leases["task"]
	if got.ProgressSHA != "sha-1" || got.UsageBaseline != 0.25 || !got.PickupAcknowledged {
		t.Fatalf("reconcile dropped worker-local observations: %+v", got)
	}
	if got.Version != 12 {
		t.Fatalf("version=%d, want the coordinator's 12", got.Version)
	}

	// A different epoch is a different lease. Nothing observed under the old
	// one may carry into it.
	epoch = "epoch-2"
	if err := w.reconcileLeases(context.Background()); err != nil {
		t.Fatal(err)
	}
	got = w.leases["task"]
	if got.ProgressSHA != "" || got.UsageBaseline != 0 || got.PickupAcknowledged {
		t.Fatalf("observations survived a new epoch: %+v", got)
	}
}

// The live shape of the run-3 stall: someone typed into the pane, the capture
// changed, and nothing the agent did changed at all. Renewal must refuse.
func TestRenewLeasesIgnoresPaneInput(t *testing.T) {
	renewals := 0
	api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		renewals++
		rw.Write([]byte(`{}`))
	}))
	defer api.Close()
	backend := &recordingBackend{status: "idle", capture: "screen\n\u276f ", progress: "screen"}
	w := &worker{
		api:         federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
		backend:     backend,
		harness:     "claude",
		sessions:    map[string]herdr.Session{"task": {PaneID: "pane"}},
		leases:      map[string]lease{"task": {Epoch: "e", Version: 1, Until: time.Now(), ProgressSHA: domain.Hash([]byte("screen"))}},
		quarantined: map[string]bool{},
		statePath:   filepath.Join(t.TempDir(), "state.json"),
	}
	// Only the input line changes. The agent has done nothing.
	backend.capture = "screen\n\u276f go ahead and implement it"
	w.renewLeases(context.Background())
	if renewals != 0 {
		t.Fatalf("pane input renewed the lease %d times", renewals)
	}
	// Harness output moves the digest, and only then does the lease renew.
	backend.progress = "screen\nedited the script"
	w.renewLeases(context.Background())
	if renewals != 1 {
		t.Fatalf("renewals=%d, want 1 after real progress", renewals)
	}
}

// TestBlockedTaskWithUnpushedReleaseFreesTheSessionSlot guards F30. A release
// transaction that never reached anchor_pushed has no artifact for a successor
// to pick up, so keeping its session mapping protects nothing. Live, one stuck
// at "prepared" pinned workpc-claude's only capacity slot: health() kept
// reporting ActiveTask and the harness never leased again, with no log line.
func TestBlockedTaskWithUnpushedReleaseFreesTheSessionSlot(t *testing.T) {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/v1/federation/events":
			_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"b","type":"TaskBlocked","task_id":"t","version":2,"payload":{"blocker":"parked"},"surface":"web"}]}`))
		case "/v1/federation/events/ack":
			w.WriteHeader(http.StatusNoContent)
		default:
			// A worker with a live backend also polls captures and controls.
			// Neither is what this test asserts on.
			w.WriteHeader(http.StatusNoContent)
		}
	}))
	defer s.Close()
	w := &worker{
		api:         federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
		harnessID:   "h",
		backend:     deadTmuxBackend(t),
		tasks:       map[string]domain.Task{},
		sessions:    map[string]herdr.Session{"t": {PaneID: "pane"}},
		leases:      map[string]lease{"t": {Epoch: "e", Version: 1}},
		releases:    map[string]releaseTransaction{"t": {ID: "tx", Phase: "prepared"}},
		quarantined: map[string]bool{},
		statePath:   t.TempDir() + "/state.json",
		hard:        .75,
	}
	if err := w.once(context.Background()); err != nil {
		t.Fatal(err)
	}
	if len(w.sessions) != 0 {
		t.Fatalf("session still pinned: %v", w.sessions)
	}
	if len(w.releases) != 0 {
		t.Fatalf("unrecoverable transaction retained: %v", w.releases)
	}
	if got := w.health(context.Background()).ActiveTask; got != "" {
		t.Fatalf("ActiveTask=%q, worker still advertises itself as busy", got)
	}
}

// The opposite branch must not regress: once an anchor exists, a successor can
// still pick it up, so the mapping stays until TaskPickupValidated.
func TestBlockedTaskWithPushedAnchorKeepsItsSession(t *testing.T) {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch r.URL.Path {
		case "/v1/federation/events":
			_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"b","type":"TaskBlocked","task_id":"t","version":2,"payload":{"blocker":"parked"},"surface":"web"}]}`))
		case "/v1/federation/events/ack":
			w.WriteHeader(http.StatusNoContent)
		default:
			t.Fatalf("unexpected %s", r.URL.Path)
		}
	}))
	defer s.Close()
	w := &worker{
		api:         federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
		harnessID:   "h",
		tasks:       map[string]domain.Task{},
		sessions:    map[string]herdr.Session{"t": {PaneID: "pane"}},
		leases:      map[string]lease{"t": {Epoch: "e", Version: 1}},
		releases:    map[string]releaseTransaction{"t": {ID: "tx", Phase: "anchor_pushed", Ref: "sha256:abc"}},
		quarantined: map[string]bool{},
		statePath:   t.TempDir() + "/state.json",
		hard:        .75,
	}
	if err := w.once(context.Background()); err != nil {
		t.Fatal(err)
	}
	if len(w.sessions) != 1 || len(w.releases) != 1 {
		t.Fatalf("recoverable handoff dropped: sessions=%v releases=%v", w.sessions, w.releases)
	}
}

// deadTmuxBackend stands in for a runtime whose server is gone. tmux answers
// "no server running", which TmuxBackend.Kill reports as an already-dead
// session rather than an error.
func deadTmuxBackend(t *testing.T) *herdr.TmuxBackend {
	t.Helper()
	stub := filepath.Join(t.TempDir(), "tmux")
	if err := os.WriteFile(stub, []byte("#!/bin/sh\necho 'no server running' >&2\nexit 1\n"), 0o755); err != nil {
		t.Fatal(err)
	}
	return &herdr.TmuxBackend{Socket: "gone", Binary: stub}
}