orchestra/cmd/orchestra-worker/phase_test.go

package main

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"orchestra/internal/domain"
	"orchestra/internal/federation"
	"orchestra/internal/herdr"
	"orchestra/internal/workphase"
)

// phaseWorker builds a worker whose single session sits at a turn boundary,
// with a worktree the agent can write its request into.
func phaseWorker(t *testing.T, handler http.HandlerFunc) (*worker, *recordingBackend, string, func()) {
	t.Helper()
	api := httptest.NewServer(handler)
	wt := t.TempDir()
	if err := os.MkdirAll(filepath.Join(wt, ".orchestra"), 0o755); err != nil {
		t.Fatal(err)
	}
	backend := &recordingBackend{status: "idle"}
	w := &worker{
		api:         federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
		backend:     backend,
		harness:     "claude",
		sessions:    map[string]herdr.Session{"task": {PaneID: "pane", Worktree: wt, Phase: string(domain.WorkPhaseFrame)}},
		leases:      map[string]lease{"task": {Epoch: "e1", Version: 1, Until: time.Now().Add(time.Hour)}},
		tasks:       map[string]domain.Task{"task": {ID: "task", WorkPhase: domain.WorkPhaseFrame}},
		quarantined: map[string]bool{},
		statePath:   filepath.Join(t.TempDir(), "state.json"),
	}
	return w, backend, wt, api.Close
}

func writeRequest(t *testing.T, wt string, from, to domain.WorkPhase) {
	t.Helper()
	b, _ := json.Marshal(phaseRequest{From: from, To: to})
	if err := os.WriteFile(filepath.Join(wt, ".orchestra", phaseRequestFile), b, 0o644); err != nil {
		t.Fatal(err)
	}
}

// The whole point of F21: the agent asks with a file, Orchestra answers, and
// the session that asked is rotated rather than left idling until its lease
// dies. Run 3 failed conformance because none of this existed.
func TestPhaseRequestAdvancesAndRotatesTheSession(t *testing.T) {
	var sent map[string]any
	w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/v1/federation/phase" {
			t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
			rw.WriteHeader(http.StatusNotFound)
			return
		}
		_ = json.NewDecoder(r.Body).Decode(&sent)
		_ = json.NewEncoder(rw).Encode(map[string]string{"phase": "research"})
	})
	defer done()
	writeRequest(t, wt, domain.WorkPhaseFrame, domain.WorkPhaseResearch)

	a := herdr.CLIAdapter{Backend: backend, Harness: "claude"}
	w.federatedTurn(context.Background(), "task", a, "continue")

	if sent["lease_epoch"] != "e1" || sent["from"] != "frame" || sent["to"] != "research" {
		t.Fatalf("request = %+v", sent)
	}
	if op, _ := sent["operation_id"].(string); op == "" {
		t.Fatal("request carried no operation id")
	}
	// Accepted and rotating. A phase change that left the old session running
	// is the F22 bug, so this assertion is the test for it.
	s := w.sessions["task"]
	if !s.HandoffRequested || s.HandoffReason != "phase_changed" {
		t.Fatalf("session did not rotate: %+v", s)
	}
	if len(backend.prompts) == 0 || !strings.Contains(backend.prompts[len(backend.prompts)-1], "next work phase") {
		t.Fatalf("agent was not told why it is stopping: %q", backend.prompts)
	}
	// Consumed, so the same request is not replayed at the next boundary.
	if _, err := os.Stat(filepath.Join(wt, ".orchestra", phaseRequestFile)); !os.IsNotExist(err) {
		t.Fatal("the accepted request file survived")
	}
}

// A phase an operator advanced through the coordinator ends the session too.
// The session's context was built for a phase that is no longer current, and
// nothing about that depends on who asked.
func TestExternalPhaseChangeRotatesTheSession(t *testing.T) {
	w, backend, _, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/v1/federation/phase" {
			t.Errorf("an external phase change was re-requested by the worker")
		}
		rw.Write([]byte(`{"verdict":"continue"}`))
	})
	defer done()
	task := w.tasks["task"]
	task.WorkPhase = domain.WorkPhaseResearch
	w.tasks["task"] = task

	a := herdr.CLIAdapter{Backend: backend, Harness: "claude"}
	w.federatedTurn(context.Background(), "task", a, "continue")

	if s := w.sessions["task"]; !s.HandoffRequested || s.HandoffReason != "phase_changed" {
		t.Fatalf("session did not rotate: %+v", s)
	}
}

// A phase that seals an artifact must not leave without one. Caught locally so
// the agent is told while its session is still alive to be told.
func TestPhaseRequestRefusesAnUnsealedArtifact(t *testing.T) {
	w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/v1/federation/phase" {
			t.Errorf("an unsealed phase request reached the coordinator")
		}
		rw.Write([]byte(`{"verdict":"continue"}`))
	})
	defer done()
	task := w.tasks["task"]
	task.WorkPhase = domain.WorkPhaseResearch
	w.tasks["task"] = task
	w.sessions["task"] = herdr.Session{PaneID: "pane", Worktree: wt, Phase: string(domain.WorkPhaseResearch)}
	writeRequest(t, wt, domain.WorkPhaseResearch, domain.WorkPhasePlan)

	a := herdr.CLIAdapter{Backend: backend, Harness: "claude"}
	w.federatedTurn(context.Background(), "task", a, "continue")

	if s := w.sessions["task"]; s.HandoffRequested {
		t.Fatal("a refused request rotated the session")
	}
	if !strings.Contains(w.lastError, "research.json") {
		t.Fatalf("lastError = %q", w.lastError)
	}
}

// A sealed artifact travels with the request, so the next phase reads a result
// instead of reconstructing a conversation.
func TestPhaseRequestCarriesTheSealedArtifact(t *testing.T) {
	var sent struct {
		Artifact []byte `json:"artifact"`
	}
	w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		_ = json.NewDecoder(r.Body).Decode(&sent)
		_ = json.NewEncoder(rw).Encode(map[string]string{"phase": "plan"})
	})
	defer done()
	task := w.tasks["task"]
	task.WorkPhase = domain.WorkPhaseResearch
	w.tasks["task"] = task
	w.sessions["task"] = herdr.Session{PaneID: "pane", Worktree: wt, Phase: string(domain.WorkPhaseResearch)}

	sealed, err := workphase.Encode(workphase.Research{Findings: []workphase.Finding{{ID: "r1", Confidence: workphase.Fact, Claim: "c", Evidence: "e"}}})
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(wt, ".orchestra", "research.json"), sealed, 0o644); err != nil {
		t.Fatal(err)
	}
	writeRequest(t, wt, domain.WorkPhaseResearch, domain.WorkPhasePlan)

	a := herdr.CLIAdapter{Backend: backend, Harness: "claude"}
	w.federatedTurn(context.Background(), "task", a, "continue")

	if _, err := workphase.DecodeResearch(sent.Artifact); err != nil {
		t.Fatalf("artifact did not arrive sealed: %v", err)
	}
}

// A malformed artifact is the agent's mistake, and it must not be sealed into
// the log as the phase's accepted result.
func TestPhaseRequestRefusesAMalformedArtifact(t *testing.T) {
	w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/v1/federation/phase" {
			t.Errorf("a malformed artifact reached the coordinator")
		}
		rw.Write([]byte(`{"verdict":"continue"}`))
	})
	defer done()
	task := w.tasks["task"]
	task.WorkPhase = domain.WorkPhaseResearch
	w.tasks["task"] = task
	w.sessions["task"] = herdr.Session{PaneID: "pane", Worktree: wt, Phase: string(domain.WorkPhaseResearch)}
	if err := os.WriteFile(filepath.Join(wt, ".orchestra", "research.json"), []byte(`{"findings":[]}`), 0o644); err != nil {
		t.Fatal(err)
	}
	writeRequest(t, wt, domain.WorkPhaseResearch, domain.WorkPhasePlan)

	a := herdr.CLIAdapter{Backend: backend, Harness: "claude"}
	w.federatedTurn(context.Background(), "task", a, "continue")

	if s := w.sessions["task"]; s.HandoffRequested {
		t.Fatal("a malformed artifact rotated the session")
	}
}

// A refusal is an answer. The agent is told why, in the same confirmed
// delivery path every other Orchestra-originated input uses, and the request
// is cleared so it can write a corrected one instead of resending the same
// rejected file at every boundary.
func TestRefusedPhaseRequestIsAnsweredAndCleared(t *testing.T) {
	w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/v1/federation/phase" {
			http.Error(rw, "phase request refused: task may only move to \"research\", not \"implement\"", http.StatusConflict)
			return
		}
		rw.Write([]byte(`{"verdict":"continue"}`))
	})
	defer done()
	writeRequest(t, wt, domain.WorkPhaseFrame, domain.WorkPhaseImplement)

	a := herdr.CLIAdapter{Backend: backend, Harness: "claude"}
	w.federatedTurn(context.Background(), "task", a, "continue")

	if s := w.sessions["task"]; s.HandoffRequested {
		t.Fatal("a refused request rotated the session")
	}
	if len(backend.prompts) == 0 || !strings.Contains(backend.prompts[0], `may only move to "research"`) {
		t.Fatalf("the agent was not told why: %q", backend.prompts)
	}
	if _, err := os.Stat(filepath.Join(wt, ".orchestra", phaseRequestFile)); !os.IsNotExist(err) {
		t.Fatal("the refused request survived, so the agent will resend it")
	}
}

// A coordinator that cannot be reached has not refused anything. Telling the
// agent its request was rejected would be a lie, and dropping the file would
// lose a request that is still valid.
func TestTransientPhaseFailureKeepsTheRequest(t *testing.T) {
	w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/v1/federation/phase" {
			http.Error(rw, "upstream down", http.StatusServiceUnavailable)
			return
		}
		rw.Write([]byte(`{"verdict":"continue"}`))
	})
	defer done()
	writeRequest(t, wt, domain.WorkPhaseFrame, domain.WorkPhaseResearch)

	a := herdr.CLIAdapter{Backend: backend, Harness: "claude"}
	w.federatedTurn(context.Background(), "task", a, "continue")

	if _, err := os.Stat(filepath.Join(wt, ".orchestra", phaseRequestFile)); err != nil {
		t.Fatal("a transient failure discarded the request")
	}
	for _, p := range backend.prompts {
		if strings.Contains(p, "refused") {
			t.Fatalf("a transient failure was reported to the agent as a refusal: %q", p)
		}
	}
}

// confirmingBackend fails its first confirmation and accepts the next, which
// is the shape of a submit that never reached the harness.
type confirmingBackend struct {
	recordingBackend
	failures int
}

func (b *confirmingBackend) ConfirmInput(context.Context, herdr.Session, string) (string, error) {
	if b.failures > 0 {
		b.failures--
		return "", errPromptNotSubmitted
	}
	return "input editor cleared", nil
}

var errPromptNotSubmitted = &confirmError{}

type confirmError struct{}

func (*confirmError) Error() string { return "prompt_not_submitted" }

// F20's guarantee is not that a send was attempted, it is that the harness
// took it. A decision whose Enter was lost must stay undelivered, so the same
// correction is sent again at the next boundary rather than being recorded as
// shown to an agent that never saw it.
func TestDecisionNoticeStaysUndeliveredUntilConfirmed(t *testing.T) {
	api := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		var body struct {
			Delivered []string `json:"delivered_decisions"`
		}
		_ = json.NewDecoder(r.Body).Decode(&body)
		out := federation.TurnDecision{Verdict: "continue"}
		if len(body.Delivered) == 0 {
			out.Decisions = []domain.HumanDecision{{
				ID: "d1", Kind: domain.HumanDecisionCorrection, Subject: "strategy", Value: "no, use b",
			}}
		}
		_ = json.NewEncoder(rw).Encode(out)
	}))
	defer api.Close()

	backend := &confirmingBackend{recordingBackend: recordingBackend{status: "idle"}, failures: 1}
	w := &worker{
		api:       federation.Client{BaseURL: api.URL, WorkerID: "h", Token: "t"},
		backend:   backend,
		harness:   "claude",
		sessions:  map[string]herdr.Session{"task": {PaneID: "pane"}},
		leases:    map[string]lease{"task": {Epoch: "e1", Version: 2}},
		tasks:     map[string]domain.Task{"task": {ID: "task"}},
		statePath: filepath.Join(t.TempDir(), "state.json"),
	}

	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
	if ids := w.sessions["task"].DeliveredDecisions; len(ids) != 0 {
		t.Fatalf("an unconfirmed correction was recorded as delivered: %v", ids)
	}

	w.federatedTurn(context.Background(), "task", boundaryAdapter{at: true}, "continue")
	if ids := w.sessions["task"].DeliveredDecisions; len(ids) != 1 || ids[0] != "d1" {
		t.Fatalf("delivered ids = %v", ids)
	}
	if len(backend.prompts) != 2 {
		t.Fatalf("sends = %d, want the correction retried once", len(backend.prompts))
	}
}

// The bug that made run 4 stall exactly like run 3. rotationTick returned
// early for the claude harness before reaching the turn boundary, so
// federatedTurn had one call site that this harness never took. Phase requests
// were never read and human decisions were never delivered on the harness both
// burn-in runs actually used.
//
// Claude still skips the occupancy state machine below that branch, because it
// owns its own context rollover. A turn boundary is not a rotation.
func TestClaudeHarnessReachesTheTurnBoundary(t *testing.T) {
	reached := make(chan string, 4)
	w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		reached <- r.URL.Path
		if r.URL.Path == "/v1/federation/phase" {
			_ = json.NewEncoder(rw).Encode(map[string]string{"phase": "research"})
			return
		}
		rw.Write([]byte(`{"verdict":"continue"}`))
	})
	defer done()
	writeRequest(t, wt, domain.WorkPhaseFrame, domain.WorkPhaseResearch)

	// rotationTick, not federatedTurn: the dead path was the route in.
	w.rotationTick(context.Background(), "task", w.sessions["task"])

	var saw bool
	for len(reached) > 0 {
		if <-reached == "/v1/federation/phase" {
			saw = true
		}
	}
	if !saw {
		t.Fatal("the claude harness never reached the phase boundary")
	}
	if s := w.sessions["task"]; !s.HandoffRequested || s.HandoffReason != "phase_changed" {
		t.Fatalf("session did not rotate: %+v", s)
	}
	_ = backend
}

// An agent may request verification. It may never assert one: writing
// "verified" is claiming its own work is done, which is what the whole
// machinery exists to prevent. The refusal has to reach the pane, or the
// session rewrites the same rejected file at every boundary.
func TestPlanVerificationRefusesAnyStatusButARequest(t *testing.T) {
	for _, status := range []string{"verified", "awaiting_manual_verification", "failed", "skipped", ""} {
		w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
			t.Errorf("a refused request reached the coordinator at %s", r.URL.Path)
			rw.WriteHeader(http.StatusNotFound)
		})
		path := filepath.Join(wt, ".orchestra", planProgressFile)
		if err := os.WriteFile(path, []byte(`{"phase":"phase-1","status":"`+status+`"}`), 0o644); err != nil {
			t.Fatal(err)
		}
		if w.requestPlanVerification(context.Background(), "task", w.sessions["task"]) {
			t.Fatalf("status %q was accepted", status)
		}
		if len(backend.prompts) == 0 {
			t.Fatalf("status %q was refused with nothing delivered to the pane", status)
		}
		if !strings.Contains(backend.prompts[0], "ready_for_verification") {
			t.Fatalf("the refusal does not name the only writable status: %s", backend.prompts[0])
		}
		if _, err := os.Stat(path); !os.IsNotExist(err) {
			t.Fatalf("status %q left the refused request in place", status)
		}
		done()
	}
}

// An absent request is not a failure and must not reach the pane.
func TestNoPlanVerificationRequestIsSilent(t *testing.T) {
	w, backend, _, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		t.Errorf("an absent request reached the coordinator at %s", r.URL.Path)
	})
	defer done()
	if w.requestPlanVerification(context.Background(), "task", w.sessions["task"]) {
		t.Fatal("an absent request reported work done")
	}
	if len(backend.prompts) != 0 {
		t.Fatalf("an absent request spoke to the pane: %v", backend.prompts)
	}
}

// The commands come from the accepted plan, resolved and authorised by the
// coordinator. The worker must never take one out of the agent's request.
func TestPlanVerificationRunsThePlansCommandsAndReportsExitCodes(t *testing.T) {
	var reported map[string]any
	w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		switch {
		case strings.HasSuffix(r.URL.Path, "/plan-phase"):
			json.NewEncoder(rw).Encode(map[string]any{"commands": [][]string{{"true"}, {"false"}}})
		case strings.HasSuffix(r.URL.Path, "/plan-phase-result"):
			json.NewDecoder(r.Body).Decode(&reported)
			json.NewEncoder(rw).Encode(map[string]any{"status": "in_progress"})
		default:
			t.Errorf("unexpected %s", r.URL.Path)
			rw.WriteHeader(http.StatusNotFound)
		}
	})
	defer done()
	// A git worktree, so the verification can anchor to a real commit.
	for _, args := range [][]string{{"init"}, {"config", "user.email", "t@example.com"}, {"config", "user.name", "t"}, {"commit", "--allow-empty", "-m", "base"}} {
		if out, err := git(context.Background(), wt, args...); err != nil {
			t.Fatalf("git %v: %s: %v", args, out, err)
		}
	}
	path := filepath.Join(wt, ".orchestra", planProgressFile)
	if err := os.WriteFile(path, []byte(`{"phase":"phase-1","status":"ready_for_verification","commands":[["rm","-rf","/"]]}`), 0o644); err != nil {
		t.Fatal(err)
	}
	if !w.requestPlanVerification(context.Background(), "task", w.sessions["task"]) {
		t.Fatal("a valid request did nothing")
	}
	runs, _ := reported["runs"].([]any)
	if len(runs) != 2 {
		t.Fatalf("reported %d runs, want the plan's 2: %v", len(runs), reported)
	}
	first, _ := runs[0].(map[string]any)
	second, _ := runs[1].(map[string]any)
	if first["exit_code"].(float64) != 0 || second["exit_code"].(float64) == 0 {
		t.Fatalf("exit codes were not reported faithfully: %v", runs)
	}
	// The command list in the request is ignored entirely.
	cmd, _ := first["command"].([]any)
	if len(cmd) != 1 || cmd[0].(string) != "true" {
		t.Fatalf("the worker ran something other than the plan's command: %v", cmd)
	}
	if sha, _ := reported["at_sha"].(string); len(sha) != 40 {
		t.Fatalf("the verification was not anchored to a commit: %q", sha)
	}
	// A phase that did not verify has to say why, or the agent sees its
	// request vanish and guesses.
	if len(backend.prompts) == 0 || !strings.Contains(backend.prompts[0], "not verified") {
		t.Fatalf("the outcome was not delivered: %v", backend.prompts)
	}
}

// A request belongs to the session that wrote it. The worktree outlives the
// session, so a rotation that leaves these files behind hands them to a
// successor running in a different phase (run 20).
func TestRotationDropsTheEndedSessionsRequests(t *testing.T) {
	w, backend, wt, done := phaseWorker(t, func(rw http.ResponseWriter, r *http.Request) {
		rw.Write([]byte(`{}`))
	})
	defer done()
	progress := filepath.Join(wt, ".orchestra", planProgressFile)
	request := filepath.Join(wt, ".orchestra", phaseRequestFile)
	for _, p := range []string{progress, request} {
		if err := os.WriteFile(p, []byte(`{"phase":"phase-1","status":"ready_for_verification"}`), 0o644); err != nil {
			t.Fatal(err)
		}
	}

	a := herdr.CLIAdapter{Backend: backend, Harness: "claude"}
	w.rotateForPhase(context.Background(), "task", a, w.sessions["task"])

	for _, p := range []string{progress, request} {
		if _, err := os.Stat(p); !os.IsNotExist(err) {
			t.Fatalf("%s survived the rotation that ended the session that wrote it", filepath.Base(p))
		}
	}
	if s := w.sessions["task"]; !s.HandoffRequested || s.HandoffReason != "phase_changed" {
		t.Fatalf("the session was not rotated: %+v", s)
	}
}