orchestra/internal/orchestrator/rotation_test.go

package orchestrator_test

import (
	"context"
	"encoding/json"
	"errors"
	"orchestra/internal/authz"
	"orchestra/internal/domain"
	"orchestra/internal/herdr"
	"orchestra/internal/orchestrator"
	"orchestra/internal/store"
	"os"
	"os/exec"
	"testing"
	"time"
)

type fakeAdapter struct {
	occupancy float64
	boundary  bool
	ref       string
	releases  int
}

func (a *fakeAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
	return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
}
func (a *fakeAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
func (a *fakeAdapter) Release(context.Context, herdr.Session) (string, error) {
	a.releases++
	return a.ref, nil
}
func (a *fakeAdapter) Kill(context.Context, herdr.Session) error         { return nil }
func (a *fakeAdapter) Occupancy(herdr.Session) (float64, error)         { return a.occupancy, nil }
func (a *fakeAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
	return a.boundary, nil
}

type worktrees struct{ path string }

func (w worktrees) Create(context.Context, domain.Task) (string, error) { return w.path, nil }

type adapters struct{ a herdr.Adapter }

func (a adapters) Adapter(string) (herdr.Adapter, error) { return a.a, nil }

func run(t *testing.T, dir string, args ...string) {
	t.Helper()
	cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Fatalf("git %v: %v: %s", args, err, out)
	}
}

// TestRotationEmitsValidReleaseWithAnchorSHA guards the highest-priority spec
// defect noted in progress.md: automated rotation must emit a TaskReleased
// event that satisfies domain.ValidatePayload (handoff_ref + anchor_sha), not
// a payload missing anchor_sha that silently fails to append.
func TestRotationEmitsValidReleaseWithAnchorSHA(t *testing.T) {
	repo := t.TempDir()
	run(t, repo, "init")
	run(t, repo, "config", "user.email", "t@t")
	run(t, repo, "config", "user.name", "t")
	run(t, repo, "commit", "--allow-empty", "-m", "init")
	head, err := herdr.HeadSHA(repo)
	if err != nil {
		t.Fatal(err)
	}

	s, err := store.Open(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
		"source": "jsonl", "external_id": "1", "project": "p",
	})}); err != nil {
		t.Fatal(err)
	}
	task := s.Tasks()[0]
	ref, err := s.PutArtifact([]byte("handoff"))
	if err != nil {
		t.Fatal(err)
	}
	a := &fakeAdapter{occupancy: .95, boundary: true, ref: ref}
	c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}

	leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if err := c.Start(context.Background(), leaseEvt); err != nil {
		t.Fatal(err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go c.Monitor(ctx, .8, time.Millisecond)

	deadline := time.Now().Add(time.Second)
	for time.Now().Before(deadline) {
		if got, ok := s.Task(task.ID); ok && got.State == domain.StateQueued {
			break
		}
		time.Sleep(time.Millisecond)
	}
	got, ok := s.Task(task.ID)
	if !ok || got.State != domain.StateQueued {
		t.Fatalf("rotation did not complete: state=%v ok=%v", got.State, ok)
	}
	if a.releases == 0 {
		t.Fatalf("adapter Release was never invoked")
	}

	// Walk raw events to confirm the coordinator itself wrote a valid
	// TaskReleased payload with anchor_sha == the worktree's real HEAD.
	found := false
	for _, e := range s.Events(0) {
		if e.TaskID != task.ID || e.Type != "TaskReleased" {
			continue
		}
		var p map[string]any
		if err := json.Unmarshal(e.Payload, &p); err != nil {
			t.Fatal(err)
		}
		if err := domain.ValidatePayload("TaskReleased", p); err != nil {
			t.Fatalf("coordinator emitted invalid TaskReleased: %v (%v)", err, p)
		}
		if p["anchor_sha"] != head {
			t.Fatalf("anchor_sha=%v want=%s", p["anchor_sha"], head)
		}
		found = true
	}
	if !found {
		t.Fatal("coordinator never emitted a TaskReleased event")
	}
}

func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }

// TestStartBlocksOnInvalidPickup guards AUDIT.md's B6/Phase 4 item 4:
// Coordinator.Start must run §6.2 pickup validation against the real
// worktree before bootstrapping a successor onto a handoff_ref, and refuse
// (TaskBlocked) rather than bootstrap on a mismatched anchor.
func TestStartBlocksOnInvalidPickup(t *testing.T) {
	repo := t.TempDir()
	run(t, repo, "init")
	run(t, repo, "config", "user.email", "t@t")
	run(t, repo, "config", "user.name", "t")
	run(t, repo, "commit", "--allow-empty", "-m", "init")

	s, err := store.Open(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
		"source": "jsonl", "external_id": "1", "project": "p",
	})}); err != nil {
		t.Fatal(err)
	}
	task := s.Tasks()[0]

	// A handoff whose anchor doesn't match anything in this fresh repo.
	badHandoff := map[string]any{
		"meta":   map[string]any{"id": "h1", "reason": "manual", "rotation_index": 0},
		"anchor": map[string]any{"git_sha": strings0(40, 'a'), "branch": "orchestra/t1"},
		"goal":   "g", "done_when": []string{"x"}, "action": "prompt", "command": "go test ./...",
	}
	ref, err := s.PutArtifact(mustJSON(badHandoff))
	if err != nil {
		t.Fatal(err)
	}

	a := &fakeAdapter{occupancy: 0}
	wt := orchestrator.GitWorktrees{Root: t.TempDir(), Repo: repo}
	c := &orchestrator.Coordinator{Store: s, Worktrees: wt, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}

	leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	b, _ := json.Marshal(map[string]string{"harness_id": "h1", "handoff_ref": ref})
	leaseEvt.Payload = b
	if err := c.Start(context.Background(), leaseEvt); err != nil {
		t.Fatal(err)
	}

	got, ok := s.Task(task.ID)
	if !ok || got.State != domain.StateBlocked {
		t.Fatalf("expected TaskBlocked on invalid pickup, got state=%v ok=%v", got.State, ok)
	}
}

func strings0(n int, c byte) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = c
	}
	return string(b)
}

// keyedHarnessAdapter reports Session.Harness as the harness kind ("claude"),
// distinct from the herdr instance id ("homesrv-claude") under which it is
// registered in AdapterFactory.Herdrs — reproducing production's real key
// mismatch (adapters are keyed by herdr instance id; CLIAdapter.Lease sets
// Session.Harness to the harness kind).
type keyedHarnessAdapter struct{ fakeAdapter }

func (a *keyedHarnessAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
	return herdr.Session{Harness: "claude", PaneID: "pane-1", Worktree: worktree}, nil
}

// TestAdapterResolvedByHerdrIDNotHarnessKind guards B2: AdapterFactory.Herdrs
// is keyed by herdr instance id (e.g. "homesrv-claude"), never by the
// harness kind Session.Harness holds (e.g. "claude"). Reconcile, expire, and
// rotate must all resolve the adapter via Session.HerdrID (set at lease
// time), not Session.Harness, or every one of them silently no-ops via a
// bare Adapter-not-registered continue.
func TestAdapterResolvedByHerdrIDNotHarnessKind(t *testing.T) {
	repo := t.TempDir()
	run(t, repo, "init")
	run(t, repo, "config", "user.email", "t@t")
	run(t, repo, "config", "user.name", "t")
	run(t, repo, "commit", "--allow-empty", "-m", "init")

	s, err := store.Open(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
		"source": "jsonl", "external_id": "1", "project": "p",
	})}); err != nil {
		t.Fatal(err)
	}
	task := s.Tasks()[0]
	ref, err := s.PutArtifact([]byte("handoff"))
	if err != nil {
		t.Fatal(err)
	}
	a := &keyedHarnessAdapter{fakeAdapter{occupancy: .95, boundary: true, ref: ref}}
	factory := orchestrator.AdapterFactory{Herdrs: map[string]herdr.Adapter{"homesrv-claude": a}}
	c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: factory, StatePath: t.TempDir() + "/sessions.json"}

	leaseEvt, err := s.Lease(task.ID, "homesrv-claude", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if err := c.Start(context.Background(), leaseEvt); err != nil {
		t.Fatal(err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go c.Monitor(ctx, .8, time.Millisecond)

	deadline := time.Now().Add(time.Second)
	for time.Now().Before(deadline) {
		if got, ok := s.Task(task.ID); ok && got.State == domain.StateQueued {
			break
		}
		time.Sleep(time.Millisecond)
	}
	got, ok := s.Task(task.ID)
	if !ok || got.State != domain.StateQueued {
		t.Fatalf("rotation did not complete via herdr-id-keyed adapter: state=%v ok=%v", got.State, ok)
	}
	if a.releases == 0 {
		t.Fatalf("adapter Release was never invoked — adapter lookup used Session.Harness instead of Session.HerdrID")
	}
}

// erroringBoundaryAdapter supports Face B but its probe always fails — this
// must block release (never silently treat an unanswerable boundary check
// as safe to interrupt), unlike an adapter that doesn't implement the
// interface at all.
type erroringBoundaryAdapter struct{ fakeAdapter }

func (a *erroringBoundaryAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
	return false, errors.New("pane.status unsupported")
}

// noBoundaryAdapter never implements herdr.TurnBoundary at all, exercising
// the genuine occupancy-only degraded fallback.
type noBoundaryAdapter struct {
	occupancy float64
	ref       string
	releases  int
}

func (a *noBoundaryAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
	return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
}
func (a *noBoundaryAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
func (a *noBoundaryAdapter) Release(context.Context, herdr.Session) (string, error) {
	a.releases++
	return a.ref, nil
}
func (a *noBoundaryAdapter) Kill(context.Context, herdr.Session) error         { return nil }
func (a *noBoundaryAdapter) Occupancy(herdr.Session) (float64, error)         { return a.occupancy, nil }

func setupRotationTask(t *testing.T, repo string) (*store.Store, string, domain.Task, string) {
	t.Helper()
	run(t, repo, "init")
	run(t, repo, "config", "user.email", "t@t")
	run(t, repo, "config", "user.name", "t")
	run(t, repo, "commit", "--allow-empty", "-m", "init")
	head, err := herdr.HeadSHA(repo)
	if err != nil {
		t.Fatal(err)
	}
	s, err := store.Open(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
		"source": "jsonl", "external_id": "1", "project": "p",
	})}); err != nil {
		t.Fatal(err)
	}
	task := s.Tasks()[0]
	ref, err := s.PutArtifact([]byte("handoff"))
	if err != nil {
		t.Fatal(err)
	}
	return s, head, task, ref
}

// TestTurnBoundaryErrorBlocksRelease proves an adapter that implements Face B
// but cannot currently answer it (a transient herdr error) never falls
// through to an unconfirmed release — spec §5.2/§5.3 treats the boundary
// check as required, not best-effort.
func TestTurnBoundaryErrorBlocksRelease(t *testing.T) {
	repo := t.TempDir()
	s, _, task, ref := setupRotationTask(t, repo)
	a := &erroringBoundaryAdapter{fakeAdapter{occupancy: .95, boundary: true, ref: ref}}
	c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}

	leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if err := c.Start(context.Background(), leaseEvt); err != nil {
		t.Fatal(err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go c.Monitor(ctx, .8, time.Millisecond)

	time.Sleep(50 * time.Millisecond)
	got, _ := s.Task(task.ID)
	if got.State != domain.StateLeased {
		t.Fatalf("release proceeded despite an unanswerable turn-boundary check: state=%s", got.State)
	}
	if a.releases != 0 {
		t.Fatalf("adapter.Release was called despite the boundary error, releases=%d", a.releases)
	}
	if c.MonitorHealth().TurnBoundaryDegraded == 0 {
		t.Fatal("turn-boundary degradation was not recorded")
	}
}

// TestNoTurnBoundarySupportDegradesVisibly proves an adapter that never
// implements Face B still falls back to occupancy-only thresholding (so
// existing deployments keep working) but the degradation is observable via
// MonitorHealth, not silent.
func TestNoTurnBoundarySupportDegradesVisibly(t *testing.T) {
	repo := t.TempDir()
	s, head, task, ref := setupRotationTask(t, repo)
	a := &noBoundaryAdapter{occupancy: .95, ref: ref}
	c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}

	leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if err := c.Start(context.Background(), leaseEvt); err != nil {
		t.Fatal(err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go c.Monitor(ctx, .8, time.Millisecond)

	deadline := time.Now().Add(time.Second)
	for time.Now().Before(deadline) {
		if got, ok := s.Task(task.ID); ok && got.State == domain.StateQueued {
			break
		}
		time.Sleep(time.Millisecond)
	}
	got, ok := s.Task(task.ID)
	if !ok || got.State != domain.StateQueued {
		t.Fatalf("rotation did not complete without Face B support: state=%v ok=%v", got.State, ok)
	}
	_ = head
	if c.MonitorHealth().TurnBoundaryDegraded == 0 {
		t.Fatal("missing Face B support was not recorded as degraded")
	}
}

type handoffRequestingAdapter struct {
	fakeAdapter
	requests int
}

func (a *handoffRequestingAdapter) RequestHandoff(context.Context, herdr.Session) error {
	a.requests++
	return nil
}

// TestRotationRequestsHandoffBeforeReleasing guards Phase 4 item 2 (AUDIT.md):
// rotate() must not call Release until the agent has been told to write its
// §6.1 handoff and the file actually exists — never invent or skip the ask.
func TestRotationRequestsHandoffBeforeReleasing(t *testing.T) {
	repo := t.TempDir()
	run(t, repo, "init")
	run(t, repo, "config", "user.email", "t@t")
	run(t, repo, "config", "user.name", "t")
	run(t, repo, "commit", "--allow-empty", "-m", "init")

	s, err := store.Open(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
		"source": "jsonl", "external_id": "1", "project": "p",
	})}); err != nil {
		t.Fatal(err)
	}
	task := s.Tasks()[0]
	ref, err := s.PutArtifact([]byte("handoff"))
	if err != nil {
		t.Fatal(err)
	}
	a := &handoffRequestingAdapter{fakeAdapter: fakeAdapter{occupancy: .95, boundary: true, ref: ref}}
	c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}

	leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	if err := c.Start(context.Background(), leaseEvt); err != nil {
		t.Fatal(err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go c.Monitor(ctx, .8, time.Millisecond)

	deadline := time.Now().Add(200 * time.Millisecond)
	for time.Now().Before(deadline) {
		time.Sleep(time.Millisecond)
	}
	if a.requests == 0 {
		t.Fatal("rotate never asked the agent to write a handoff")
	}
	if a.releases != 0 {
		t.Fatal("rotate called Release before the handoff file existed")
	}
	if got, ok := s.Task(task.ID); !ok || got.State != domain.StateLeased {
		t.Fatalf("task rotated without a handoff file: state=%v ok=%v", got.State, ok)
	}

	if err := os.WriteFile(repo+"/"+herdr.HandoffFile, []byte("{}"), 0644); err != nil {
		t.Fatal(err)
	}

	deadline = time.Now().Add(time.Second)
	for time.Now().Before(deadline) {
		if got, ok := s.Task(task.ID); ok && got.State == domain.StateQueued {
			break
		}
		time.Sleep(time.Millisecond)
	}
	if a.releases == 0 {
		t.Fatal("rotate never called Release once the handoff file appeared")
	}
}