kami
ac38b59322
AdapterFactory.Herdrs is keyed by herdr instance id (e.g. "homesrv-claude"), but Reconcile, expire, and rotate all looked adapters up by session.Harness (the harness kind, e.g. "claude"). In production this key never resolves, so every one of those call sites silently no-ops via a bare `continue`: orphaned panes are never killed on restart, expired leases never kill their pane, and rotation exits before it begins. Add Coordinator.adapterFor(taskID, session), matching the fallback already used correctly by refreshSessionHealth (HerdrID, then the lease's HarnessID, then Harness as a last resort), and route all four call sites through it. Regression test TestAdapterResolvedByHerdrIDNotHarnessKind registers an adapter under "homesrv-claude" and leases with Session.Harness == "claude" (reproducing the real key mismatch) and asserts rotation still fires — the existing rotation tests used a keyed-by-nothing fake adapter that matched any lookup string and so masked this bug entirely. AUDIT.md B2.
338 lines
12 KiB
Go
338 lines
12 KiB
Go
package orchestrator_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"orchestra/internal/authz"
|
|
"orchestra/internal/domain"
|
|
"orchestra/internal/herdr"
|
|
"orchestra/internal/orchestrator"
|
|
"orchestra/internal/store"
|
|
"os/exec"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type fakeAdapter struct {
|
|
occupancy float64
|
|
boundary bool
|
|
ref string
|
|
releases int
|
|
}
|
|
|
|
func (a *fakeAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
|
|
return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
|
|
}
|
|
func (a *fakeAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
|
|
func (a *fakeAdapter) Release(context.Context, herdr.Session) (string, error) {
|
|
a.releases++
|
|
return a.ref, nil
|
|
}
|
|
func (a *fakeAdapter) Kill(context.Context, herdr.Session) error { return nil }
|
|
func (a *fakeAdapter) Occupancy(herdr.Session) (float64, error) { return a.occupancy, nil }
|
|
func (a *fakeAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
|
|
return a.boundary, nil
|
|
}
|
|
|
|
type worktrees struct{ path string }
|
|
|
|
func (w worktrees) Create(context.Context, domain.Task) (string, error) { return w.path, nil }
|
|
|
|
type adapters struct{ a herdr.Adapter }
|
|
|
|
func (a adapters) Adapter(string) (herdr.Adapter, error) { return a.a, nil }
|
|
|
|
func run(t *testing.T, dir string, args ...string) {
|
|
t.Helper()
|
|
cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
|
|
if out, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("git %v: %v: %s", args, err, out)
|
|
}
|
|
}
|
|
|
|
// TestRotationEmitsValidReleaseWithAnchorSHA guards the highest-priority spec
|
|
// defect noted in progress.md: automated rotation must emit a TaskReleased
|
|
// event that satisfies domain.ValidatePayload (handoff_ref + anchor_sha), not
|
|
// a payload missing anchor_sha that silently fails to append.
|
|
func TestRotationEmitsValidReleaseWithAnchorSHA(t *testing.T) {
|
|
repo := t.TempDir()
|
|
run(t, repo, "init")
|
|
run(t, repo, "config", "user.email", "t@t")
|
|
run(t, repo, "config", "user.name", "t")
|
|
run(t, repo, "commit", "--allow-empty", "-m", "init")
|
|
head, err := herdr.HeadSHA(repo)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
s, err := store.Open(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
|
"source": "jsonl", "external_id": "1", "project": "p",
|
|
})}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
task := s.Tasks()[0]
|
|
ref, err := s.PutArtifact([]byte("handoff"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := &fakeAdapter{occupancy: .95, boundary: true, ref: ref}
|
|
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
|
|
|
|
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go c.Monitor(ctx, .8, time.Millisecond)
|
|
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if got, ok := s.Task(task.ID); ok && got.State == domain.StateQueued {
|
|
break
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
got, ok := s.Task(task.ID)
|
|
if !ok || got.State != domain.StateQueued {
|
|
t.Fatalf("rotation did not complete: state=%v ok=%v", got.State, ok)
|
|
}
|
|
if a.releases == 0 {
|
|
t.Fatalf("adapter Release was never invoked")
|
|
}
|
|
|
|
// Walk raw events to confirm the coordinator itself wrote a valid
|
|
// TaskReleased payload with anchor_sha == the worktree's real HEAD.
|
|
found := false
|
|
for _, e := range s.Events(0) {
|
|
if e.TaskID != task.ID || e.Type != "TaskReleased" {
|
|
continue
|
|
}
|
|
var p map[string]any
|
|
if err := json.Unmarshal(e.Payload, &p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := domain.ValidatePayload("TaskReleased", p); err != nil {
|
|
t.Fatalf("coordinator emitted invalid TaskReleased: %v (%v)", err, p)
|
|
}
|
|
if p["anchor_sha"] != head {
|
|
t.Fatalf("anchor_sha=%v want=%s", p["anchor_sha"], head)
|
|
}
|
|
found = true
|
|
}
|
|
if !found {
|
|
t.Fatal("coordinator never emitted a TaskReleased event")
|
|
}
|
|
}
|
|
|
|
func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }
|
|
|
|
// keyedHarnessAdapter reports Session.Harness as the harness kind ("claude"),
|
|
// distinct from the herdr instance id ("homesrv-claude") under which it is
|
|
// registered in AdapterFactory.Herdrs — reproducing production's real key
|
|
// mismatch (adapters are keyed by herdr instance id; CLIAdapter.Lease sets
|
|
// Session.Harness to the harness kind).
|
|
type keyedHarnessAdapter struct{ fakeAdapter }
|
|
|
|
func (a *keyedHarnessAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
|
|
return herdr.Session{Harness: "claude", PaneID: "pane-1", Worktree: worktree}, nil
|
|
}
|
|
|
|
// TestAdapterResolvedByHerdrIDNotHarnessKind guards B2: AdapterFactory.Herdrs
|
|
// is keyed by herdr instance id (e.g. "homesrv-claude"), never by the
|
|
// harness kind Session.Harness holds (e.g. "claude"). Reconcile, expire, and
|
|
// rotate must all resolve the adapter via Session.HerdrID (set at lease
|
|
// time), not Session.Harness, or every one of them silently no-ops via a
|
|
// bare Adapter-not-registered continue.
|
|
func TestAdapterResolvedByHerdrIDNotHarnessKind(t *testing.T) {
|
|
repo := t.TempDir()
|
|
run(t, repo, "init")
|
|
run(t, repo, "config", "user.email", "t@t")
|
|
run(t, repo, "config", "user.name", "t")
|
|
run(t, repo, "commit", "--allow-empty", "-m", "init")
|
|
|
|
s, err := store.Open(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
|
"source": "jsonl", "external_id": "1", "project": "p",
|
|
})}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
task := s.Tasks()[0]
|
|
ref, err := s.PutArtifact([]byte("handoff"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := &keyedHarnessAdapter{fakeAdapter{occupancy: .95, boundary: true, ref: ref}}
|
|
factory := orchestrator.AdapterFactory{Herdrs: map[string]herdr.Adapter{"homesrv-claude": a}}
|
|
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: factory, StatePath: t.TempDir() + "/sessions.json"}
|
|
|
|
leaseEvt, err := s.Lease(task.ID, "homesrv-claude", time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go c.Monitor(ctx, .8, time.Millisecond)
|
|
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if got, ok := s.Task(task.ID); ok && got.State == domain.StateQueued {
|
|
break
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
got, ok := s.Task(task.ID)
|
|
if !ok || got.State != domain.StateQueued {
|
|
t.Fatalf("rotation did not complete via herdr-id-keyed adapter: state=%v ok=%v", got.State, ok)
|
|
}
|
|
if a.releases == 0 {
|
|
t.Fatalf("adapter Release was never invoked — adapter lookup used Session.Harness instead of Session.HerdrID")
|
|
}
|
|
}
|
|
|
|
// erroringBoundaryAdapter supports Face B but its probe always fails — this
|
|
// must block release (never silently treat an unanswerable boundary check
|
|
// as safe to interrupt), unlike an adapter that doesn't implement the
|
|
// interface at all.
|
|
type erroringBoundaryAdapter struct{ fakeAdapter }
|
|
|
|
func (a *erroringBoundaryAdapter) AtTurnBoundary(context.Context, herdr.Session) (bool, error) {
|
|
return false, errors.New("pane.status unsupported")
|
|
}
|
|
|
|
// noBoundaryAdapter never implements herdr.TurnBoundary at all, exercising
|
|
// the genuine occupancy-only degraded fallback.
|
|
type noBoundaryAdapter struct {
|
|
occupancy float64
|
|
ref string
|
|
releases int
|
|
}
|
|
|
|
func (a *noBoundaryAdapter) Lease(_ context.Context, _ string, worktree string) (herdr.Session, error) {
|
|
return herdr.Session{Harness: "h1", PaneID: "pane-1", Worktree: worktree}, nil
|
|
}
|
|
func (a *noBoundaryAdapter) Bootstrap(context.Context, herdr.Session, string) error { return nil }
|
|
func (a *noBoundaryAdapter) Release(context.Context, herdr.Session) (string, error) {
|
|
a.releases++
|
|
return a.ref, nil
|
|
}
|
|
func (a *noBoundaryAdapter) Kill(context.Context, herdr.Session) error { return nil }
|
|
func (a *noBoundaryAdapter) Occupancy(herdr.Session) (float64, error) { return a.occupancy, nil }
|
|
|
|
func setupRotationTask(t *testing.T, repo string) (*store.Store, string, domain.Task, string) {
|
|
t.Helper()
|
|
run(t, repo, "init")
|
|
run(t, repo, "config", "user.email", "t@t")
|
|
run(t, repo, "config", "user.name", "t")
|
|
run(t, repo, "commit", "--allow-empty", "-m", "init")
|
|
head, err := herdr.HeadSHA(repo)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s, err := store.Open(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "TaskCreated", TaskID: "t1", Surface: string(authz.System), Payload: mustJSON(map[string]any{
|
|
"source": "jsonl", "external_id": "1", "project": "p",
|
|
})}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
task := s.Tasks()[0]
|
|
ref, err := s.PutArtifact([]byte("handoff"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return s, head, task, ref
|
|
}
|
|
|
|
// TestTurnBoundaryErrorBlocksRelease proves an adapter that implements Face B
|
|
// but cannot currently answer it (a transient herdr error) never falls
|
|
// through to an unconfirmed release — spec §5.2/§5.3 treats the boundary
|
|
// check as required, not best-effort.
|
|
func TestTurnBoundaryErrorBlocksRelease(t *testing.T) {
|
|
repo := t.TempDir()
|
|
s, _, task, ref := setupRotationTask(t, repo)
|
|
a := &erroringBoundaryAdapter{fakeAdapter{occupancy: .95, boundary: true, ref: ref}}
|
|
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
|
|
|
|
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go c.Monitor(ctx, .8, time.Millisecond)
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
got, _ := s.Task(task.ID)
|
|
if got.State != domain.StateLeased {
|
|
t.Fatalf("release proceeded despite an unanswerable turn-boundary check: state=%s", got.State)
|
|
}
|
|
if a.releases != 0 {
|
|
t.Fatalf("adapter.Release was called despite the boundary error, releases=%d", a.releases)
|
|
}
|
|
if c.MonitorHealth().TurnBoundaryDegraded == 0 {
|
|
t.Fatal("turn-boundary degradation was not recorded")
|
|
}
|
|
}
|
|
|
|
// TestNoTurnBoundarySupportDegradesVisibly proves an adapter that never
|
|
// implements Face B still falls back to occupancy-only thresholding (so
|
|
// existing deployments keep working) but the degradation is observable via
|
|
// MonitorHealth, not silent.
|
|
func TestNoTurnBoundarySupportDegradesVisibly(t *testing.T) {
|
|
repo := t.TempDir()
|
|
s, head, task, ref := setupRotationTask(t, repo)
|
|
a := &noBoundaryAdapter{occupancy: .95, ref: ref}
|
|
c := &orchestrator.Coordinator{Store: s, Worktrees: worktrees{path: repo}, Adapters: adapters{a}, StatePath: t.TempDir() + "/sessions.json"}
|
|
|
|
leaseEvt, err := s.Lease(task.ID, "h1", time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Start(context.Background(), leaseEvt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go c.Monitor(ctx, .8, time.Millisecond)
|
|
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if got, ok := s.Task(task.ID); ok && got.State == domain.StateQueued {
|
|
break
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
got, ok := s.Task(task.ID)
|
|
if !ok || got.State != domain.StateQueued {
|
|
t.Fatalf("rotation did not complete without Face B support: state=%v ok=%v", got.State, ok)
|
|
}
|
|
_ = head
|
|
if c.MonitorHealth().TurnBoundaryDegraded == 0 {
|
|
t.Fatal("missing Face B support was not recorded as degraded")
|
|
}
|
|
}
|