package orchestrator_test import ( "context" "encoding/json" "errors" "os" "strings" "testing" "orchestra/internal/continuity" "orchestra/internal/domain" "orchestra/internal/herdr" "orchestra/internal/orchestrator" ) // reasoningAdapter records the rotation reasons Orchestra asked a handoff for. type reasoningAdapter struct { fakeAdapter reasons []string } func (a *reasoningAdapter) RequestHandoffReason(_ context.Context, _ herdr.Session, reason string, _ []continuity.DeadEnd) error { a.reasons = append(a.reasons, reason) return nil } var down = errors.New("gitea unreachable") // Repeated failure at a verified boundary means Orchestra can no longer uphold // "the newest human input outranks the agent's current intent". The first two // turns continue, because one outage should not stop work. The third hands the // task to a successor, whose pre-lease reconcile fails closed while the source // is still down. func TestReconcileFailureStreakEscalatesToHandoff(t *testing.T) { a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .5}} c, _, task := leasedCoordinator(t, a, gitRepo(t)) c.ReconcileFailureHandoff = 3 c.ReconcileHumanInput = func(context.Context, string) error { return down } for turn := 1; turn <= 2; turn++ { verdict, err := c.TurnDecision(context.Background(), task.ID) if err != nil { t.Fatal(err) } if verdict != orchestrator.TurnContinue { t.Fatalf("turn %d verdict = %q, want continue", turn, verdict) } if len(a.reasons) != 0 { t.Fatalf("turn %d asked for a handoff: %v", turn, a.reasons) } } verdict, err := c.TurnDecision(context.Background(), task.ID) if err != nil { t.Fatal(err) } if verdict != orchestrator.TurnPrepareHandoff { t.Fatalf("verdict = %q, want prepare_handoff", verdict) } if len(a.reasons) != 1 || a.reasons[0] != "reconcile_failure" { t.Fatalf("reasons = %v", a.reasons) } // The count is observable, not just acted on. if h := c.MonitorHealth().Sessions[task.ID]; !strings.Contains(h.LastError, "3 consecutive") { t.Fatalf("streak not observable: %+v", h) } } // One success clears the streak. Two failures then a success then a failure is // one failure, not three. func TestReconcileSuccessResetsTheStreak(t *testing.T) { a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .5}} c, _, task := leasedCoordinator(t, a, gitRepo(t)) c.ReconcileFailureHandoff = 3 failing := true c.ReconcileHumanInput = func(context.Context, string) error { if failing { return down } return nil } turn := func() string { t.Helper() v, err := c.TurnDecision(context.Background(), task.ID) if err != nil { t.Fatal(err) } return v } turn() turn() failing = false turn() failing = true if v := turn(); v != orchestrator.TurnContinue { t.Fatalf("verdict = %q, want continue after the streak reset", v) } if len(a.reasons) != 0 { t.Fatalf("escalated on a reset streak: %v", a.reasons) } } // A rotation that already wants to stop keeps its own reason. Orchestra must // not manufacture a second trigger for a session that is already handing off. func TestReconcileStreakDoesNotOverrideAnExistingRotation(t *testing.T) { repo := gitRepo(t) a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .95, boundary: true}} c, s, task := leasedCoordinator(t, a, repo) ref, err := s.PutArtifact([]byte("handoff")) if err != nil { t.Fatal(err) } a.ref = ref c.ReconcileFailureHandoff = 1 c.ReconcileHumanInput = func(context.Context, string) error { return down } verdict, err := c.TurnDecision(context.Background(), task.ID) if err != nil { t.Fatal(err) } if verdict != orchestrator.TurnRotateNow { t.Fatalf("verdict = %q, want rotate_now", verdict) } if len(a.reasons) != 0 { t.Fatalf("manufactured a reason for a rotating session: %v", a.reasons) } } // The escape path has to complete. Once the agent writes the handoff with this // reason, the ordinary bypass releases the task, exactly as it does for // manual, milestone and thrash. func TestReconcileFailureHandoffReleasesTheTask(t *testing.T) { repo := gitRepo(t) a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: 0, boundary: false}} c, s, task := leasedCoordinator(t, a, repo) ref, err := s.PutArtifact([]byte("handoff")) if err != nil { t.Fatal(err) } a.ref = ref head, err := herdr.HeadSHA(repo) if err != nil { t.Fatal(err) } b, err := json.Marshal(map[string]any{ "meta": map[string]any{"id": "h2", "reason": "reconcile_failure", "rotation_index": 0}, "anchor": map[string]any{"git_sha": head, "branch": "orchestra/t1"}, "action": "re-read the task intent before continuing", "command": "go test ./...", }) if err != nil { t.Fatal(err) } if err := os.WriteFile(repo+"/"+herdr.HandoffFile, b, 0o644); err != nil { t.Fatal(err) } // The reason must survive handoff validation, or the successor cannot read // the artifact this release produced. if _, err := continuity.Decode(b); err != nil { t.Fatalf("handoff rejected: %v", err) } c.ReconcileHumanInput = func(context.Context, string) error { return down } verdict, err := c.TurnDecision(context.Background(), task.ID) if err != nil { t.Fatal(err) } if verdict != orchestrator.TurnRotateNow { t.Fatalf("verdict = %q, want rotate_now", verdict) } if got, _ := s.Task(task.ID); got.State != domain.StateQueued { t.Fatalf("state = %s, want queued", got.State) } } // No human source means nothing to fail, so nothing ever escalates. func TestNoHumanSourceNeverEscalates(t *testing.T) { a := &reasoningAdapter{fakeAdapter: fakeAdapter{occupancy: .5}} c, _, task := leasedCoordinator(t, a, gitRepo(t)) c.ReconcileFailureHandoff = 1 for turn := 0; turn < 5; turn++ { verdict, err := c.TurnDecision(context.Background(), task.ID) if err != nil { t.Fatal(err) } if verdict != orchestrator.TurnContinue { t.Fatalf("verdict = %q, want continue", verdict) } } if len(a.reasons) != 0 { t.Fatalf("escalated with no reconciler configured: %v", a.reasons) } } // Delivering a decision is not reconciling one. A pane that cannot be written // to must not spend the reconcile budget. func TestDeliveryFailureIsNotAReconcileFailure(t *testing.T) { a := ¬ifyingAdapter{fakeAdapter: fakeAdapter{occupancy: .5}, err: context.DeadlineExceeded} c, s, task := leasedCoordinator(t, a, gitRepo(t)) c.ReconcileFailureHandoff = 2 c.ReconcileHumanInput = func(context.Context, string) error { return nil } recordDecision(t, s, task.ID, "d1", "no, use b") for turn := 0; turn < 3; turn++ { verdict, err := c.TurnDecision(context.Background(), task.ID) if err != nil { t.Fatal(err) } if verdict != orchestrator.TurnContinue { t.Fatalf("verdict = %q, want continue", verdict) } } } // The federated half uses the same threshold, so a worker-owned session and a // local one behave identically. func TestRemoteTurnEscalatesOnTheSameThreshold(t *testing.T) { c, _, task := remoteLeased(t) c.ReconcileFailureHandoff = 3 c.ReconcileHumanInput = func(context.Context, string) error { return down } for turn := 1; turn <= 2; turn++ { verdict, _, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnContinue, nil) if err != nil { t.Fatal(err) } if verdict != orchestrator.TurnContinue { t.Fatalf("turn %d verdict = %q, want continue", turn, verdict) } } verdict, decisions, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnContinue, nil) if err != nil { t.Fatal(err) } if verdict != orchestrator.TurnPrepareHandoff { t.Fatalf("verdict = %q, want prepare_handoff", verdict) } if len(decisions) != 0 { t.Fatalf("decisions returned to a rotating session: %+v", decisions) } } // A worker that already reported a stop keeps its own verdict. func TestRemoteTurnKeepsTheWorkersVerdict(t *testing.T) { c, _, task := remoteLeased(t) c.ReconcileFailureHandoff = 1 c.ReconcileHumanInput = func(context.Context, string) error { return down } verdict, _, err := c.RemoteTurn(context.Background(), task.ID, task.Lease.Epoch, orchestrator.TurnRotateNow, nil) if err != nil { t.Fatal(err) } if verdict != orchestrator.TurnRotateNow { t.Fatalf("verdict = %q, want the worker's own rotate_now", verdict) } }