package operations import ( "encoding/json" "errors" "testing" "orchestra/internal/authz" "orchestra/internal/domain" "orchestra/internal/store" ) // exhausted builds a task that the router gave up on: three reclaims and a // retry-limit failure, with a work phase and a research ref to prove retry // preserves them. func exhausted(t *testing.T) *store.Store { t.Helper() s, err := store.Open(t.TempDir()) if err != nil { t.Fatal(err) } created := mustJSONBytes(t, map[string]any{ "source": "gitea:test-e2e", "external_id": "3", "project": "p", "title": "extend the healthcheck", "description": "do the thing", "acceptance": []string{"--help keeps working", "no arguments exits 0"}, }) if err := s.Append(domain.Event{ID: "task", TaskID: "task", Type: "TaskCreated", Version: 1, Payload: created, Surface: string(authz.System)}); err != nil { t.Fatal(err) } phase := mustJSONBytes(t, map[string]any{"phase": string(domain.WorkPhaseResearch), "research_ref": domain.Hash([]byte("research"))}) task, _ := s.Task("task") if err := s.Append(domain.Event{ID: "phase", TaskID: "task", Type: domain.EventWorkPhaseChanged, Version: task.Version + 1, Payload: phase, Surface: string(authz.System)}); err != nil { t.Fatal(err) } for i := 0; i < 3; i++ { lease(t, s, "task") release(t, s) } task, _ = s.Task("task") failed := mustJSONBytes(t, map[string]any{"reason": "retry_limit", "attempts": task.Attempt, "failure_class": "prompt_not_submitted"}) if err := s.Append(domain.Event{ID: "failed", TaskID: "task", Type: "TaskFailed", Version: task.Version + 1, Payload: failed, Surface: string(authz.System)}); err != nil { t.Fatal(err) } return s } func release(t *testing.T, s *store.Store) { t.Helper() task, _ := s.Task("task") p := mustJSONBytes(t, map[string]any{ "reason": "lease_expired", "failure_class": "prompt_not_submitted", "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, }) if err := s.Append(domain.Event{ID: domain.NewID(), TaskID: "task", Type: "TaskReleased", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil { t.Fatal(err) } } func TestRetryTaskRestoresTheAttemptBudget(t *testing.T) { s := exhausted(t) before, _ := s.Task("task") if before.State != domain.StateFailed || before.Attempt != 3 { t.Fatalf("setup: state=%s attempt=%d, want failed at 3", before.State, before.Attempt) } if _, err := RetryTask(s, authz.TUI, "task", "op-1"); err != nil { t.Fatal(err) } got, _ := s.Task("task") if got.State != domain.StateQueued || got.Attempt != 0 { t.Fatalf("state=%s attempt=%d, want queued at 0", got.State, got.Attempt) } if !got.NextRetryAt.IsZero() || got.FailureClass != "" || got.LastError != "" { t.Fatalf("retry left failure bookkeeping behind: %+v", got) } // The task is the same task, not a second one for the same issue. if got.ID != before.ID || got.Source != before.Source || got.ExternalID != before.ExternalID { t.Fatalf("identity changed: %s/%s/%s", got.ID, got.Source, got.ExternalID) } if got.Title != before.Title || got.Description != before.Description || len(got.Acceptance) != len(before.Acceptance) { t.Fatalf("goal or acceptance changed: %+v", got) } if got.WorkPhase != before.WorkPhase || got.ResearchRef != before.ResearchRef { t.Fatalf("phase artifacts changed: phase=%q research=%q", got.WorkPhase, got.ResearchRef) } // History is appended to, never rewritten. var failures, releases int for _, e := range s.Events(0) { switch e.Type { case "TaskFailed": failures++ case "TaskReleased": releases++ } } if failures != 1 || releases != 3 { t.Fatalf("history lost: %d failures, %d releases", failures, releases) } } func TestRetryTaskRefusesATaskThatIsNotTerminal(t *testing.T) { s := exhausted(t) if _, err := RetryTask(s, authz.TUI, "task", "op-1"); err != nil { t.Fatal(err) } // Queued: nothing to recover. if _, err := RetryTask(s, authz.TUI, "task", "op-2"); !errors.Is(err, ErrNotRetryable) { t.Fatalf("err=%v, want ErrNotRetryable for a queued task", err) } lease(t, s, "task") if _, err := RetryTask(s, authz.TUI, "task", "op-3"); !errors.Is(err, ErrNotRetryable) { t.Fatalf("err=%v, want ErrNotRetryable for a leased task", err) } } // A failure that was not the retry budget running out needs a decision about // the failure, not another identical attempt. func TestRetryTaskRefusesANonRetryFailure(t *testing.T) { s, err := store.Open(t.TempDir()) if err != nil { t.Fatal(err) } created := mustJSONBytes(t, map[string]any{"source": "test", "external_id": "1", "project": "p", "title": "x"}) if err := s.Append(domain.Event{ID: "task", TaskID: "task", Type: "TaskCreated", Version: 1, Payload: created, Surface: string(authz.System)}); err != nil { t.Fatal(err) } task, _ := s.Task("task") p := mustJSONBytes(t, map[string]any{"reason": "unrecoverable"}) if err := s.Append(domain.Event{ID: "failed", TaskID: "task", Type: "TaskFailed", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil { t.Fatal(err) } if _, err := RetryTask(s, authz.TUI, "task", "op-1"); !errors.Is(err, ErrNotRetryable) { t.Fatalf("err=%v, want ErrNotRetryable", err) } } // A gated surface holds an agent credential. Reviving a terminal task is an // operator decision. func TestRetryTaskRefusesAGatedSurface(t *testing.T) { s := exhausted(t) for _, surface := range []authz.Surface{authz.Agent, authz.MCP, authz.Maven, authz.Ntfy} { if _, err := RetryTask(s, surface, "task", "op-1"); err == nil { t.Fatalf("surface %s retried a terminal task", surface) } } if got, _ := s.Task("task"); got.State != domain.StateFailed { t.Fatalf("state=%s, want the task untouched", got.State) } } // Idempotency: a repeated request under the same operation id must not reset // an attempt that has since started running. func TestRetryTaskIsIdempotentPerOperation(t *testing.T) { s := exhausted(t) first, err := RetryTask(s, authz.TUI, "task", "op-1") if err != nil { t.Fatal(err) } lease(t, s, "task") again, err := RetryTask(s, authz.TUI, "task", "op-1") if err != nil { t.Fatalf("duplicate retry returned %v, want the recorded event", err) } if again.ID != first.ID { t.Fatalf("event %s, want the original %s", again.ID, first.ID) } got, _ := s.Task("task") if got.State != domain.StateLeased || got.Lease == nil { t.Fatalf("duplicate retry disturbed a running attempt: %+v", got) } } // Exhausting the budget a second time is retryable again, explicitly. func TestRetryTaskWorksAfterASecondExhaustion(t *testing.T) { s := exhausted(t) if _, err := RetryTask(s, authz.TUI, "task", "op-1"); err != nil { t.Fatal(err) } for i := 0; i < 3; i++ { lease(t, s, "task") release(t, s) } task, _ := s.Task("task") failed := mustJSONBytes(t, map[string]any{"reason": "retry_limit", "attempts": task.Attempt}) if err := s.Append(domain.Event{ID: "failed-2", TaskID: "task", Type: "TaskFailed", Version: task.Version + 1, Payload: failed, Surface: string(authz.System)}); err != nil { t.Fatal(err) } if _, err := RetryTask(s, authz.TUI, "task", "op-2"); err != nil { t.Fatal(err) } got, _ := s.Task("task") if got.State != domain.StateQueued || got.Attempt != 0 { t.Fatalf("state=%s attempt=%d, want queued at 0", got.State, got.Attempt) } // The correction names the second failure, not the first. var corrects string for _, e := range s.Events(0) { if e.Type != "TaskCorrected" { continue } var p struct { Corrects string `json:"corrects"` OperationID string `json:"operation_id"` } if json.Unmarshal(e.Payload, &p) == nil && p.OperationID == "op-2" { corrects = p.Corrects } } if corrects != "failed-2" { t.Fatalf("corrects=%q, want failed-2", corrects) } }