orchestra/internal/operations/retry_test.go

package operations

import (
	"encoding/json"
	"errors"
	"testing"

	"orchestra/internal/authz"
	"orchestra/internal/domain"
	"orchestra/internal/store"
)

// exhausted builds a task that the router gave up on: three reclaims and a
// retry-limit failure, with a work phase and a research ref to prove retry
// preserves them.
func exhausted(t *testing.T) *store.Store {
	t.Helper()
	s, err := store.Open(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	created := mustJSONBytes(t, map[string]any{
		"source": "gitea:test-e2e", "external_id": "3", "project": "p",
		"title": "extend the healthcheck", "description": "do the thing",
		"acceptance": []string{"--help keeps working", "no arguments exits 0"},
	})
	if err := s.Append(domain.Event{ID: "task", TaskID: "task", Type: "TaskCreated", Version: 1, Payload: created, Surface: string(authz.System)}); err != nil {
		t.Fatal(err)
	}
	phase := mustJSONBytes(t, map[string]any{"phase": string(domain.WorkPhaseResearch), "research_ref": domain.Hash([]byte("research"))})
	task, _ := s.Task("task")
	if err := s.Append(domain.Event{ID: "phase", TaskID: "task", Type: domain.EventWorkPhaseChanged, Version: task.Version + 1, Payload: phase, Surface: string(authz.System)}); err != nil {
		t.Fatal(err)
	}
	for i := 0; i < 3; i++ {
		lease(t, s, "task")
		release(t, s)
	}
	task, _ = s.Task("task")
	failed := mustJSONBytes(t, map[string]any{"reason": "retry_limit", "attempts": task.Attempt, "failure_class": "prompt_not_submitted"})
	if err := s.Append(domain.Event{ID: "failed", TaskID: "task", Type: "TaskFailed", Version: task.Version + 1, Payload: failed, Surface: string(authz.System)}); err != nil {
		t.Fatal(err)
	}
	return s
}

func release(t *testing.T, s *store.Store) {
	t.Helper()
	task, _ := s.Task("task")
	p := mustJSONBytes(t, map[string]any{
		"reason": "lease_expired", "failure_class": "prompt_not_submitted",
		"harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch,
	})
	if err := s.Append(domain.Event{ID: domain.NewID(), TaskID: "task", Type: "TaskReleased", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
		t.Fatal(err)
	}
}

func TestRetryTaskRestoresTheAttemptBudget(t *testing.T) {
	s := exhausted(t)
	before, _ := s.Task("task")
	if before.State != domain.StateFailed || before.Attempt != 3 {
		t.Fatalf("setup: state=%s attempt=%d, want failed at 3", before.State, before.Attempt)
	}
	if _, err := RetryTask(s, authz.TUI, "task", "op-1"); err != nil {
		t.Fatal(err)
	}
	got, _ := s.Task("task")
	if got.State != domain.StateQueued || got.Attempt != 0 {
		t.Fatalf("state=%s attempt=%d, want queued at 0", got.State, got.Attempt)
	}
	if !got.NextRetryAt.IsZero() || got.FailureClass != "" || got.LastError != "" {
		t.Fatalf("retry left failure bookkeeping behind: %+v", got)
	}
	// The task is the same task, not a second one for the same issue.
	if got.ID != before.ID || got.Source != before.Source || got.ExternalID != before.ExternalID {
		t.Fatalf("identity changed: %s/%s/%s", got.ID, got.Source, got.ExternalID)
	}
	if got.Title != before.Title || got.Description != before.Description || len(got.Acceptance) != len(before.Acceptance) {
		t.Fatalf("goal or acceptance changed: %+v", got)
	}
	if got.WorkPhase != before.WorkPhase || got.ResearchRef != before.ResearchRef {
		t.Fatalf("phase artifacts changed: phase=%q research=%q", got.WorkPhase, got.ResearchRef)
	}
	// History is appended to, never rewritten.
	var failures, releases int
	for _, e := range s.Events(0) {
		switch e.Type {
		case "TaskFailed":
			failures++
		case "TaskReleased":
			releases++
		}
	}
	if failures != 1 || releases != 3 {
		t.Fatalf("history lost: %d failures, %d releases", failures, releases)
	}
}

func TestRetryTaskRefusesATaskThatIsNotTerminal(t *testing.T) {
	s := exhausted(t)
	if _, err := RetryTask(s, authz.TUI, "task", "op-1"); err != nil {
		t.Fatal(err)
	}
	// Queued: nothing to recover.
	if _, err := RetryTask(s, authz.TUI, "task", "op-2"); !errors.Is(err, ErrNotRetryable) {
		t.Fatalf("err=%v, want ErrNotRetryable for a queued task", err)
	}
	lease(t, s, "task")
	if _, err := RetryTask(s, authz.TUI, "task", "op-3"); !errors.Is(err, ErrNotRetryable) {
		t.Fatalf("err=%v, want ErrNotRetryable for a leased task", err)
	}
}

// A failure that was not the retry budget running out needs a decision about
// the failure, not another identical attempt.
func TestRetryTaskRefusesANonRetryFailure(t *testing.T) {
	s, err := store.Open(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	created := mustJSONBytes(t, map[string]any{"source": "test", "external_id": "1", "project": "p", "title": "x"})
	if err := s.Append(domain.Event{ID: "task", TaskID: "task", Type: "TaskCreated", Version: 1, Payload: created, Surface: string(authz.System)}); err != nil {
		t.Fatal(err)
	}
	task, _ := s.Task("task")
	p := mustJSONBytes(t, map[string]any{"reason": "unrecoverable"})
	if err := s.Append(domain.Event{ID: "failed", TaskID: "task", Type: "TaskFailed", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
		t.Fatal(err)
	}
	if _, err := RetryTask(s, authz.TUI, "task", "op-1"); !errors.Is(err, ErrNotRetryable) {
		t.Fatalf("err=%v, want ErrNotRetryable", err)
	}
}

// A gated surface holds an agent credential. Reviving a terminal task is an
// operator decision.
func TestRetryTaskRefusesAGatedSurface(t *testing.T) {
	s := exhausted(t)
	for _, surface := range []authz.Surface{authz.Agent, authz.MCP, authz.Maven, authz.Ntfy} {
		if _, err := RetryTask(s, surface, "task", "op-1"); err == nil {
			t.Fatalf("surface %s retried a terminal task", surface)
		}
	}
	if got, _ := s.Task("task"); got.State != domain.StateFailed {
		t.Fatalf("state=%s, want the task untouched", got.State)
	}
}

// Idempotency: a repeated request under the same operation id must not reset
// an attempt that has since started running.
func TestRetryTaskIsIdempotentPerOperation(t *testing.T) {
	s := exhausted(t)
	first, err := RetryTask(s, authz.TUI, "task", "op-1")
	if err != nil {
		t.Fatal(err)
	}
	lease(t, s, "task")
	again, err := RetryTask(s, authz.TUI, "task", "op-1")
	if err != nil {
		t.Fatalf("duplicate retry returned %v, want the recorded event", err)
	}
	if again.ID != first.ID {
		t.Fatalf("event %s, want the original %s", again.ID, first.ID)
	}
	got, _ := s.Task("task")
	if got.State != domain.StateLeased || got.Lease == nil {
		t.Fatalf("duplicate retry disturbed a running attempt: %+v", got)
	}
}

// Exhausting the budget a second time is retryable again, explicitly.
func TestRetryTaskWorksAfterASecondExhaustion(t *testing.T) {
	s := exhausted(t)
	if _, err := RetryTask(s, authz.TUI, "task", "op-1"); err != nil {
		t.Fatal(err)
	}
	for i := 0; i < 3; i++ {
		lease(t, s, "task")
		release(t, s)
	}
	task, _ := s.Task("task")
	failed := mustJSONBytes(t, map[string]any{"reason": "retry_limit", "attempts": task.Attempt})
	if err := s.Append(domain.Event{ID: "failed-2", TaskID: "task", Type: "TaskFailed", Version: task.Version + 1, Payload: failed, Surface: string(authz.System)}); err != nil {
		t.Fatal(err)
	}
	if _, err := RetryTask(s, authz.TUI, "task", "op-2"); err != nil {
		t.Fatal(err)
	}
	got, _ := s.Task("task")
	if got.State != domain.StateQueued || got.Attempt != 0 {
		t.Fatalf("state=%s attempt=%d, want queued at 0", got.State, got.Attempt)
	}
	// The correction names the second failure, not the first.
	var corrects string
	for _, e := range s.Events(0) {
		if e.Type != "TaskCorrected" {
			continue
		}
		var p struct {
			Corrects    string `json:"corrects"`
			OperationID string `json:"operation_id"`
		}
		if json.Unmarshal(e.Payload, &p) == nil && p.OperationID == "op-2" {
			corrects = p.Corrects
		}
	}
	if corrects != "failed-2" {
		t.Fatalf("corrects=%q, want failed-2", corrects)
	}
}