Give an operator one way to retry a terminal task

A task that reached the router's MaxAttempts was permanently terminal.
TaskReleased only ever increments Attempt, TaskCorrected could not touch it,
and no HTTP route emitted a correction at all. The only way to work an
exhausted issue again was to invent a second task for it, which defeats
(source, external_id) dedupe and abandons the task's own history.

POST /v1/tasks/{id}/retry, full-control surfaces only. It requires the task
to be failed, unleased, and failed with reason retry_limit: restoring a retry
budget is not an answer to a failure that was not the budget running out. The
effect is one TaskCorrected naming that failure, setting state queued and
attempt 0 and clearing next_retry_at, failure_class and last_error. Task id,
source pair, goal, acceptance, decisions, work phase and artifact refs all
stay, and the original failure events stay in the log.

operation_id is required and makes the call idempotent, so a repeated request
cannot reset an attempt that has since started running.

This is RetryTask, not a generic correction endpoint: arbitrary task mutation
over HTTP is a different and much larger authority. It also does not address
F9, which is an operator releasing a lease someone else owns.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-27 13:07:48 +04:00
parent f54fb0036d
commit edff021265
6 changed files with 446 additions and 1 deletions
+67
View File
@@ -490,3 +490,70 @@ func backoffPayload(at time.Time) []byte {
b, _ := json.Marshal(map[string]any{"next_retry_at": at.UTC().Format(time.RFC3339Nano)})
return b
}
// F19: a retry-exhausted task the operator revived must be leasable again.
// Before RetryTask nothing lowered Attempt, so the router refailed the task on
// sight and an exhausted issue could never be worked again.
func TestOperatorRetryMakesAnExhaustedTaskLeasableAgain(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
r, err := registry.New(registry.Config{
Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}},
Machines: []registry.Machine{{ID: "m", Address: "unused"}},
Herdrs: []registry.Herdr{{ID: "h", MachineID: "m", Concurrency: 1}},
})
if err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(map[string]any{"source": "test", "external_id": "a", "project": "p"})
if err := s.Append(domain.Event{ID: "a", TaskID: "a", Type: "TaskCreated", Version: 1, Payload: b, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
// Each reclaim schedules a backoff, and the retry-limit check sits behind
// it, so the clock has to move for the router to reach the task at all.
clock := time.Now()
rt := Router{Store: s, Registry: r, Reachability: reachable{}, Retry: RetryPolicy{MaxAttempts: 3}, Now: func() time.Time { return clock }}
for i := 0; i < 3; i++ {
clock = clock.Add(time.Hour)
if _, err := rt.AssignPending(); err != nil {
t.Fatal(err)
}
task, _ := s.Task("a")
if task.Lease == nil {
t.Fatalf("round %d: task not leased", i)
}
p, _ := json.Marshal(map[string]any{
"reason": "lease_expired", "failure_class": "prompt_not_submitted",
"harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch,
})
if err := s.Append(domain.Event{ID: domain.NewID(), TaskID: "a", Type: "TaskReleased", Version: task.Version + 1, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
}
clock = clock.Add(time.Hour)
if _, err := rt.AssignPending(); err != nil {
t.Fatal(err)
}
task, _ := s.Task("a")
if task.State != domain.StateFailed {
t.Fatalf("state=%s, want failed at the retry limit", task.State)
}
// The correction operations.RetryTask appends. Asserted here against the
// router, because the router is what refused the revived task.
fix, _ := json.Marshal(map[string]any{
"corrects": "a", "state": string(domain.StateQueued),
"attempt": 0, "next_retry_at": "", "failure_class": "",
})
if err := s.Append(domain.Event{ID: domain.NewID(), TaskID: "a", Type: "TaskCorrected", Version: task.Version + 1, Payload: fix, Surface: string(authz.TUI)}); err != nil {
t.Fatal(err)
}
if _, err := rt.AssignPending(); err != nil {
t.Fatal(err)
}
if got, _ := s.Task("a"); got.State != domain.StateLeased {
t.Fatalf("state=%s, want the revived task leased", got.State)
}
}