From 8e37989526d8ea14088872138e32438b0df061c3 Mon Sep 17 00:00:00 2001 From: kami Date: Fri, 28 Aug 2026 23:14:09 +0400 Subject: [PATCH] Drop a release transaction when its task fails The supersession rule fires on TaskLeased, and a failed task is never leased again. Run 12's rig task reached retry_limit still holding a transaction whose commit the coordinator refuses permanently, so it kept asking every five seconds with nothing that could ever change. Terminal means terminal: TaskFailed now drops the transaction and quarantines the session even when the anchor was pushed. Blocked keeps the old rule, because a reopen still produces a successor that can pick the anchor up. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1 --- cmd/orchestra-worker/main.go | 6 +++++- cmd/orchestra-worker/main_test.go | 33 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/cmd/orchestra-worker/main.go b/cmd/orchestra-worker/main.go index f4477de..2617992 100644 --- a/cmd/orchestra-worker/main.go +++ b/cmd/orchestra-worker/main.go @@ -1395,8 +1395,12 @@ func (w *worker) once(ctx context.Context) error { // mapping protects nothing. F30: a transaction stuck at "prepared" // held the session forever once its pane was gone, health() kept // reporting ActiveTask, and the harness never leased again. + // A failed task is terminal: no successor will ever lease it, so + // its anchor protects nothing and its transaction can only retry + // a refusal forever. Blocked is different, because a reopen still + // produces a successor. tx, releasing := w.releases[e.TaskID] - if !releasing || tx.Ref == "" { + if !releasing || tx.Ref == "" || e.Type == "TaskFailed" { if releasing { delete(w.releases, e.TaskID) } diff --git a/cmd/orchestra-worker/main_test.go b/cmd/orchestra-worker/main_test.go index 8b9b861..c5d6e00 100644 --- a/cmd/orchestra-worker/main_test.go +++ b/cmd/orchestra-worker/main_test.go @@ -1235,3 +1235,36 @@ func TestOwnPickupLeaseKeepsTheReleaseTransaction(t *testing.T) { t.Fatal("pickup lease dropped its own release transaction") } } + +// A failed task never comes back, so its release transaction can only retry a +// permanent refusal. Live on run 12: task 06G4KENHXY12M5BNC5TXAF3MXR reached +// retry_limit with a superseded transaction still asking every five seconds. +func TestFailedTaskDropsItsReleaseTransaction(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/federation/events": + _, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"f","type":"TaskFailed","task_id":"t","version":9,"payload":{"reason":"retry_limit"},"surface":"system"}]}`)) + default: + w.WriteHeader(http.StatusNoContent) + } + })) + defer s.Close() + w := &worker{ + api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"}, + harnessID: "h", + backend: deadTmuxBackend(t), + tasks: map[string]domain.Task{"t": {ID: "t"}}, + sessions: map[string]herdr.Session{"t": {PaneID: "pane"}}, + leases: map[string]lease{}, + releases: map[string]releaseTransaction{"t": {ID: "tx", Phase: "anchor_pushed", Ref: "sha256:abc", AnchorSHA: "abc", LeaseEpoch: "e1"}}, + quarantined: map[string]bool{}, + statePath: t.TempDir() + "/state.json", + hard: .75, + } + if err := w.once(context.Background()); err != nil { + t.Fatal(err) + } + if len(w.releases) != 0 || len(w.sessions) != 0 { + t.Fatalf("terminal task kept its release: releases=%v sessions=%v", w.releases, w.sessions) + } +}