Settle a release transaction deterministically in every case

F60, and the general rule F58 and F59 were reaching for one case at a
time: a transaction must settle or be abandoned deterministically, and
must never spin on an answer that cannot change.

Terminal now means failed or completed. Both drop the transaction and
free the session; nothing will ever lease either task again.

Blocked keeps the transaction, because a reopen returns the task to the
queue and that exact owner can still commit. TaskBlocked therefore
retains the ending epoch the way TaskReleased already did, or the
late-handoff path would have nothing to fence against after the reopen.

A refusal parks the commit instead of retrying every five seconds. It
is the coordinator's answer about who owns the task, so it stays true
until an event about that task arrives, and any such event un-parks it.
A reopen arrives as TaskCorrected, so the rule cannot be a list of
event types. Backoff runs 30s to a 5 minute cap.

A transport failure is not an answer and keeps retrying at once. That
distinction is the whole reason the park keys on a 4xx StatusError
rather than on any error at all.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1

cmd/orchestra-worker/main_test.go

@@ -1327,3 +1327,120 @@ func TestObservationRingEvictsLeastRecentlySeen(t *testing.T) {
 		t.Fatal("least recently seen entry survived")
 	}
 }
+
+// F60. A refusal is an answer about the task, not a transport failure, and it
+// stays true until something about that task changes. Run 10's blocked task
+// asked 5,000 times over seven hours and got the same 409 every time.
+func TestRefusedCommitParksUntilSomethingChanges(t *testing.T) {
+	var commits int
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/handoff"):
+			commits++
+			http.Error(w, "lease not owned", http.StatusConflict)
+		case strings.HasSuffix(r.URL.Path, "/events"):
+			_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"c","type":"TaskCorrected","task_id":"t","version":9,"payload":{"state":"queued"},"surface":"web"}]}`))
+		default:
+			w.WriteHeader(http.StatusNoContent)
+		}
+	}))
+	defer s.Close()
+	tx := releaseTransaction{ID: "tx", Phase: "anchor_pushed", Ref: "sha256:abc", AnchorSHA: "abc", LeaseEpoch: "e1"}
+	w := &worker{
+		api:         federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
+		harnessID:   "h",
+		tasks:       map[string]domain.Task{"t": {ID: "t"}},
+		sessions:    map[string]herdr.Session{"t": {PaneID: "pane"}},
+		leases:      map[string]lease{},
+		releases:    map[string]releaseTransaction{"t": tx},
+		quarantined: map[string]bool{},
+		statePath:   t.TempDir() + "/state.json",
+		hard:        .75,
+	}
+	w.advanceRelease(context.Background(), "t", w.sessions["t"])
+	w.advanceRelease(context.Background(), "t", w.sessions["t"])
+	if commits != 1 {
+		t.Fatalf("refused commit retried %d times without waiting", commits)
+	}
+	if w.releases["t"].NextAttemptAt.IsZero() {
+		t.Fatal("refused commit was not parked")
+	}
+	// A reopen arrives as TaskCorrected. Any event about the task is the
+	// change the parked commit was waiting for, so the same tick retries it
+	// and, still refused, parks it again.
+	if err := w.once(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if commits != 2 {
+		t.Fatalf("an event about the task did not un-park its commit, commits=%d", commits)
+	}
+	if w.releases["t"].NextAttemptAt.IsZero() {
+		t.Fatal("the second refusal did not park it again")
+	}
+}
+
+// The opposite case, and the one a backoff must not break: the coordinator is
+// unreachable or broken rather than answering. That says nothing about who
+// owns the task, so it has to retry at once.
+func TestTransientCommitFailureKeepsRetryingAtOnce(t *testing.T) {
+	var commits int
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/handoff") {
+			commits++
+			http.Error(w, "upstream unavailable", http.StatusBadGateway)
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer s.Close()
+	w := &worker{
+		api:         federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
+		harnessID:   "h",
+		tasks:       map[string]domain.Task{"t": {ID: "t"}},
+		sessions:    map[string]herdr.Session{"t": {PaneID: "pane"}},
+		leases:      map[string]lease{},
+		releases:    map[string]releaseTransaction{"t": {ID: "tx", Phase: "anchor_pushed", Ref: "sha256:abc", AnchorSHA: "abc", LeaseEpoch: "e1"}},
+		quarantined: map[string]bool{},
+		statePath:   t.TempDir() + "/state.json",
+		hard:        .75,
+	}
+	w.advanceRelease(context.Background(), "t", w.sessions["t"])
+	w.advanceRelease(context.Background(), "t", w.sessions["t"])
+	if commits != 2 {
+		t.Fatalf("transport failure was parked like a refusal, commits=%d", commits)
+	}
+	if !w.releases["t"].NextAttemptAt.IsZero() {
+		t.Fatal("transport failure must not park the transaction")
+	}
+}
+
+// Completion is terminal for a release transaction just as failure is. The
+// task is done; nothing will ever lease it again to pick the anchor up.
+func TestCompletedTaskDropsItsReleaseTransaction(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/events") {
+			_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"c","type":"TaskCompleted","task_id":"t","version":9,"payload":{"report_ref":"sha256:r"},"surface":"system"}]}`))
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer s.Close()
+	w := &worker{
+		api:         federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
+		harnessID:   "h",
+		backend:     deadTmuxBackend(t),
+		tasks:       map[string]domain.Task{"t": {ID: "t"}},
+		sessions:    map[string]herdr.Session{"t": {PaneID: "pane"}},
+		leases:      map[string]lease{},
+		releases:    map[string]releaseTransaction{"t": {ID: "tx", Phase: "anchor_pushed", Ref: "sha256:abc", AnchorSHA: "abc", LeaseEpoch: "e1"}},
+		quarantined: map[string]bool{},
+		statePath:   t.TempDir() + "/state.json",
+		hard:        .75,
+	}
+	if err := w.once(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if len(w.releases) != 0 || len(w.sessions) != 0 {
+		t.Fatalf("completed task kept its release: releases=%v sessions=%v", w.releases, w.sessions)
+	}
+}