Drop a release transaction when its task fails
The supersession rule fires on TaskLeased, and a failed task is never leased again. Run 12's rig task reached retry_limit still holding a transaction whose commit the coordinator refuses permanently, so it kept asking every five seconds with nothing that could ever change. Terminal means terminal: TaskFailed now drops the transaction and quarantines the session even when the anchor was pushed. Blocked keeps the old rule, because a reopen still produces a successor that can pick the anchor up. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
@@ -1395,8 +1395,12 @@ func (w *worker) once(ctx context.Context) error {
|
||||
// mapping protects nothing. F30: a transaction stuck at "prepared"
|
||||
// held the session forever once its pane was gone, health() kept
|
||||
// reporting ActiveTask, and the harness never leased again.
|
||||
// A failed task is terminal: no successor will ever lease it, so
|
||||
// its anchor protects nothing and its transaction can only retry
|
||||
// a refusal forever. Blocked is different, because a reopen still
|
||||
// produces a successor.
|
||||
tx, releasing := w.releases[e.TaskID]
|
||||
if !releasing || tx.Ref == "" {
|
||||
if !releasing || tx.Ref == "" || e.Type == "TaskFailed" {
|
||||
if releasing {
|
||||
delete(w.releases, e.TaskID)
|
||||
}
|
||||
|
||||
@@ -1235,3 +1235,36 @@ func TestOwnPickupLeaseKeepsTheReleaseTransaction(t *testing.T) {
|
||||
t.Fatal("pickup lease dropped its own release transaction")
|
||||
}
|
||||
}
|
||||
|
||||
// A failed task never comes back, so its release transaction can only retry a
|
||||
// permanent refusal. Live on run 12: task 06G4KENHXY12M5BNC5TXAF3MXR reached
|
||||
// retry_limit with a superseded transaction still asking every five seconds.
|
||||
func TestFailedTaskDropsItsReleaseTransaction(t *testing.T) {
|
||||
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v1/federation/events":
|
||||
_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"f","type":"TaskFailed","task_id":"t","version":9,"payload":{"reason":"retry_limit"},"surface":"system"}]}`))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
}))
|
||||
defer s.Close()
|
||||
w := &worker{
|
||||
api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
|
||||
harnessID: "h",
|
||||
backend: deadTmuxBackend(t),
|
||||
tasks: map[string]domain.Task{"t": {ID: "t"}},
|
||||
sessions: map[string]herdr.Session{"t": {PaneID: "pane"}},
|
||||
leases: map[string]lease{},
|
||||
releases: map[string]releaseTransaction{"t": {ID: "tx", Phase: "anchor_pushed", Ref: "sha256:abc", AnchorSHA: "abc", LeaseEpoch: "e1"}},
|
||||
quarantined: map[string]bool{},
|
||||
statePath: t.TempDir() + "/state.json",
|
||||
hard: .75,
|
||||
}
|
||||
if err := w.once(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.releases) != 0 || len(w.sessions) != 0 {
|
||||
t.Fatalf("terminal task kept its release: releases=%v sessions=%v", w.releases, w.sessions)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user