Drop a release transaction when its task fails
The supersession rule fires on TaskLeased, and a failed task is never leased again. Run 12's rig task reached retry_limit still holding a transaction whose commit the coordinator refuses permanently, so it kept asking every five seconds with nothing that could ever change. Terminal means terminal: TaskFailed now drops the transaction and quarantines the session even when the anchor was pushed. Blocked keeps the old rule, because a reopen still produces a successor that can pick the anchor up. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
@@ -1395,8 +1395,12 @@ func (w *worker) once(ctx context.Context) error {
|
|||||||
// mapping protects nothing. F30: a transaction stuck at "prepared"
|
// mapping protects nothing. F30: a transaction stuck at "prepared"
|
||||||
// held the session forever once its pane was gone, health() kept
|
// held the session forever once its pane was gone, health() kept
|
||||||
// reporting ActiveTask, and the harness never leased again.
|
// reporting ActiveTask, and the harness never leased again.
|
||||||
|
// A failed task is terminal: no successor will ever lease it, so
|
||||||
|
// its anchor protects nothing and its transaction can only retry
|
||||||
|
// a refusal forever. Blocked is different, because a reopen still
|
||||||
|
// produces a successor.
|
||||||
tx, releasing := w.releases[e.TaskID]
|
tx, releasing := w.releases[e.TaskID]
|
||||||
if !releasing || tx.Ref == "" {
|
if !releasing || tx.Ref == "" || e.Type == "TaskFailed" {
|
||||||
if releasing {
|
if releasing {
|
||||||
delete(w.releases, e.TaskID)
|
delete(w.releases, e.TaskID)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1235,3 +1235,36 @@ func TestOwnPickupLeaseKeepsTheReleaseTransaction(t *testing.T) {
|
|||||||
t.Fatal("pickup lease dropped its own release transaction")
|
t.Fatal("pickup lease dropped its own release transaction")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A failed task never comes back, so its release transaction can only retry a
|
||||||
|
// permanent refusal. Live on run 12: task 06G4KENHXY12M5BNC5TXAF3MXR reached
|
||||||
|
// retry_limit with a superseded transaction still asking every five seconds.
|
||||||
|
func TestFailedTaskDropsItsReleaseTransaction(t *testing.T) {
|
||||||
|
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/v1/federation/events":
|
||||||
|
_, _ = w.Write([]byte(`{"cursor":0,"events":[{"seq":1,"id":"f","type":"TaskFailed","task_id":"t","version":9,"payload":{"reason":"retry_limit"},"surface":"system"}]}`))
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer s.Close()
|
||||||
|
w := &worker{
|
||||||
|
api: federation.Client{BaseURL: s.URL, WorkerID: "h", Token: "t"},
|
||||||
|
harnessID: "h",
|
||||||
|
backend: deadTmuxBackend(t),
|
||||||
|
tasks: map[string]domain.Task{"t": {ID: "t"}},
|
||||||
|
sessions: map[string]herdr.Session{"t": {PaneID: "pane"}},
|
||||||
|
leases: map[string]lease{},
|
||||||
|
releases: map[string]releaseTransaction{"t": {ID: "tx", Phase: "anchor_pushed", Ref: "sha256:abc", AnchorSHA: "abc", LeaseEpoch: "e1"}},
|
||||||
|
quarantined: map[string]bool{},
|
||||||
|
statePath: t.TempDir() + "/state.json",
|
||||||
|
hard: .75,
|
||||||
|
}
|
||||||
|
if err := w.once(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(w.releases) != 0 || len(w.sessions) != 0 {
|
||||||
|
t.Fatalf("terminal task kept its release: releases=%v sessions=%v", w.releases, w.sessions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user