Let an expired lease still commit the anchor it pushed
A release pushes the anchor first and commits second. When the lease expired in between, the commit could never land: the worker sent the epoch from w.leases, which the expiry replay had already deleted, and the coordinator refused a /handoff without a live owned lease. Run 10 lost a finished task this way, its work sitting in the worktree until retry_limit. The epoch now belongs to the release transaction, so it survives the lease. TaskReleased retains the ending epoch as Task.LastLeaseEpoch, and lateHandoffAccepted lets exactly that owner commit while the task is queued, unleased, and carrying no handoff of its own. A successor that has already re-leased the task holds the lease, so the late handoff loses. Store.Append's version fence settles the race, which is why the late path skips the version check the worker cannot satisfy. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
@@ -173,6 +173,12 @@ type Task struct {
|
||||
BlockedAt time.Time `json:"blocked_at,omitempty"`
|
||||
LastPaneID string `json:"last_pane_id,omitempty"`
|
||||
LastHarness string `json:"last_harness_id,omitempty"`
|
||||
// LastLeaseEpoch is the fencing token of the lease that most recently
|
||||
// ended. A worker can push its release anchor and only then discover the
|
||||
// lease expired; the finished work is durable in git but the commit can
|
||||
// never land. Retaining the epoch lets exactly that owner still commit
|
||||
// while the task sits unleased.
|
||||
LastLeaseEpoch string `json:"last_lease_epoch,omitempty"`
|
||||
PaneState string `json:"pane_state,omitempty"` // open, closed, unreachable, unknown
|
||||
LastSession SessionEvidence `json:"last_session,omitempty"`
|
||||
// Recovery state is part of the durable projection, never process-local
|
||||
|
||||
@@ -387,6 +387,9 @@ func (s *Store) apply(e domain.Event) error {
|
||||
case "TaskLaunchAcknowledged":
|
||||
t.LifecyclePhase = "started"
|
||||
case "TaskReleased":
|
||||
if t.Lease != nil {
|
||||
t.LastLeaseEpoch = t.Lease.Epoch
|
||||
}
|
||||
t.State = domain.StateQueued
|
||||
t.LifecyclePhase = "reclaimed"
|
||||
t.Lease = nil
|
||||
|
||||
@@ -663,3 +663,33 @@ func TestQuotaSinceReportsEmptyWindowAsKnownZero(t *testing.T) {
|
||||
}
|
||||
|
||||
const fiveHours = 5 * time.Hour
|
||||
|
||||
// TestExpiryRetainsLeaseEpoch covers the late-handoff fence: a worker that
|
||||
// pushed its release anchor and then lost the lease can only commit if the
|
||||
// projection still knows which epoch just ended.
|
||||
func TestExpiryRetainsLeaseEpoch(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(created("e1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := s.Tasks()[0].ID
|
||||
if _, err := s.Lease(id, "h1", time.Millisecond); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
leased, _ := s.Task(id)
|
||||
epoch := leased.Lease.Epoch
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
if _, err := s.ExpireLease(id, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := s.Task(id)
|
||||
if after.Lease != nil {
|
||||
t.Fatal("expired task still holds a lease")
|
||||
}
|
||||
if after.LastLeaseEpoch != epoch || epoch == "" {
|
||||
t.Fatalf("last lease epoch %q, want %q", after.LastLeaseEpoch, epoch)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user