Let an expired lease still commit the anchor it pushed

A release pushes the anchor first and commits second. When the lease
expired in between, the commit could never land: the worker sent the
epoch from w.leases, which the expiry replay had already deleted, and
the coordinator refused a /handoff without a live owned lease. Run 10
lost a finished task this way, its work sitting in the worktree until
retry_limit.

The epoch now belongs to the release transaction, so it survives the
lease. TaskReleased retains the ending epoch as Task.LastLeaseEpoch,
and lateHandoffAccepted lets exactly that owner commit while the task
is queued, unleased, and carrying no handoff of its own. A successor
that has already re-leased the task holds the lease, so the late
handoff loses. Store.Append's version fence settles the race, which is
why the late path skips the version check the worker cannot satisfy.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
2026-08-28 20:57:19 +04:00
parent 8850de3782
commit 6565b9fce2
7 changed files with 120 additions and 5 deletions
+18 -2
View File
@@ -162,6 +162,19 @@ func validateLocalMachine(rr registry.Registry, localMachine string) error {
return nil
}
// lateHandoffAccepted reports whether a release may still commit after its
// lease expired. A worker pushes the anchor first and commits second; when the
// lease dies in between, the finished work is durable in git and unreachable
// forever (run 10 lost a whole task that way). The expired owner is the only
// caller that can present the ended lease's epoch, and this fires only while
// the task is unleased and its reclaim recorded no handoff, so it can never
// overwrite a successor's work. Store.Append's version fence settles a race
// with a concurrent re-lease.
func lateHandoffAccepted(t domain.Task, leaseEpoch string) bool {
return t.State == domain.StateQueued && t.Lease == nil && t.HandoffRef == "" &&
leaseEpoch != "" && leaseEpoch == t.LastLeaseEpoch
}
func main() {
dir := os.Getenv("ORCHESTRA_DATA")
if dir == "" {
@@ -1428,7 +1441,8 @@ func main() {
w.WriteHeader(http.StatusNoContent)
return
}
if !ownedLease {
lateHandoff := !ownedLease && strings.HasSuffix(r.URL.Path, "/handoff") && lateHandoffAccepted(t, b.LeaseEpoch)
if !ownedLease && !lateHandoff {
http.Error(w, "lease not owned", 409)
return
}
@@ -1656,7 +1670,9 @@ func main() {
http.Error(w, "anchor_sha required", 400)
return
}
if b.TransactionID == "" || b.ExpectedVersion != t.Version {
// A late handoff cannot know the version: expiry bumped it after the
// worker read it. Its fence is the epoch plus the append.
if b.TransactionID == "" || (!lateHandoff && b.ExpectedVersion != t.Version) {
http.Error(w, "release transaction and current lease version required", http.StatusConflict)
return
}
+20
View File
@@ -6,6 +6,7 @@ import (
"testing"
"time"
"orchestra/internal/domain"
"orchestra/internal/registry"
)
@@ -89,3 +90,22 @@ func TestTmuxRegistryRequiresMachineIdentityEvenOnOneMachine(t *testing.T) {
t.Fatal(err)
}
}
func TestLateHandoffAcceptedOnlyFromTheExpiredOwner(t *testing.T) {
expired := domain.Task{State: domain.StateQueued, LastLeaseEpoch: "ep1"}
if !lateHandoffAccepted(expired, "ep1") {
t.Fatal("the expired owner must still commit its pushed anchor")
}
for name, tk := range map[string]domain.Task{
"re-leased": {State: domain.StateLeased, Lease: &domain.Lease{Epoch: "ep2"}, LastLeaseEpoch: "ep1"},
"already handed off": {State: domain.StateQueued, LastLeaseEpoch: "ep1", HandoffRef: "sha256:x"},
"blocked": {State: domain.StateBlocked, LastLeaseEpoch: "ep1"},
} {
if lateHandoffAccepted(tk, "ep1") {
t.Fatalf("%s must refuse a late handoff", name)
}
}
if lateHandoffAccepted(expired, "ep2") || lateHandoffAccepted(domain.Task{State: domain.StateQueued}, "") {
t.Fatal("a foreign or empty epoch must refuse")
}
}