Let an expired lease still commit the anchor it pushed
A release pushes the anchor first and commits second. When the lease expired in between, the commit could never land: the worker sent the epoch from w.leases, which the expiry replay had already deleted, and the coordinator refused a /handoff without a live owned lease. Run 10 lost a finished task this way, its work sitting in the worktree until retry_limit. The epoch now belongs to the release transaction, so it survives the lease. TaskReleased retains the ending epoch as Task.LastLeaseEpoch, and lateHandoffAccepted lets exactly that owner commit while the task is queued, unleased, and carrying no handoff of its own. A successor that has already re-leased the task holds the lease, so the late handoff loses. Store.Append's version fence settles the race, which is why the late path skips the version check the worker cannot satisfy. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
+18
-2
@@ -162,6 +162,19 @@ func validateLocalMachine(rr registry.Registry, localMachine string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// lateHandoffAccepted reports whether a release may still commit after its
|
||||
// lease expired. A worker pushes the anchor first and commits second; when the
|
||||
// lease dies in between, the finished work is durable in git and unreachable
|
||||
// forever (run 10 lost a whole task that way). The expired owner is the only
|
||||
// caller that can present the ended lease's epoch, and this fires only while
|
||||
// the task is unleased and its reclaim recorded no handoff, so it can never
|
||||
// overwrite a successor's work. Store.Append's version fence settles a race
|
||||
// with a concurrent re-lease.
|
||||
func lateHandoffAccepted(t domain.Task, leaseEpoch string) bool {
|
||||
return t.State == domain.StateQueued && t.Lease == nil && t.HandoffRef == "" &&
|
||||
leaseEpoch != "" && leaseEpoch == t.LastLeaseEpoch
|
||||
}
|
||||
|
||||
func main() {
|
||||
dir := os.Getenv("ORCHESTRA_DATA")
|
||||
if dir == "" {
|
||||
@@ -1428,7 +1441,8 @@ func main() {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
if !ownedLease {
|
||||
lateHandoff := !ownedLease && strings.HasSuffix(r.URL.Path, "/handoff") && lateHandoffAccepted(t, b.LeaseEpoch)
|
||||
if !ownedLease && !lateHandoff {
|
||||
http.Error(w, "lease not owned", 409)
|
||||
return
|
||||
}
|
||||
@@ -1656,7 +1670,9 @@ func main() {
|
||||
http.Error(w, "anchor_sha required", 400)
|
||||
return
|
||||
}
|
||||
if b.TransactionID == "" || b.ExpectedVersion != t.Version {
|
||||
// A late handoff cannot know the version: expiry bumped it after the
|
||||
// worker read it. Its fence is the epoch plus the append.
|
||||
if b.TransactionID == "" || (!lateHandoff && b.ExpectedVersion != t.Version) {
|
||||
http.Error(w, "release transaction and current lease version required", http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"orchestra/internal/domain"
|
||||
"orchestra/internal/registry"
|
||||
)
|
||||
|
||||
@@ -89,3 +90,22 @@ func TestTmuxRegistryRequiresMachineIdentityEvenOnOneMachine(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLateHandoffAcceptedOnlyFromTheExpiredOwner(t *testing.T) {
|
||||
expired := domain.Task{State: domain.StateQueued, LastLeaseEpoch: "ep1"}
|
||||
if !lateHandoffAccepted(expired, "ep1") {
|
||||
t.Fatal("the expired owner must still commit its pushed anchor")
|
||||
}
|
||||
for name, tk := range map[string]domain.Task{
|
||||
"re-leased": {State: domain.StateLeased, Lease: &domain.Lease{Epoch: "ep2"}, LastLeaseEpoch: "ep1"},
|
||||
"already handed off": {State: domain.StateQueued, LastLeaseEpoch: "ep1", HandoffRef: "sha256:x"},
|
||||
"blocked": {State: domain.StateBlocked, LastLeaseEpoch: "ep1"},
|
||||
} {
|
||||
if lateHandoffAccepted(tk, "ep1") {
|
||||
t.Fatalf("%s must refuse a late handoff", name)
|
||||
}
|
||||
}
|
||||
if lateHandoffAccepted(expired, "ep2") || lateHandoffAccepted(domain.Task{State: domain.StateQueued}, "") {
|
||||
t.Fatal("a foreign or empty epoch must refuse")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user