Preserve leases needing recovery

This commit is contained in:
kami
2026-07-30 14:37:34 +04:00
parent f6ee0e3060
commit 8174400b1a
12 changed files with 106 additions and 39 deletions
+19 -8
View File
@@ -163,9 +163,15 @@ func (s *Store) apply(e domain.Event) error {
case "TaskFailed":
t.State = domain.StateFailed
t.Lease = nil
case "TaskBlocked":
t.State = domain.StateBlocked
t.Lease = nil
case "TaskBlocked", "TaskNeedsAttention":
if e.Type == "TaskBlocked" {
t.State = domain.StateBlocked
t.Lease = nil
} else {
// Recovery diagnostics must not revoke the fenced owner. A late
// completion is still valid only from this exact lease epoch.
t.State = domain.StateNeedsAttention
}
t.Blocker, _ = p["blocker"].(string)
t.BlockReason = domain.InferBlockReason(t.Blocker)
if v, ok := p["block_reason"].(string); ok && domain.BlockReason(v).Valid() {
@@ -316,7 +322,7 @@ func (s *Store) Append(e domain.Event) error {
if !taskExists && e.Type != "TaskCreated" && !global {
return domain.ErrNotFound
}
if e.Type != "TaskCreated" && (e.Type == "TaskCompleted" || e.Type == "TaskBlocked" || e.Type == "TaskReleased") {
if e.Type != "TaskCreated" && (e.Type == "TaskCompleted" || e.Type == "TaskBlocked" || e.Type == "TaskNeedsAttention" || e.Type == "TaskReleased") {
var p map[string]any
_ = json.Unmarshal(e.Payload, &p)
for _, k := range []string{"handoff_ref", "report_ref"} {
@@ -366,11 +372,16 @@ func (s *Store) validateTransition(e domain.Event, t domain.Task, exists bool, p
if e.Type == "TaskLeased" && t.State != domain.StateQueued {
return domain.ErrConflict
}
if t.State != domain.StateLeased || t.Lease == nil {
// Needs-attention is specifically a recoverable leased state, never a
// second spelling of a terminal operator block on an unowned task.
if e.Type == "TaskNeedsAttention" && ((t.State != domain.StateLeased && t.State != domain.StateNeedsAttention) || t.Lease == nil) {
return domain.ErrConflict
}
if (t.State != domain.StateLeased && t.State != domain.StateNeedsAttention) || t.Lease == nil {
return nil
}
switch e.Type {
case "TaskLeaseRenewed", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskFailed":
case "TaskLeaseRenewed", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskNeedsAttention", "TaskFailed":
owner, _ := p["harness_id"].(string)
epoch, _ := p["lease_epoch"].(string)
// Expiry is the one coordinator-owned relinquish path. It still binds
@@ -561,7 +572,7 @@ func (s *Store) RenewLease(id, harness, epoch string, expectedVersion int, ttl t
if !ok {
return domain.Event{}, domain.ErrNotFound
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != harness || t.Lease.Epoch != epoch || t.Version != expectedVersion {
if (t.State != domain.StateLeased && t.State != domain.StateNeedsAttention) || t.Lease == nil || t.Lease.HarnessID != harness || t.Lease.Epoch != epoch || t.Version != expectedVersion {
return domain.Event{}, domain.ErrConflict
}
p, _ := json.Marshal(map[string]any{"harness_id": harness, "lease_epoch": epoch, "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": expectedVersion})
@@ -592,7 +603,7 @@ func (s *Store) ExpireLease(id string, now time.Time) (domain.Event, error) {
if !ok {
return domain.Event{}, domain.ErrNotFound
}
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.Until.After(now) {
if (t.State != domain.StateLeased && t.State != domain.StateNeedsAttention) || t.Lease == nil || t.Lease.Until.After(now) {
return domain.Event{}, domain.ErrConflict
}
p, _ := json.Marshal(map[string]any{"reason": "lease_expired", "harness_id": t.Lease.HarnessID, "lease_epoch": t.Lease.Epoch, "expected_version": t.Version})
+34
View File
@@ -125,6 +125,40 @@ func TestLeaseEpochFencesStaleOwnerLifecycleWrites(t *testing.T) {
}
}
func TestNeedsAttentionRetainsFencedLeaseForLateCompletion(t *testing.T) {
s, err := Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"attention","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("t", "worker", time.Hour); err != nil {
t.Fatal(err)
}
leased, _ := s.Task("t")
attention, _ := json.Marshal(map[string]any{"blocker": "prompt response uncertain", "block_reason": "lease_failure", "harness_id": leased.Lease.HarnessID, "lease_epoch": leased.Lease.Epoch, "expected_version": leased.Version})
if err := s.Append(domain.Event{ID: "attention", Type: "TaskNeedsAttention", TaskID: "t", Version: leased.Version + 1, Payload: attention, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
current, _ := s.Task("t")
if current.State != domain.StateNeedsAttention || current.Lease == nil || current.Lease.Epoch != leased.Lease.Epoch {
t.Fatalf("attention revoked or replaced lease: %+v", current)
}
report, err := s.PutArtifact([]byte("late completion"))
if err != nil {
t.Fatal(err)
}
completion, _ := json.Marshal(map[string]any{"report_ref": report, "receipt": map[string]any{"consumed": 1}, "harness_id": current.Lease.HarnessID, "lease_epoch": current.Lease.Epoch, "expected_version": current.Version})
if err := s.Append(domain.Event{ID: "late-complete", Type: "TaskCompleted", TaskID: "t", Version: current.Version + 1, Payload: completion, Surface: string(authz.System)}); err != nil {
t.Fatalf("late completion from retained owner: %v", err)
}
completed, _ := s.Task("t")
if completed.State != domain.StateCompleted || completed.Lease != nil {
t.Fatalf("late completion did not settle task: %+v", completed)
}
}
func TestOpenRebuildsOnlyFromLogAndIgnoresCorruptSnapshot(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)