Preserve leases needing recovery
This commit is contained in:
+19
-8
@@ -163,9 +163,15 @@ func (s *Store) apply(e domain.Event) error {
|
||||
case "TaskFailed":
|
||||
t.State = domain.StateFailed
|
||||
t.Lease = nil
|
||||
case "TaskBlocked":
|
||||
t.State = domain.StateBlocked
|
||||
t.Lease = nil
|
||||
case "TaskBlocked", "TaskNeedsAttention":
|
||||
if e.Type == "TaskBlocked" {
|
||||
t.State = domain.StateBlocked
|
||||
t.Lease = nil
|
||||
} else {
|
||||
// Recovery diagnostics must not revoke the fenced owner. A late
|
||||
// completion is still valid only from this exact lease epoch.
|
||||
t.State = domain.StateNeedsAttention
|
||||
}
|
||||
t.Blocker, _ = p["blocker"].(string)
|
||||
t.BlockReason = domain.InferBlockReason(t.Blocker)
|
||||
if v, ok := p["block_reason"].(string); ok && domain.BlockReason(v).Valid() {
|
||||
@@ -316,7 +322,7 @@ func (s *Store) Append(e domain.Event) error {
|
||||
if !taskExists && e.Type != "TaskCreated" && !global {
|
||||
return domain.ErrNotFound
|
||||
}
|
||||
if e.Type != "TaskCreated" && (e.Type == "TaskCompleted" || e.Type == "TaskBlocked" || e.Type == "TaskReleased") {
|
||||
if e.Type != "TaskCreated" && (e.Type == "TaskCompleted" || e.Type == "TaskBlocked" || e.Type == "TaskNeedsAttention" || e.Type == "TaskReleased") {
|
||||
var p map[string]any
|
||||
_ = json.Unmarshal(e.Payload, &p)
|
||||
for _, k := range []string{"handoff_ref", "report_ref"} {
|
||||
@@ -366,11 +372,16 @@ func (s *Store) validateTransition(e domain.Event, t domain.Task, exists bool, p
|
||||
if e.Type == "TaskLeased" && t.State != domain.StateQueued {
|
||||
return domain.ErrConflict
|
||||
}
|
||||
if t.State != domain.StateLeased || t.Lease == nil {
|
||||
// Needs-attention is specifically a recoverable leased state, never a
|
||||
// second spelling of a terminal operator block on an unowned task.
|
||||
if e.Type == "TaskNeedsAttention" && ((t.State != domain.StateLeased && t.State != domain.StateNeedsAttention) || t.Lease == nil) {
|
||||
return domain.ErrConflict
|
||||
}
|
||||
if (t.State != domain.StateLeased && t.State != domain.StateNeedsAttention) || t.Lease == nil {
|
||||
return nil
|
||||
}
|
||||
switch e.Type {
|
||||
case "TaskLeaseRenewed", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskFailed":
|
||||
case "TaskLeaseRenewed", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskNeedsAttention", "TaskFailed":
|
||||
owner, _ := p["harness_id"].(string)
|
||||
epoch, _ := p["lease_epoch"].(string)
|
||||
// Expiry is the one coordinator-owned relinquish path. It still binds
|
||||
@@ -561,7 +572,7 @@ func (s *Store) RenewLease(id, harness, epoch string, expectedVersion int, ttl t
|
||||
if !ok {
|
||||
return domain.Event{}, domain.ErrNotFound
|
||||
}
|
||||
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.HarnessID != harness || t.Lease.Epoch != epoch || t.Version != expectedVersion {
|
||||
if (t.State != domain.StateLeased && t.State != domain.StateNeedsAttention) || t.Lease == nil || t.Lease.HarnessID != harness || t.Lease.Epoch != epoch || t.Version != expectedVersion {
|
||||
return domain.Event{}, domain.ErrConflict
|
||||
}
|
||||
p, _ := json.Marshal(map[string]any{"harness_id": harness, "lease_epoch": epoch, "until_ns": time.Now().Add(ttl).UnixNano(), "expected_version": expectedVersion})
|
||||
@@ -592,7 +603,7 @@ func (s *Store) ExpireLease(id string, now time.Time) (domain.Event, error) {
|
||||
if !ok {
|
||||
return domain.Event{}, domain.ErrNotFound
|
||||
}
|
||||
if t.State != domain.StateLeased || t.Lease == nil || t.Lease.Until.After(now) {
|
||||
if (t.State != domain.StateLeased && t.State != domain.StateNeedsAttention) || t.Lease == nil || t.Lease.Until.After(now) {
|
||||
return domain.Event{}, domain.ErrConflict
|
||||
}
|
||||
p, _ := json.Marshal(map[string]any{"reason": "lease_expired", "harness_id": t.Lease.HarnessID, "lease_epoch": t.Lease.Epoch, "expected_version": t.Version})
|
||||
|
||||
@@ -125,6 +125,40 @@ func TestLeaseEpochFencesStaleOwnerLifecycleWrites(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeedsAttentionRetainsFencedLeaseForLateCompletion(t *testing.T) {
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"attention","project":"p"}`), Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Lease("t", "worker", time.Hour); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
leased, _ := s.Task("t")
|
||||
attention, _ := json.Marshal(map[string]any{"blocker": "prompt response uncertain", "block_reason": "lease_failure", "harness_id": leased.Lease.HarnessID, "lease_epoch": leased.Lease.Epoch, "expected_version": leased.Version})
|
||||
if err := s.Append(domain.Event{ID: "attention", Type: "TaskNeedsAttention", TaskID: "t", Version: leased.Version + 1, Payload: attention, Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, _ := s.Task("t")
|
||||
if current.State != domain.StateNeedsAttention || current.Lease == nil || current.Lease.Epoch != leased.Lease.Epoch {
|
||||
t.Fatalf("attention revoked or replaced lease: %+v", current)
|
||||
}
|
||||
report, err := s.PutArtifact([]byte("late completion"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completion, _ := json.Marshal(map[string]any{"report_ref": report, "receipt": map[string]any{"consumed": 1}, "harness_id": current.Lease.HarnessID, "lease_epoch": current.Lease.Epoch, "expected_version": current.Version})
|
||||
if err := s.Append(domain.Event{ID: "late-complete", Type: "TaskCompleted", TaskID: "t", Version: current.Version + 1, Payload: completion, Surface: string(authz.System)}); err != nil {
|
||||
t.Fatalf("late completion from retained owner: %v", err)
|
||||
}
|
||||
completed, _ := s.Task("t")
|
||||
if completed.State != domain.StateCompleted || completed.Lease != nil {
|
||||
t.Fatalf("late completion did not settle task: %+v", completed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenRebuildsOnlyFromLogAndIgnoresCorruptSnapshot(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s, err := Open(dir)
|
||||
|
||||
Reference in New Issue
Block a user