Complete autonomous recovery controls
This commit is contained in:
+40
-1
@@ -131,6 +131,8 @@ func (s *Store) apply(e domain.Event) error {
|
||||
s.external[t.Source+"\x00"+t.ExternalID] = t.ID
|
||||
case "TaskLeased":
|
||||
t.State = domain.StateLeased
|
||||
t.LifecyclePhase = "lease_issued"
|
||||
t.LastError = ""
|
||||
epoch, _ := p["lease_epoch"].(string)
|
||||
if epoch == "" {
|
||||
// A pre-fencing event cannot safely be renewed by an old worker.
|
||||
@@ -145,13 +147,30 @@ func (s *Store) apply(e domain.Event) error {
|
||||
epoch = t.Lease.Epoch
|
||||
}
|
||||
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Epoch: epoch, Until: time.Unix(0, int64(p["until_ns"].(float64)))}
|
||||
case "TaskLaunchAcknowledged":
|
||||
t.LifecyclePhase = "started"
|
||||
case "TaskReleased":
|
||||
t.State = domain.StateQueued
|
||||
t.LifecyclePhase = "reclaimed"
|
||||
t.Lease = nil
|
||||
t.HandoffRef, _ = p["handoff_ref"].(string)
|
||||
t.ReleaseTransaction, _ = p["transaction_id"].(string)
|
||||
t.ReleaseAnchor, _ = p["anchor_sha"].(string)
|
||||
t.PickupTransaction, t.PickupLeaseVersion = "", 0
|
||||
if t.HandoffRef == "" {
|
||||
// A handoff-less release is a reclaim. Persist the retry decision
|
||||
// here so expiry, pane exit, and a worker NACK all use the same
|
||||
// crash-safe transition instead of router-local counters.
|
||||
t.Attempt++
|
||||
t.FailureClass, _ = p["failure_class"].(string)
|
||||
if t.FailureClass == "" {
|
||||
t.FailureClass, _ = p["reason"].(string)
|
||||
}
|
||||
t.NextRetryAt = e.At.Add(retryBackoff(t.Attempt))
|
||||
} else {
|
||||
t.NextRetryAt = time.Time{}
|
||||
t.FailureClass = ""
|
||||
}
|
||||
case "TaskPickupValidated":
|
||||
t.PickupTransaction, _ = p["transaction_id"].(string)
|
||||
if v, ok := p["lease_version"].(float64); ok {
|
||||
@@ -221,6 +240,12 @@ func (s *Store) apply(e domain.Event) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
if phase, ok := p["lifecycle_phase"].(string); ok && phase != "" {
|
||||
t.LifecyclePhase = phase
|
||||
}
|
||||
if last, ok := p["last_error"].(string); ok {
|
||||
t.LastError = last
|
||||
}
|
||||
// Terminal and release events may carry an owner-produced snapshot from
|
||||
// immediately before a worker/coordinator drops its live session mapping.
|
||||
// Preserve it independently of the current task state so historical task
|
||||
@@ -244,6 +269,20 @@ func (s *Store) apply(e domain.Event) error {
|
||||
s.tasks[e.TaskID] = t
|
||||
return nil
|
||||
}
|
||||
|
||||
func retryBackoff(attempt int) time.Duration {
|
||||
if attempt < 1 {
|
||||
attempt = 1
|
||||
}
|
||||
backoff := time.Minute
|
||||
for i := 1; i < attempt && backoff < 30*time.Minute; i++ {
|
||||
backoff *= 2
|
||||
}
|
||||
if backoff > 30*time.Minute {
|
||||
return 30 * time.Minute
|
||||
}
|
||||
return backoff
|
||||
}
|
||||
func (s *Store) Append(e domain.Event) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
@@ -381,7 +420,7 @@ func (s *Store) validateTransition(e domain.Event, t domain.Task, exists bool, p
|
||||
return nil
|
||||
}
|
||||
switch e.Type {
|
||||
case "TaskLeaseRenewed", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskNeedsAttention", "TaskFailed":
|
||||
case "TaskLeaseRenewed", "TaskLaunchAcknowledged", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskNeedsAttention", "TaskFailed":
|
||||
owner, _ := p["harness_id"].(string)
|
||||
epoch, _ := p["lease_epoch"].(string)
|
||||
// Expiry is the one coordinator-owned relinquish path. It still binds
|
||||
|
||||
@@ -159,6 +159,38 @@ func TestNeedsAttentionRetainsFencedLeaseForLateCompletion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReclaimPersistsAttemptAndBackoffAcrossReopen(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s, err := Open(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"retry","project":"p"}`), Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Lease("t", "worker", time.Hour); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, _ := s.Task("t")
|
||||
at := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC)
|
||||
p, _ := json.Marshal(map[string]any{"reason": "lease_expired", "failure_class": "worker_lost", "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version})
|
||||
if err := s.Append(domain.Event{ID: "reclaim", Type: "TaskReleased", TaskID: "t", Version: task.Version + 1, At: at, Payload: p, Surface: string(authz.System)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := s.Task("t")
|
||||
if got.Attempt != 1 || got.FailureClass != "worker_lost" || !got.NextRetryAt.Equal(at.Add(time.Minute)) {
|
||||
t.Fatalf("reclaim projection=%+v", got)
|
||||
}
|
||||
reopened, err := Open(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = reopened.Task("t")
|
||||
if got.Attempt != 1 || !got.NextRetryAt.Equal(at.Add(time.Minute)) {
|
||||
t.Fatalf("reopen lost retry state: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenRebuildsOnlyFromLogAndIgnoresCorruptSnapshot(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s, err := Open(dir)
|
||||
|
||||
Reference in New Issue
Block a user