Complete autonomous recovery controls

This commit is contained in:
kami
2026-07-30 14:57:25 +04:00
parent 8174400b1a
commit e8fadfc998
18 changed files with 364 additions and 77 deletions
+40 -1
View File
@@ -131,6 +131,8 @@ func (s *Store) apply(e domain.Event) error {
s.external[t.Source+"\x00"+t.ExternalID] = t.ID
case "TaskLeased":
t.State = domain.StateLeased
t.LifecyclePhase = "lease_issued"
t.LastError = ""
epoch, _ := p["lease_epoch"].(string)
if epoch == "" {
// A pre-fencing event cannot safely be renewed by an old worker.
@@ -145,13 +147,30 @@ func (s *Store) apply(e domain.Event) error {
epoch = t.Lease.Epoch
}
t.Lease = &domain.Lease{HarnessID: p["harness_id"].(string), Epoch: epoch, Until: time.Unix(0, int64(p["until_ns"].(float64)))}
case "TaskLaunchAcknowledged":
t.LifecyclePhase = "started"
case "TaskReleased":
t.State = domain.StateQueued
t.LifecyclePhase = "reclaimed"
t.Lease = nil
t.HandoffRef, _ = p["handoff_ref"].(string)
t.ReleaseTransaction, _ = p["transaction_id"].(string)
t.ReleaseAnchor, _ = p["anchor_sha"].(string)
t.PickupTransaction, t.PickupLeaseVersion = "", 0
if t.HandoffRef == "" {
// A handoff-less release is a reclaim. Persist the retry decision
// here so expiry, pane exit, and a worker NACK all use the same
// crash-safe transition instead of router-local counters.
t.Attempt++
t.FailureClass, _ = p["failure_class"].(string)
if t.FailureClass == "" {
t.FailureClass, _ = p["reason"].(string)
}
t.NextRetryAt = e.At.Add(retryBackoff(t.Attempt))
} else {
t.NextRetryAt = time.Time{}
t.FailureClass = ""
}
case "TaskPickupValidated":
t.PickupTransaction, _ = p["transaction_id"].(string)
if v, ok := p["lease_version"].(float64); ok {
@@ -221,6 +240,12 @@ func (s *Store) apply(e domain.Event) error {
}
}
}
if phase, ok := p["lifecycle_phase"].(string); ok && phase != "" {
t.LifecyclePhase = phase
}
if last, ok := p["last_error"].(string); ok {
t.LastError = last
}
// Terminal and release events may carry an owner-produced snapshot from
// immediately before a worker/coordinator drops its live session mapping.
// Preserve it independently of the current task state so historical task
@@ -244,6 +269,20 @@ func (s *Store) apply(e domain.Event) error {
s.tasks[e.TaskID] = t
return nil
}
func retryBackoff(attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
backoff := time.Minute
for i := 1; i < attempt && backoff < 30*time.Minute; i++ {
backoff *= 2
}
if backoff > 30*time.Minute {
return 30 * time.Minute
}
return backoff
}
func (s *Store) Append(e domain.Event) error {
s.mu.Lock()
defer s.mu.Unlock()
@@ -381,7 +420,7 @@ func (s *Store) validateTransition(e domain.Event, t domain.Task, exists bool, p
return nil
}
switch e.Type {
case "TaskLeaseRenewed", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskNeedsAttention", "TaskFailed":
case "TaskLeaseRenewed", "TaskLaunchAcknowledged", "TaskReleased", "TaskPickupValidated", "TaskCompleted", "TaskBlocked", "TaskNeedsAttention", "TaskFailed":
owner, _ := p["harness_id"].(string)
epoch, _ := p["lease_epoch"].(string)
// Expiry is the one coordinator-owned relinquish path. It still binds
+32
View File
@@ -159,6 +159,38 @@ func TestNeedsAttentionRetainsFencedLeaseForLateCompletion(t *testing.T) {
}
}
func TestReclaimPersistsAttemptAndBackoffAcrossReopen(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)
if err != nil {
t.Fatal(err)
}
if err := s.Append(domain.Event{ID: "create", Type: "TaskCreated", TaskID: "t", Version: 1, Payload: []byte(`{"source":"s","external_id":"retry","project":"p"}`), Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("t", "worker", time.Hour); err != nil {
t.Fatal(err)
}
task, _ := s.Task("t")
at := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC)
p, _ := json.Marshal(map[string]any{"reason": "lease_expired", "failure_class": "worker_lost", "harness_id": task.Lease.HarnessID, "lease_epoch": task.Lease.Epoch, "expected_version": task.Version})
if err := s.Append(domain.Event{ID: "reclaim", Type: "TaskReleased", TaskID: "t", Version: task.Version + 1, At: at, Payload: p, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
got, _ := s.Task("t")
if got.Attempt != 1 || got.FailureClass != "worker_lost" || !got.NextRetryAt.Equal(at.Add(time.Minute)) {
t.Fatalf("reclaim projection=%+v", got)
}
reopened, err := Open(dir)
if err != nil {
t.Fatal(err)
}
got, _ = reopened.Task("t")
if got.Attempt != 1 || !got.NextRetryAt.Equal(at.Add(time.Minute)) {
t.Fatalf("reopen lost retry state: %+v", got)
}
}
func TestOpenRebuildsOnlyFromLogAndIgnoresCorruptSnapshot(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)