Report a bounded ring of worker failures, not one slot

F18. A single last_error slot destroyed causal evidence twice. Run 7
kept only the last of four failures. In run 11 a five-second retry loop
on a dead task pinned the slot for twenty-six minutes, so the live
task's own expiry was never visible at all, and run 12 lost diagnosis
time to the same thing before F58 removed the flood.

WorkerHealth now carries up to sixteen distinct observations, each with
its repeat count and first/last times. Collapsing is by message rather
than by position, because a loop interleaved with other failures would
otherwise still flush the ring. Eviction drops the least recently seen.
last_error and error_at keep their wire names and still report only the
newest failure, so nothing reading them has to change.

The ring lives in memory beside last_error and is not persisted, which
is the behaviour last_error already had across a restart.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
2026-08-28 23:43:26 +04:00
parent b317ecb1bd
commit d6ee10f028
3 changed files with 106 additions and 2 deletions
+32 -2
View File
@@ -54,6 +54,7 @@ type worker struct {
registration federation.Worker
lastError string
lastErrorAt time.Time
observations []federation.Observation
soft float64
window int64
}
@@ -68,12 +69,38 @@ func (w *worker) executionBackend() herdr.Backend {
return nil
}
// observationRing bounds how many distinct failures a worker reports at once.
const observationRing = 16
func (w *worker) recordError(err error) {
if err == nil {
return
}
w.lastError = err.Error()
w.lastErrorAt = time.Now().UTC()
msg, now := err.Error(), time.Now().UTC()
w.lastError, w.lastErrorAt = msg, now
// Collapse by message rather than by position. A retry loop interleaved
// with other failures would otherwise still flush the ring, which is the
// whole reason one slot was not enough.
for i, o := range w.observations {
if o.Message == msg {
w.observations[i].Count++
w.observations[i].Last = now
return
}
}
if len(w.observations) >= observationRing {
// Evict the least recently seen. A loop keeps its slot, but it carries
// a count that says so, and a one-off from an hour ago is the entry
// worth losing first.
oldest := 0
for i, o := range w.observations {
if o.Last.Before(w.observations[oldest].Last) {
oldest = i
}
}
w.observations = append(w.observations[:oldest], w.observations[oldest+1:]...)
}
w.observations = append(w.observations, federation.Observation{Message: msg, Count: 1, First: now, Last: now})
}
func (w *worker) health(ctx context.Context) federation.WorkerHealth {
@@ -101,6 +128,9 @@ func (w *worker) health(ctx context.Context) federation.WorkerHealth {
}
}
h.LastError, h.ErrorAt = w.lastError, w.lastErrorAt
if len(w.observations) > 0 {
h.Observations = append([]federation.Observation(nil), w.observations...)
}
return h
}