Stop the ring's history from manufacturing recurrence
Both defects came from the first real run of slice B against live data, and neither was visible in a unit test written from the design. The ring is a bounded history, not a set of live conditions. A quiet timeout closed an incident, its entry stayed in the ring because nothing evicts it, and every later heartbeat opened the same incident again: three signatures, four incidents each, from failures that never happened twice. An incident now opens only when the entry actually advances past what was already accounted, and the high-water mark survives the close. The ring also outlives the work it describes, so attributing its entries to whatever the worker is running now invented an association. The task is read out of the message, and only a failure that names no task belongs to the current lease. An incident that names an older task has no live lease to bound it, so it closes on quiet timeout rather than on the next epoch change. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
This commit is contained in:
@@ -218,3 +218,78 @@ func TestSignatureCollapsesAPaneName(t *testing.T) {
|
||||
t.Fatalf("one failure has two signatures:\n%s\n%s", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
// The first live run manufactured recurrence out of one old failure: a quiet
|
||||
// timeout closed the incident, the entry stayed in the ring because the ring
|
||||
// is a bounded history rather than a set of live conditions, and every later
|
||||
// heartbeat opened it again. Four incidents, one failure, no new occurrence.
|
||||
func TestAClosedIncidentDoesNotReopenFromAStaleRingEntry(t *testing.T) {
|
||||
tr, s := tracker(t)
|
||||
at := time.Unix(1700000000, 0).UTC()
|
||||
entry := ring("heartbeat: connection refused", 4, at)
|
||||
if _, err := tr.Ingest(WorkerReport{WorkerID: "w", Incarnation: "b1", Observations: entry, At: at}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Long enough to close on quiet timeout, with the entry still reported.
|
||||
quiet := at.Add(QuietTimeout + time.Minute)
|
||||
for i := 0; i < 4; i++ {
|
||||
if _, err := tr.Ingest(WorkerReport{
|
||||
WorkerID: "w", Incarnation: "b1", Observations: entry,
|
||||
At: quiet.Add(time.Duration(i) * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
opened, closed := 0, 0
|
||||
for _, e := range s.Events(0) {
|
||||
switch e.Type {
|
||||
case domain.EventObservationIncidentOpened:
|
||||
opened++
|
||||
case domain.EventObservationIncidentClosed:
|
||||
closed++
|
||||
}
|
||||
}
|
||||
if opened != 1 || closed != 1 {
|
||||
t.Fatalf("opened=%d closed=%d for one failure that never happened again", opened, closed)
|
||||
}
|
||||
|
||||
// A real new occurrence, which the ring shows by advancing the entry.
|
||||
later := quiet.Add(time.Hour)
|
||||
if _, err := tr.Ingest(WorkerReport{
|
||||
WorkerID: "w", Incarnation: "b1", Observations: ring("heartbeat: connection refused", 5, later), At: later,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened := 0
|
||||
for _, e := range s.Events(0) {
|
||||
if e.Type == domain.EventObservationIncidentOpened {
|
||||
reopened++
|
||||
}
|
||||
}
|
||||
if reopened != 2 {
|
||||
t.Fatalf("a genuine new occurrence did not open an incident: opened=%d", reopened)
|
||||
}
|
||||
}
|
||||
|
||||
// The ring outlives the work it describes, so the task comes from the message
|
||||
// rather than from whatever the worker happens to be running now.
|
||||
func TestTheTaskComesFromTheMessageNotTheCurrentLease(t *testing.T) {
|
||||
tr, s := tracker(t)
|
||||
at := time.Unix(1700000000, 0).UTC()
|
||||
if _, err := tr.Ingest(WorkerReport{
|
||||
WorkerID: "w", Incarnation: "b1", TaskID: "06G4XAFH1MBPC35VSJN7V3NS14", LeaseEpoch: "now",
|
||||
Observations: ring("renew lease 06G4WW6TND26M16CZA6WE5T458: 409 conflict", 3, at), At: at,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open := s.OpenObservations()
|
||||
if len(open) != 1 {
|
||||
t.Fatalf("open = %+v", open)
|
||||
}
|
||||
if open[0].TaskID != "06G4WW6TND26M16CZA6WE5T458" {
|
||||
t.Fatalf("the failure was attributed to the wrong task: %q", open[0].TaskID)
|
||||
}
|
||||
if open[0].LeaseEpoch != "" {
|
||||
t.Fatalf("an old failure inherited the current lease's epoch: %q", open[0].LeaseEpoch)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user