kami/Maven
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge task/536-kuma-flap-debounce (V-merge)

Browse Source
This commit is contained in:
claude
2026-08-05 02:51:50 +04:00
parent 113508eaac 0560684b35
commit a98d25ecac
8 changed files with 115 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mavend/alarm_stop_test.go
+4 -4
View File
@@ -72,7 +72,7 @@ func TestAlarmStopsWhenTheServiceComesBackUp(t *testing.T) {
st := newTestStore(t)
ctx := context.Background()
now := refNow()
seedDown(t, st, ctx, "down", now)
seedDown(t, st, ctx, "down", now.Add(-5*time.Minute))
sink := &fakeSink{}
tl := newAlarmTickLoop(t, st, sink)
@@ -101,7 +101,7 @@ func TestAlarmStopsAtTheAgeCapWhileStillDown(t *testing.T) {
st := newTestStore(t)
ctx := context.Background()
now := refNow()
seedDown(t, st, ctx, "down", now)
seedDown(t, st, ctx, "down", now.Add(-5*time.Minute))
sink := &fakeSink{}
tl := newAlarmTickLoop(t, st, sink)
@@ -132,7 +132,7 @@ func TestAFlapRaisesAFreshAlarmRatherThanReviveTheClosedOne(t *testing.T) {
st := newTestStore(t)
ctx := context.Background()
now := refNow()
seedDown(t, st, ctx, "down", now)
seedDown(t, st, ctx, "down", now.Add(-5*time.Minute))
sink := &fakeSink{}
tl := newAlarmTickLoop(t, st, sink)
@@ -145,7 +145,7 @@ func TestAFlapRaisesAFreshAlarmRatherThanReviveTheClosedOne(t *testing.T) {
t.Fatalf("closing the run changed the row count: %d → %d", first, len(got))
}
seedDown(t, st, ctx, "down", now.Add(30*time.Minute))
seedDown(t, st, ctx, "down", now.Add(25*time.Minute))
sink.sends = nil
tl.tick(ctx, now.Add(31*time.Minute))
cmd/mavend/tick_test.go
+3 -1
View File
@@ -584,7 +584,9 @@ func TestDigestSev4BypassesQueue(t *testing.T) {
ctx := context.Background()
now := refNow()
markPresent(t, st, ctx, now)
if _, err := st.SetValue(ctx, store.KindSelf, "service_down:db", "poll:uptimekuma", "down", now); err != nil {
// Older than loop.MinDownAge, so this tests the digest bypass and not the
// flap debounce (Vikunja #536).
if _, err := st.SetValue(ctx, store.KindSelf, "service_down:db", "poll:uptimekuma", "down", now.Add(-5*time.Minute)); err != nil {
t.Fatalf("seed service_down: %v", err)
}
sink := &fakeSink{}
internal/loop/explain_test.go
+2 -2
View File
@@ -103,7 +103,7 @@ func TestExplainGate_PresenceAway(t *testing.T) {
func TestExplainGate_PresenceAwayOpsBypass(t *testing.T) {
now := refTime()
s := State{Now: now, Presence: store.Away,
Facts: map[string]store.Fact{"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute))},
Facts: map[string]store.Fact{"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-5*time.Minute))},
}
r := ServiceDownRule() // Sev4 ops
passed, blocked, d := ExplainGate(s, r)
@@ -260,7 +260,7 @@ func TestExplainTick_WinnerRecorded(t *testing.T) {
Presence: store.Present,
Facts: map[string]store.Fact{
"water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)),
"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)),
"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-5*time.Minute)),
},
}
cand, trace := ExplainTick(s, DefaultRules())
internal/loop/gate_test.go
+1 -1
View File
@@ -202,7 +202,7 @@ func TestTickNeverDogpilesAndPicksLoudest(t *testing.T) {
"meal": ago("meal", "voice", `"lunch"`, 8*time.Hour),
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
"break": ago("break", "voice", `"walk"`, 3*time.Hour),
"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, time.Minute),
"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, 5*time.Minute),
"netdata_alarm": ago("netdata_alarm", "poll:netdata", `"critical"`, time.Minute),
},
}
internal/loop/gather_test.go
+3 -1
View File
@@ -25,7 +25,9 @@ func TestGatherStateLoadsPrefixFamilies(t *testing.T) {
"service_down:db": "down",
"service_down:web": "up",
} {
if _, err := s.SetValue(ctx, store.KindEnv, key, ServiceDownSource, val, now.Add(-time.Minute)); err != nil {
// Older than MinDownAge, so this tests the gather path and not the
// flap debounce.
if _, err := s.SetValue(ctx, store.KindEnv, key, ServiceDownSource, val, now.Add(-5*time.Minute)); err != nil {
t.Fatalf("SetValue %s: %v", key, err)
}
}
internal/loop/loop_test.go
+3 -3
View File
@@ -82,7 +82,7 @@ func TestTickOpsHardSurvivesAwayAndQuiet(t *testing.T) {
Presence: store.Away,
QuietHours: true,
Facts: map[string]store.Fact{
"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)),
"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-5*time.Minute)),
},
}
got := Tick(s, DefaultRules())
@@ -99,7 +99,7 @@ func TestTickServiceSourceTrustRefusesForgedTrigger(t *testing.T) {
Now: now,
Presence: store.Present,
Facts: map[string]store.Fact{
"service_down:db": factAt("service_down:db", "ambient", `"down"`, now.Add(-1*time.Minute)),
"service_down:db": factAt("service_down:db", "ambient", `"down"`, now.Add(-5*time.Minute)),
},
}
if got := Tick(s, DefaultRules()); got != nil {
@@ -116,7 +116,7 @@ func TestTickOneNudgePerTickMaxSeverityWins(t *testing.T) {
Presence: store.Present,
Facts: map[string]store.Fact{
"water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)),
"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)),
"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-5*time.Minute)),
},
}
got := Tick(s, DefaultRules())
internal/loop/rules.go
+42 -10
View File
@@ -144,6 +144,44 @@ func DownServices(s State) []string {
return out
}
// MinDownAge — how long a monitor must have read "down" before it is worth
// waking him (Vikunja #536).
//
// Technitium read down on one kuma poll and up on the next, sixty seconds
// apart, and the alarm arrived after the service was already back. mavpoll
// writes a service_down fact only when the state CHANGES, so the fact's
// timestamp is the instant the monitor went down and its age is how long it
// has stayed there. That is the whole debounce: no history to keep, no counter
// to persist.
//
// Ninety seconds is one poll interval plus room for jitter, so a monitor must
// survive at least one further poll as down. The cost is up to ninety seconds
// of alarm latency on a real outage, against never being paged for a blip.
//
// It bounds the alarm, not the truth: DownServices still reports a monitor the
// instant it goes down, because /dash showing a fresh outage is right even
// when phoning him about it is not.
const MinDownAge = 90 * time.Second
// downLongEnough — the newest down fact that has aged past MinDownAge, or the
// zero time when no monitor has. The rule fires off this and not off the
// newest down fact outright.
func downLongEnough(s State, now time.Time) time.Time {
var newest time.Time
for _, f := range s.FactsUnder(ServiceDownPrefix) {
if f.Source != ServiceDownSource || f.Value != `"down"` {
continue
}
if now.Sub(f.Ts) < MinDownAge {
continue
}
if f.Ts.After(newest) {
newest = f.Ts
}
}
return newest
}
// ServiceDownRule — sev4 ops hard: at least one kuma monitor reads "down".
//
// It used to read one aggregate `service_down` fact, which is why it was
@@ -160,17 +198,11 @@ func ServiceDownRule() Rule {
Cooldown: Cooldown{Base: 15 * time.Minute, Min: 5 * time.Minute, Max: 1 * time.Hour},
WantPrefixes: []string{ServiceDownPrefix},
Predicate: func(s State) bool {
var newest time.Time
for _, f := range s.FactsUnder(ServiceDownPrefix) {
if f.Source != ServiceDownSource || f.Value != `"down"` {
continue
}
if f.Ts.After(newest) {
newest = f.Ts
}
}
newest := downLongEnough(s, s.Now)
if newest.IsZero() {
return false // nothing down, or no data at all → shut up
// Nothing down, no data at all, or nothing down long enough
// to be more than a flap → shut up. See MinDownAge.
return false
}
return !s.NudgedSince("service_down", newest)
},
internal/loop/rules_test.go
+57 -17
View File
@@ -193,13 +193,13 @@ func TestOpsRulePredicates(t *testing.T) {
{
name: "service_down fires on a kuma down fact",
rule: ServiceDownRule(),
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, time.Minute)},
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, 5*time.Minute)},
want: true,
},
{
name: "service_down quiet when kuma says up",
rule: ServiceDownRule(),
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"up"`, time.Minute)},
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"up"`, 5*time.Minute)},
want: false,
},
{
@@ -218,31 +218,31 @@ func TestOpsRulePredicates(t *testing.T) {
{
name: "service_down refuses a forgery from the netdata poller",
rule: ServiceDownRule(),
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:netdata", `"down"`, time.Minute)},
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:netdata", `"down"`, 5*time.Minute)},
want: false,
},
{
name: "service_down refuses a forgery from ambient audio",
rule: ServiceDownRule(),
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "ambient:other", `"down"`, time.Minute)},
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "ambient:other", `"down"`, 5*time.Minute)},
want: false,
},
{
name: "service_down refuses a forgery from the user's own voice",
rule: ServiceDownRule(),
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "voice", `"down"`, time.Minute)},
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "voice", `"down"`, 5*time.Minute)},
want: false,
},
{
name: "service_down refuses a source that only looks like kuma",
rule: ServiceDownRule(),
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma-staging", `"down"`, time.Minute)},
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma-staging", `"down"`, 5*time.Minute)},
want: false,
},
{
name: "service_down refuses an unquoted down value",
rule: ServiceDownRule(),
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `down`, time.Minute)},
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `down`, 5*time.Minute)},
want: false,
},
@@ -383,7 +383,7 @@ func TestPredicatesArePure(t *testing.T) {
"meal": ago("meal", "voice", `"lunch"`, 7*time.Hour),
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
"break": ago("break", "voice", `"walk"`, 2*time.Hour),
"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, time.Minute),
"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, 5*time.Minute),
})
for _, r := range DefaultRules() {
first := r.Predicate(s)
@@ -445,12 +445,12 @@ func TestDownServicesNamesOnlyTheDownOnes(t *testing.T) {
s := State{
Now: refTime(),
Facts: map[string]store.Fact{
"service_down:web": ago("service_down:web", ServiceDownSource, `"up"`, time.Minute),
"service_down:db": ago("service_down:db", ServiceDownSource, `"down"`, time.Minute),
"service_down:vault": ago("service_down:vault", ServiceDownSource, `"down"`, time.Minute),
"service_down:paused": ago("service_down:paused", ServiceDownSource, `"maintenance"`, time.Minute),
"service_down:forged": ago("service_down:forged", "voice", `"down"`, time.Minute),
"service_down:missing": ago("service_down:missing", ServiceDownSource, `"unknown"`, time.Minute),
"service_down:web": ago("service_down:web", ServiceDownSource, `"up"`, 5*time.Minute),
"service_down:db": ago("service_down:db", ServiceDownSource, `"down"`, 5*time.Minute),
"service_down:vault": ago("service_down:vault", ServiceDownSource, `"down"`, 5*time.Minute),
"service_down:paused": ago("service_down:paused", ServiceDownSource, `"maintenance"`, 5*time.Minute),
"service_down:forged": ago("service_down:forged", "voice", `"down"`, 5*time.Minute),
"service_down:missing": ago("service_down:missing", ServiceDownSource, `"unknown"`, 5*time.Minute),
},
}
got := DownServices(s)
@@ -469,8 +469,8 @@ func TestDownServicesNamesOnlyTheDownOnes(t *testing.T) {
// the aggregate stayed "down" and pausing achieved nothing.
func TestPausedMonitorSilencesOnlyItself(t *testing.T) {
base := map[string]store.Fact{
"service_down:db": ago("service_down:db", ServiceDownSource, `"maintenance"`, time.Minute),
"service_down:web": ago("service_down:web", ServiceDownSource, `"down"`, time.Minute),
"service_down:db": ago("service_down:db", ServiceDownSource, `"maintenance"`, 5*time.Minute),
"service_down:web": ago("service_down:web", ServiceDownSource, `"down"`, 5*time.Minute),
}
if !ServiceDownRule().Predicate(State{Now: refTime(), Facts: base}) {
t.Fatal("web is still down, the rule must fire")
@@ -494,8 +494,48 @@ func TestServiceDownFiresOncePerTransition(t *testing.T) {
t.Fatal("already told about this transition, must be quiet")
}
// A second service goes down after that nudge — a new edge, so it fires.
s.Facts["service_down:web"] = ago("service_down:web", ServiceDownSource, `"down"`, time.Minute)
s.Facts["service_down:web"] = ago("service_down:web", ServiceDownSource, `"down"`, 5*time.Minute)
if !ServiceDownRule().Predicate(s) {
t.Fatal("a later transition must fire again")
}
}
// A monitor that reads down on one poll and up on the next raises nothing
// (Vikunja #536). Technitium did exactly that at 01:27:59 on 05-08-2026, and
// the alarm arrived after the service was already back.
func TestAFlappingMonitorRaisesNothing(t *testing.T) {
fresh := ago("service_down:dns", ServiceDownSource, `"down"`, 10*time.Second)
s := State{Now: refTime(), Facts: map[string]store.Fact{"service_down:dns": fresh}}
if ServiceDownRule().Predicate(s) {
t.Fatal("a monitor down for ten seconds must not wake him")
}
// It stays down. mavpoll writes only on change, so the fact does not move
// and its age is how long the outage has run.
s.Now = fresh.Ts.Add(MinDownAge)
if !ServiceDownRule().Predicate(s) {
t.Fatal("past MinDownAge this is a real outage and must fire")
}
}
// The debounce bounds the alarm, not the truth. /dash reads DownServices and
// showing a fresh outage there is right even when phoning him about it is not.
func TestDownServicesReportsAFreshOutage(t *testing.T) {
s := State{
Now: refTime(),
Facts: map[string]store.Fact{"service_down:dns": ago("service_down:dns", ServiceDownSource, `"down"`, 10*time.Second)},
}
if got := DownServices(s); len(got) != 1 || got[0] != "dns" {
t.Fatalf("DownServices = %v, want [dns]", got)
}
}
// One monitor flapping must not hide another that has been down for an hour.
func TestAFlapDoesNotMaskARealOutage(t *testing.T) {
s := State{Now: refTime(), Facts: map[string]store.Fact{
"service_down:dns": ago("service_down:dns", ServiceDownSource, `"down"`, 10*time.Second),
"service_down:db": ago("service_down:db", ServiceDownSource, `"down"`, time.Hour),
}}
if !ServiceDownRule().Predicate(s) {
t.Fatal("db has been down an hour, the rule must fire")
}
}