kuma: a monitor must stay down before it wakes him (V-536)

Technitium read down on one poll and up on the next, sixty seconds apart, and
the sev4 arrived after the service was already back.

mavpoll writes a service_down fact only when the state changes, so the fact's
timestamp IS the moment the monitor went down and its age is how long it has
stayed there. The debounce is that age against MinDownAge, 90s — one poll
interval plus jitter. No history to keep and no counter to persist.

It bounds the alarm and not the truth: DownServices still reports a monitor the
instant it goes down, because /dash showing a fresh outage is right even when
phoning him about it is not. Existing fixtures that seeded a one-minute-old
down fact now seed five, which is what they always meant.

internal/loop/rules.go

@@ -144,6 +144,44 @@ func DownServices(s State) []string {
 	return out
 }
 
+// MinDownAge — how long a monitor must have read "down" before it is worth
+// waking him (Vikunja #536).
+//
+// Technitium read down on one kuma poll and up on the next, sixty seconds
+// apart, and the alarm arrived after the service was already back. mavpoll
+// writes a service_down fact only when the state CHANGES, so the fact's
+// timestamp is the instant the monitor went down and its age is how long it
+// has stayed there. That is the whole debounce: no history to keep, no counter
+// to persist.
+//
+// Ninety seconds is one poll interval plus room for jitter, so a monitor must
+// survive at least one further poll as down. The cost is up to ninety seconds
+// of alarm latency on a real outage, against never being paged for a blip.
+//
+// It bounds the alarm, not the truth: DownServices still reports a monitor the
+// instant it goes down, because /dash showing a fresh outage is right even
+// when phoning him about it is not.
+const MinDownAge = 90 * time.Second
+
+// downLongEnough — the newest down fact that has aged past MinDownAge, or the
+// zero time when no monitor has. The rule fires off this and not off the
+// newest down fact outright.
+func downLongEnough(s State, now time.Time) time.Time {
+	var newest time.Time
+	for _, f := range s.FactsUnder(ServiceDownPrefix) {
+		if f.Source != ServiceDownSource || f.Value != `"down"` {
+			continue
+		}
+		if now.Sub(f.Ts) < MinDownAge {
+			continue
+		}
+		if f.Ts.After(newest) {
+			newest = f.Ts
+		}
+	}
+	return newest
+}
+
 // ServiceDownRule — sev4 ops hard: at least one kuma monitor reads "down".
 //
 // It used to read one aggregate `service_down` fact, which is why it was
@@ -160,17 +198,11 @@ func ServiceDownRule() Rule {
 		Cooldown:     Cooldown{Base: 15 * time.Minute, Min: 5 * time.Minute, Max: 1 * time.Hour},
 		WantPrefixes: []string{ServiceDownPrefix},
 		Predicate: func(s State) bool {
-			var newest time.Time
-			for _, f := range s.FactsUnder(ServiceDownPrefix) {
-				if f.Source != ServiceDownSource || f.Value != `"down"` {
-					continue
-				}
-				if f.Ts.After(newest) {
-					newest = f.Ts
-				}
-			}
+			newest := downLongEnough(s, s.Now)
 			if newest.IsZero() {
-				return false // nothing down, or no data at all → shut up
+				// Nothing down, no data at all, or nothing down long enough
+				// to be more than a flap → shut up. See MinDownAge.
+				return false
 			}
 			return !s.NudgedSince("service_down", newest)
 		},