0560684b35
Technitium read down on one poll and up on the next, sixty seconds apart, and the sev4 arrived after the service was already back. mavpoll writes a service_down fact only when the state changes, so the fact's timestamp IS the moment the monitor went down and its age is how long it has stayed there. The debounce is that age against MinDownAge, 90s — one poll interval plus jitter. No history to keep and no counter to persist. It bounds the alarm and not the truth: DownServices still reports a monitor the instant it goes down, because /dash showing a fresh outage is right even when phoning him about it is not. Existing fixtures that seeded a one-minute-old down fact now seed five, which is what they always meant.
280 lines
10 KiB
Go
280 lines
10 KiB
Go
package loop
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Rule — a proactive rule. Rules are CODE, not a DSL config — until ~30 rules
|
|
// and you feel the pain (per spec). A Rule has a name (ids it in nudges.outcome
|
|
// for the feedback loop), a Severity, a pure Predicate, and a cooldown.
|
|
//
|
|
// The Predicate answers "should this rule want to fire given the State?" —
|
|
// only the check against the snapshot. It is pure, no I/O. The GATE answers
|
|
// "are we allowed to fire it right now?" (quiet hours, cooldown, etc) —
|
|
// applied by the loop, never per-rule.
|
|
//
|
|
// Cooldown is the BASE duration between same-rule nudges. The feedback
|
|
// auto-tuner scales it over time (mostly ignored → lengthen; acted → leave).
|
|
// Bounds belong at the daemon-config level; here we just carry the base.
|
|
type Rule struct {
|
|
Name string
|
|
Severity Severity
|
|
Cooldown // base cooldown + bounded-duration envelope for the auto-tuner
|
|
Predicate func(State) bool
|
|
|
|
// InertWhenNoData — most rules should be silent when their substrate key is
|
|
// missing (since(key)==null → don't fire). If the predicate already encodes
|
|
// that check itself, leave this empty. Otherwise set to the key(s) the rule
|
|
// needs and the gate will skip the rule when any are missing.
|
|
InertWhenNoData []string
|
|
|
|
// WantPrefixes — key prefixes whose whole family the gatherer must load.
|
|
// InertWhenNoData names keys that exist at wiring time; a rule over a key
|
|
// set that is only known at read time (one fact per kuma monitor) declares
|
|
// the prefix here instead. Prefixes never make a rule inert: an empty
|
|
// family is the predicate's own "no data" case.
|
|
WantPrefixes []string
|
|
|
|
// StillTrue — is the CONDITION still true, ignoring whether it is worth
|
|
// saying again? Distinct from Predicate on purpose, and the distinction is
|
|
// the whole reason this field exists (Vikunja #535).
|
|
//
|
|
// Predicate answers "should this fire now", which folds in edge-triggering:
|
|
// ServiceDownRule ends in !s.NudgedSince(...), so it reads false the instant
|
|
// a nudge goes out even though the service is still down. A repeat loop that
|
|
// consulted Predicate would cancel every alarm one tick after raising it,
|
|
// which is exactly backwards.
|
|
//
|
|
// Only a rule whose alarm repeats needs this. nil means "I cannot tell you",
|
|
// and the caller must then fall back to a bound it can enforce without the
|
|
// rule's help. nil must never be read as "the condition cleared": a rule
|
|
// that says nothing about its condition is not a rule that resolved.
|
|
StillTrue func(State) bool
|
|
}
|
|
|
|
// Cooldown — tunable bounded by the envelope so a weird week (auto-tuned) can't
|
|
// mutate maven silent or stalker. Base is what the rule ships with; Min/Max
|
|
// bound the feedback-driven adjustments persisted as `facts (source=feedback)`.
|
|
type Cooldown struct {
|
|
Base time.Duration
|
|
Min time.Duration
|
|
Max time.Duration
|
|
}
|
|
|
|
// Canonical care/ops rules — NOT a config DSL. Code, so the predicate is
|
|
// inspectable and unit-tested. Daemon wires these up; the loop just iterates.
|
|
|
|
// WaterRule — sev1 care: if it's been ≥3h since a `water` fact, fire.
|
|
// Inert when no water fact exists at all (shuts up when uncertain).
|
|
func WaterRule() Rule {
|
|
return Rule{
|
|
Name: "water",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 30 * time.Minute, Min: 15 * time.Minute, Max: 6 * time.Hour},
|
|
InertWhenNoData: []string{"water"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("water")
|
|
if !ok {
|
|
return false // no data → shut up
|
|
}
|
|
return d >= 3*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// MealRule — sev1 care: if ≥6h since an `meal` fact, fire. Inert without data.
|
|
func MealRule() Rule {
|
|
return Rule{
|
|
Name: "meal",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 60 * time.Minute, Min: 30 * time.Minute, Max: 8 * time.Hour},
|
|
InertWhenNoData: []string{"meal"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("meal")
|
|
if !ok {
|
|
return false
|
|
}
|
|
return d >= 6*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// BreakRule — sev2 care: ≥90min of continuous desk activity without a break.
|
|
// Reads both `desk_active` (fresh input ⇒ at desk) and `break` (last taken).
|
|
// Inert unless both exist — can't claim continuous activity without both anchors.
|
|
func BreakRule() Rule {
|
|
return Rule{
|
|
Name: "break",
|
|
Severity: Sev2,
|
|
Cooldown: Cooldown{Base: 45 * time.Minute, Min: 20 * time.Minute, Max: 4 * time.Hour},
|
|
InertWhenNoData: []string{"desk_active", "break"},
|
|
Predicate: func(s State) bool {
|
|
dDesk, ok1 := s.Since("desk_active")
|
|
dBreak, ok2 := s.Since("break")
|
|
if !ok1 || !ok2 {
|
|
return false // shut up until we have both anchors
|
|
}
|
|
// at desk (fresh input within 2min) AND no break for ≥90min.
|
|
return dDesk <= 2*time.Minute && dBreak >= 90*time.Minute
|
|
},
|
|
}
|
|
}
|
|
|
|
// ServiceDownPrefix — mavpoll writes one fact per kuma monitor under this
|
|
// prefix, `service_down:<monitor name>`. The suffix is the name he hears.
|
|
const ServiceDownPrefix = "service_down:"
|
|
|
|
// ServiceDownSource — kuma is the source of truth for service up/down. The
|
|
// rule is provenance-scoped: a poller writing under a different source cannot
|
|
// forge the trigger.
|
|
const ServiceDownSource = "poll:uptimekuma"
|
|
|
|
// DownServices — the monitors currently reading "down", by name, in key order.
|
|
//
|
|
// Pure, and the rule and the phraser both call it, so the message can never
|
|
// name a service the predicate did not fire on.
|
|
func DownServices(s State) []string {
|
|
var out []string
|
|
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
|
if f.Source == ServiceDownSource && f.Value == `"down"` {
|
|
out = append(out, strings.TrimPrefix(f.Key, ServiceDownPrefix))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// MinDownAge — how long a monitor must have read "down" before it is worth
|
|
// waking him (Vikunja #536).
|
|
//
|
|
// Technitium read down on one kuma poll and up on the next, sixty seconds
|
|
// apart, and the alarm arrived after the service was already back. mavpoll
|
|
// writes a service_down fact only when the state CHANGES, so the fact's
|
|
// timestamp is the instant the monitor went down and its age is how long it
|
|
// has stayed there. That is the whole debounce: no history to keep, no counter
|
|
// to persist.
|
|
//
|
|
// Ninety seconds is one poll interval plus room for jitter, so a monitor must
|
|
// survive at least one further poll as down. The cost is up to ninety seconds
|
|
// of alarm latency on a real outage, against never being paged for a blip.
|
|
//
|
|
// It bounds the alarm, not the truth: DownServices still reports a monitor the
|
|
// instant it goes down, because /dash showing a fresh outage is right even
|
|
// when phoning him about it is not.
|
|
const MinDownAge = 90 * time.Second
|
|
|
|
// downLongEnough — the newest down fact that has aged past MinDownAge, or the
|
|
// zero time when no monitor has. The rule fires off this and not off the
|
|
// newest down fact outright.
|
|
func downLongEnough(s State, now time.Time) time.Time {
|
|
var newest time.Time
|
|
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
|
if f.Source != ServiceDownSource || f.Value != `"down"` {
|
|
continue
|
|
}
|
|
if now.Sub(f.Ts) < MinDownAge {
|
|
continue
|
|
}
|
|
if f.Ts.After(newest) {
|
|
newest = f.Ts
|
|
}
|
|
}
|
|
return newest
|
|
}
|
|
|
|
// ServiceDownRule — sev4 ops hard: at least one kuma monitor reads "down".
|
|
//
|
|
// It used to read one aggregate `service_down` fact, which is why it was
|
|
// disabled in deploy: the nudge could say that something on homesrv was down
|
|
// but never which thing. Per-monitor facts fix that, and pausing a monitor in
|
|
// kuma now silences that monitor rather than nothing.
|
|
//
|
|
// Edge-triggered — see State.NudgedSince. Without it a service that stays down
|
|
// for a day qualifies on every tick and cooldown alone is the only brake.
|
|
func ServiceDownRule() Rule {
|
|
return Rule{
|
|
Name: "service_down",
|
|
Severity: Sev4,
|
|
Cooldown: Cooldown{Base: 15 * time.Minute, Min: 5 * time.Minute, Max: 1 * time.Hour},
|
|
WantPrefixes: []string{ServiceDownPrefix},
|
|
Predicate: func(s State) bool {
|
|
newest := downLongEnough(s, s.Now)
|
|
if newest.IsZero() {
|
|
// Nothing down, no data at all, or nothing down long enough
|
|
// to be more than a flap → shut up. See MinDownAge.
|
|
return false
|
|
}
|
|
return !s.NudgedSince("service_down", newest)
|
|
},
|
|
// The condition without the edge trigger. DownServices is the same
|
|
// helper the predicate and the phraser read, so the repeat stops on
|
|
// exactly the monitors he was told about.
|
|
StillTrue: func(s State) bool { return len(DownServices(s)) > 0 },
|
|
}
|
|
}
|
|
|
|
// NetdataCriticalRule — sev3 ops soft: netdata has a CRITICAL alarm active
|
|
// (disk/mem/cert/temp). The `netdata_alarm` aggregate fact (mavpoll, source
|
|
// poll:netdata) reads "critical". Sev3 not sev4: netdata resource alarms are
|
|
// "look soon", not "wake me" — a full disk matters, but kuma's service_down is
|
|
// the hard page. Provenance-scoped to poll:netdata.
|
|
func NetdataCriticalRule() Rule {
|
|
return Rule{
|
|
Name: "netdata_critical",
|
|
Severity: Sev3,
|
|
Cooldown: Cooldown{Base: 20 * time.Minute, Min: 10 * time.Minute, Max: 2 * time.Hour},
|
|
InertWhenNoData: []string{"netdata_alarm"},
|
|
Predicate: func(s State) bool {
|
|
f, ok := s.Fact("netdata_alarm")
|
|
if !ok || f.Ts.IsZero() {
|
|
return false
|
|
}
|
|
return f.Source == "poll:netdata" && f.Value == `"critical"`
|
|
},
|
|
}
|
|
}
|
|
|
|
// RulesExcept returns DefaultRules minus the named ones (config's
|
|
// `disabled_rules`). Config subtracts from the canonical set; it never adds to
|
|
// it and never reorders it, so the "rules are code" line above still holds.
|
|
//
|
|
// Names are matched exactly and an unknown one is ignored, on purpose: a
|
|
// config that still lists a rule someone deleted must not stop the daemon from
|
|
// booting. The logging of what was actually dropped belongs to the caller,
|
|
// which knows whether anyone is listening.
|
|
func RulesExcept(disabled []string) ([]Rule, []string) {
|
|
all := DefaultRules()
|
|
if len(disabled) == 0 {
|
|
return all, nil
|
|
}
|
|
off := make(map[string]bool, len(disabled))
|
|
for _, n := range disabled {
|
|
if n = strings.TrimSpace(n); n != "" {
|
|
off[n] = true
|
|
}
|
|
}
|
|
out := make([]Rule, 0, len(all))
|
|
var dropped []string
|
|
for _, r := range all {
|
|
if off[r.Name] {
|
|
dropped = append(dropped, r.Name)
|
|
continue
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out, dropped
|
|
}
|
|
|
|
// DefaultRules — the canonical set the daemon wires. Add more as code, not config.
|
|
// Order here is NOT load-bearing — the loop picks max severity, ties broken by
|
|
// (severity desc, name asc) for deterministic output.
|
|
func DefaultRules() []Rule {
|
|
return []Rule{
|
|
WaterRule(),
|
|
MealRule(),
|
|
BreakRule(),
|
|
ServiceDownRule(),
|
|
NetdataCriticalRule(),
|
|
}
|
|
}
|