4f516657da
A rule over a key set that only exists at read time cannot declare its keys at wiring time. Kuma has one monitor per service and the names live in the gauge, so the rule declares a prefix and the gatherer resolves the family per tick. ServiceDownRule now fires on any monitor reading down, names it through DownServices, and is edge-triggered: a service that stays down is one nudge, not one per tick with cooldown as the only brake.
228 lines
7.9 KiB
Go
228 lines
7.9 KiB
Go
package loop
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Rule — a proactive rule. Rules are CODE, not a DSL config — until ~30 rules
|
|
// and you feel the pain (per spec). A Rule has a name (ids it in nudges.outcome
|
|
// for the feedback loop), a Severity, a pure Predicate, and a cooldown.
|
|
//
|
|
// The Predicate answers "should this rule want to fire given the State?" —
|
|
// only the check against the snapshot. It is pure, no I/O. The GATE answers
|
|
// "are we allowed to fire it right now?" (quiet hours, cooldown, etc) —
|
|
// applied by the loop, never per-rule.
|
|
//
|
|
// Cooldown is the BASE duration between same-rule nudges. The feedback
|
|
// auto-tuner scales it over time (mostly ignored → lengthen; acted → leave).
|
|
// Bounds belong at the daemon-config level; here we just carry the base.
|
|
type Rule struct {
|
|
Name string
|
|
Severity Severity
|
|
Cooldown // base cooldown + bounded-duration envelope for the auto-tuner
|
|
Predicate func(State) bool
|
|
|
|
// InertWhenNoData — most rules should be silent when their substrate key is
|
|
// missing (since(key)==null → don't fire). If the predicate already encodes
|
|
// that check itself, leave this empty. Otherwise set to the key(s) the rule
|
|
// needs and the gate will skip the rule when any are missing.
|
|
InertWhenNoData []string
|
|
|
|
// WantPrefixes — key prefixes whose whole family the gatherer must load.
|
|
// InertWhenNoData names keys that exist at wiring time; a rule over a key
|
|
// set that is only known at read time (one fact per kuma monitor) declares
|
|
// the prefix here instead. Prefixes never make a rule inert: an empty
|
|
// family is the predicate's own "no data" case.
|
|
WantPrefixes []string
|
|
}
|
|
|
|
// Cooldown — tunable bounded by the envelope so a weird week (auto-tuned) can't
|
|
// mutate maven silent or stalker. Base is what the rule ships with; Min/Max
|
|
// bound the feedback-driven adjustments persisted as `facts (source=feedback)`.
|
|
type Cooldown struct {
|
|
Base time.Duration
|
|
Min time.Duration
|
|
Max time.Duration
|
|
}
|
|
|
|
// Canonical care/ops rules — NOT a config DSL. Code, so the predicate is
|
|
// inspectable and unit-tested. Daemon wires these up; the loop just iterates.
|
|
|
|
// WaterRule — sev1 care: if it's been ≥3h since a `water` fact, fire.
|
|
// Inert when no water fact exists at all (shuts up when uncertain).
|
|
func WaterRule() Rule {
|
|
return Rule{
|
|
Name: "water",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 30 * time.Minute, Min: 15 * time.Minute, Max: 6 * time.Hour},
|
|
InertWhenNoData: []string{"water"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("water")
|
|
if !ok {
|
|
return false // no data → shut up
|
|
}
|
|
return d >= 3*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// MealRule — sev1 care: if ≥6h since an `meal` fact, fire. Inert without data.
|
|
func MealRule() Rule {
|
|
return Rule{
|
|
Name: "meal",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 60 * time.Minute, Min: 30 * time.Minute, Max: 8 * time.Hour},
|
|
InertWhenNoData: []string{"meal"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("meal")
|
|
if !ok {
|
|
return false
|
|
}
|
|
return d >= 6*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// BreakRule — sev2 care: ≥90min of continuous desk activity without a break.
|
|
// Reads both `desk_active` (fresh input ⇒ at desk) and `break` (last taken).
|
|
// Inert unless both exist — can't claim continuous activity without both anchors.
|
|
func BreakRule() Rule {
|
|
return Rule{
|
|
Name: "break",
|
|
Severity: Sev2,
|
|
Cooldown: Cooldown{Base: 45 * time.Minute, Min: 20 * time.Minute, Max: 4 * time.Hour},
|
|
InertWhenNoData: []string{"desk_active", "break"},
|
|
Predicate: func(s State) bool {
|
|
dDesk, ok1 := s.Since("desk_active")
|
|
dBreak, ok2 := s.Since("break")
|
|
if !ok1 || !ok2 {
|
|
return false // shut up until we have both anchors
|
|
}
|
|
// at desk (fresh input within 2min) AND no break for ≥90min.
|
|
return dDesk <= 2*time.Minute && dBreak >= 90*time.Minute
|
|
},
|
|
}
|
|
}
|
|
|
|
// ServiceDownPrefix — mavpoll writes one fact per kuma monitor under this
|
|
// prefix, `service_down:<monitor name>`. The suffix is the name he hears.
|
|
const ServiceDownPrefix = "service_down:"
|
|
|
|
// ServiceDownSource — kuma is the source of truth for service up/down. The
|
|
// rule is provenance-scoped: a poller writing under a different source cannot
|
|
// forge the trigger.
|
|
const ServiceDownSource = "poll:uptimekuma"
|
|
|
|
// DownServices — the monitors currently reading "down", by name, in key order.
|
|
//
|
|
// Pure, and the rule and the phraser both call it, so the message can never
|
|
// name a service the predicate did not fire on.
|
|
func DownServices(s State) []string {
|
|
var out []string
|
|
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
|
if f.Source == ServiceDownSource && f.Value == `"down"` {
|
|
out = append(out, strings.TrimPrefix(f.Key, ServiceDownPrefix))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ServiceDownRule — sev4 ops hard: at least one kuma monitor reads "down".
|
|
//
|
|
// It used to read one aggregate `service_down` fact, which is why it was
|
|
// disabled in deploy: the nudge could say that something on homesrv was down
|
|
// but never which thing. Per-monitor facts fix that, and pausing a monitor in
|
|
// kuma now silences that monitor rather than nothing.
|
|
//
|
|
// Edge-triggered — see State.NudgedSince. Without it a service that stays down
|
|
// for a day qualifies on every tick and cooldown alone is the only brake.
|
|
func ServiceDownRule() Rule {
|
|
return Rule{
|
|
Name: "service_down",
|
|
Severity: Sev4,
|
|
Cooldown: Cooldown{Base: 15 * time.Minute, Min: 5 * time.Minute, Max: 1 * time.Hour},
|
|
WantPrefixes: []string{ServiceDownPrefix},
|
|
Predicate: func(s State) bool {
|
|
var newest time.Time
|
|
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
|
if f.Source != ServiceDownSource || f.Value != `"down"` {
|
|
continue
|
|
}
|
|
if f.Ts.After(newest) {
|
|
newest = f.Ts
|
|
}
|
|
}
|
|
if newest.IsZero() {
|
|
return false // nothing down, or no data at all → shut up
|
|
}
|
|
return !s.NudgedSince("service_down", newest)
|
|
},
|
|
}
|
|
}
|
|
|
|
// NetdataCriticalRule — sev3 ops soft: netdata has a CRITICAL alarm active
|
|
// (disk/mem/cert/temp). The `netdata_alarm` aggregate fact (mavpoll, source
|
|
// poll:netdata) reads "critical". Sev3 not sev4: netdata resource alarms are
|
|
// "look soon", not "wake me" — a full disk matters, but kuma's service_down is
|
|
// the hard page. Provenance-scoped to poll:netdata.
|
|
func NetdataCriticalRule() Rule {
|
|
return Rule{
|
|
Name: "netdata_critical",
|
|
Severity: Sev3,
|
|
Cooldown: Cooldown{Base: 20 * time.Minute, Min: 10 * time.Minute, Max: 2 * time.Hour},
|
|
InertWhenNoData: []string{"netdata_alarm"},
|
|
Predicate: func(s State) bool {
|
|
f, ok := s.Fact("netdata_alarm")
|
|
if !ok || f.Ts.IsZero() {
|
|
return false
|
|
}
|
|
return f.Source == "poll:netdata" && f.Value == `"critical"`
|
|
},
|
|
}
|
|
}
|
|
|
|
// RulesExcept returns DefaultRules minus the named ones (config's
|
|
// `disabled_rules`). Config subtracts from the canonical set; it never adds to
|
|
// it and never reorders it, so the "rules are code" line above still holds.
|
|
//
|
|
// Names are matched exactly and an unknown one is ignored, on purpose: a
|
|
// config that still lists a rule someone deleted must not stop the daemon from
|
|
// booting. The logging of what was actually dropped belongs to the caller,
|
|
// which knows whether anyone is listening.
|
|
func RulesExcept(disabled []string) ([]Rule, []string) {
|
|
all := DefaultRules()
|
|
if len(disabled) == 0 {
|
|
return all, nil
|
|
}
|
|
off := make(map[string]bool, len(disabled))
|
|
for _, n := range disabled {
|
|
if n = strings.TrimSpace(n); n != "" {
|
|
off[n] = true
|
|
}
|
|
}
|
|
out := make([]Rule, 0, len(all))
|
|
var dropped []string
|
|
for _, r := range all {
|
|
if off[r.Name] {
|
|
dropped = append(dropped, r.Name)
|
|
continue
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out, dropped
|
|
}
|
|
|
|
// DefaultRules — the canonical set the daemon wires. Add more as code, not config.
|
|
// Order here is NOT load-bearing — the loop picks max severity, ties broken by
|
|
// (severity desc, name asc) for deterministic output.
|
|
func DefaultRules() []Rule {
|
|
return []Rule{
|
|
WaterRule(),
|
|
MealRule(),
|
|
BreakRule(),
|
|
ServiceDownRule(),
|
|
NetdataCriticalRule(),
|
|
}
|
|
}
|