8fdb9e5cd1
The kuma service_down nudge cannot name the service. mavpoll folds the whole monitor_status gauge into one boolean fact keyed `service_down`, and the phraser names a service only when the fact key is the service name, so the message is always the generic "a service on homesrv is down". Every fifteen minutes, with nothing to act on. A fact per monitor is the real fix and it is filed as Vikunja #444; this is what to do until then. `disabled_rules` in mavend.json subtracts from loop.DefaultRules by name. Config only subtracts — rules stay code, the set stays canonical and ordered as written. A disabled rule is not gathered for either, since the gatherer derives its key set from the rules it was given. Unknown names are ignored so deleting a rule cannot brick a config that still lists it, and the boot log says what was dropped, because a rule that vanishes silently looks exactly like a rule that is broken. Deploy turns service_down off. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TrVSBKe3RFDF4fGYKWYQnX
186 lines
6.4 KiB
Go
186 lines
6.4 KiB
Go
package loop
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Rule — a proactive rule. Rules are CODE, not a DSL config — until ~30 rules
|
|
// and you feel the pain (per spec). A Rule has a name (ids it in nudges.outcome
|
|
// for the feedback loop), a Severity, a pure Predicate, and a cooldown.
|
|
//
|
|
// The Predicate answers "should this rule want to fire given the State?" —
|
|
// only the check against the snapshot. It is pure, no I/O. The GATE answers
|
|
// "are we allowed to fire it right now?" (quiet hours, cooldown, etc) —
|
|
// applied by the loop, never per-rule.
|
|
//
|
|
// Cooldown is the BASE duration between same-rule nudges. The feedback
|
|
// auto-tuner scales it over time (mostly ignored → lengthen; acted → leave).
|
|
// Bounds belong at the daemon-config level; here we just carry the base.
|
|
type Rule struct {
|
|
Name string
|
|
Severity Severity
|
|
Cooldown // base cooldown + bounded-duration envelope for the auto-tuner
|
|
Predicate func(State) bool
|
|
|
|
// InertWhenNoData — most rules should be silent when their substrate key is
|
|
// missing (since(key)==null → don't fire). If the predicate already encodes
|
|
// that check itself, leave this empty. Otherwise set to the key(s) the rule
|
|
// needs and the gate will skip the rule when any are missing.
|
|
InertWhenNoData []string
|
|
}
|
|
|
|
// Cooldown — tunable bounded by the envelope so a weird week (auto-tuned) can't
|
|
// mutate maven silent or stalker. Base is what the rule ships with; Min/Max
|
|
// bound the feedback-driven adjustments persisted as `facts (source=feedback)`.
|
|
type Cooldown struct {
|
|
Base time.Duration
|
|
Min time.Duration
|
|
Max time.Duration
|
|
}
|
|
|
|
// Canonical care/ops rules — NOT a config DSL. Code, so the predicate is
|
|
// inspectable and unit-tested. Daemon wires these up; the loop just iterates.
|
|
|
|
// WaterRule — sev1 care: if it's been ≥3h since a `water` fact, fire.
|
|
// Inert when no water fact exists at all (shuts up when uncertain).
|
|
func WaterRule() Rule {
|
|
return Rule{
|
|
Name: "water",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 30 * time.Minute, Min: 15 * time.Minute, Max: 6 * time.Hour},
|
|
InertWhenNoData: []string{"water"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("water")
|
|
if !ok {
|
|
return false // no data → shut up
|
|
}
|
|
return d >= 3*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// MealRule — sev1 care: if ≥6h since an `meal` fact, fire. Inert without data.
|
|
func MealRule() Rule {
|
|
return Rule{
|
|
Name: "meal",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 60 * time.Minute, Min: 30 * time.Minute, Max: 8 * time.Hour},
|
|
InertWhenNoData: []string{"meal"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("meal")
|
|
if !ok {
|
|
return false
|
|
}
|
|
return d >= 6*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// BreakRule — sev2 care: ≥90min of continuous desk activity without a break.
|
|
// Reads both `desk_active` (fresh input ⇒ at desk) and `break` (last taken).
|
|
// Inert unless both exist — can't claim continuous activity without both anchors.
|
|
func BreakRule() Rule {
|
|
return Rule{
|
|
Name: "break",
|
|
Severity: Sev2,
|
|
Cooldown: Cooldown{Base: 45 * time.Minute, Min: 20 * time.Minute, Max: 4 * time.Hour},
|
|
InertWhenNoData: []string{"desk_active", "break"},
|
|
Predicate: func(s State) bool {
|
|
dDesk, ok1 := s.Since("desk_active")
|
|
dBreak, ok2 := s.Since("break")
|
|
if !ok1 || !ok2 {
|
|
return false // shut up until we have both anchors
|
|
}
|
|
// at desk (fresh input within 2min) AND no break for ≥90min.
|
|
return dDesk <= 2*time.Minute && dBreak >= 90*time.Minute
|
|
},
|
|
}
|
|
}
|
|
|
|
// ServiceDownRule — sev4 ops hard: the `service_down` aggregate fact reads
|
|
// "down". Source must be poll:uptimekuma — kuma is the source of truth for
|
|
// service up/down (mavpoll writes this key). The predicate is provenance-scoped:
|
|
// a compromised poller writing under a different source can't forge the trigger.
|
|
func ServiceDownRule() Rule {
|
|
return Rule{
|
|
Name: "service_down",
|
|
Severity: Sev4,
|
|
Cooldown: Cooldown{Base: 15 * time.Minute, Min: 5 * time.Minute, Max: 1 * time.Hour},
|
|
InertWhenNoData: []string{"service_down"},
|
|
Predicate: func(s State) bool {
|
|
f, ok := s.Fact("service_down")
|
|
if !ok || f.Ts.IsZero() {
|
|
return false
|
|
}
|
|
// value is json `"down"`; trivial check keyed off source provenance.
|
|
return f.Source == "poll:uptimekuma" && f.Value == `"down"`
|
|
},
|
|
}
|
|
}
|
|
|
|
// NetdataCriticalRule — sev3 ops soft: netdata has a CRITICAL alarm active
|
|
// (disk/mem/cert/temp). The `netdata_alarm` aggregate fact (mavpoll, source
|
|
// poll:netdata) reads "critical". Sev3 not sev4: netdata resource alarms are
|
|
// "look soon", not "wake me" — a full disk matters, but kuma's service_down is
|
|
// the hard page. Provenance-scoped to poll:netdata.
|
|
func NetdataCriticalRule() Rule {
|
|
return Rule{
|
|
Name: "netdata_critical",
|
|
Severity: Sev3,
|
|
Cooldown: Cooldown{Base: 20 * time.Minute, Min: 10 * time.Minute, Max: 2 * time.Hour},
|
|
InertWhenNoData: []string{"netdata_alarm"},
|
|
Predicate: func(s State) bool {
|
|
f, ok := s.Fact("netdata_alarm")
|
|
if !ok || f.Ts.IsZero() {
|
|
return false
|
|
}
|
|
return f.Source == "poll:netdata" && f.Value == `"critical"`
|
|
},
|
|
}
|
|
}
|
|
|
|
// RulesExcept returns DefaultRules minus the named ones (config's
|
|
// `disabled_rules`). Config subtracts from the canonical set; it never adds to
|
|
// it and never reorders it, so the "rules are code" line above still holds.
|
|
//
|
|
// Names are matched exactly and an unknown one is ignored, on purpose: a
|
|
// config that still lists a rule someone deleted must not stop the daemon from
|
|
// booting. The logging of what was actually dropped belongs to the caller,
|
|
// which knows whether anyone is listening.
|
|
func RulesExcept(disabled []string) ([]Rule, []string) {
|
|
all := DefaultRules()
|
|
if len(disabled) == 0 {
|
|
return all, nil
|
|
}
|
|
off := make(map[string]bool, len(disabled))
|
|
for _, n := range disabled {
|
|
if n = strings.TrimSpace(n); n != "" {
|
|
off[n] = true
|
|
}
|
|
}
|
|
out := make([]Rule, 0, len(all))
|
|
var dropped []string
|
|
for _, r := range all {
|
|
if off[r.Name] {
|
|
dropped = append(dropped, r.Name)
|
|
continue
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out, dropped
|
|
}
|
|
|
|
// DefaultRules — the canonical set the daemon wires. Add more as code, not config.
|
|
// Order here is NOT load-bearing — the loop picks max severity, ties broken by
|
|
// (severity desc, name asc) for deterministic output.
|
|
func DefaultRules() []Rule {
|
|
return []Rule{
|
|
WaterRule(),
|
|
MealRule(),
|
|
BreakRule(),
|
|
ServiceDownRule(),
|
|
NetdataCriticalRule(),
|
|
}
|
|
}
|