5c01fe338b
The digest needs to know whether a candidate is already pending before it pays the phraser, and prose is not identity: phrasing varies, and State.Now advancing does not turn the same unmet condition into a new event. A rule eligible for the digest declares DigestIdentity beside its predicate. DigestCandidateFingerprint frames the rule name and severity around it so two rules cannot alias on a shared fact. BreakRule anchors on the last completed break, not on desk_active, which the poller refreshes without the unmet need changing. A rule that declares no identity does not enter the digest, since a generic state hash would either change every tick or ignore an input the rule reads.
298 lines
11 KiB
Go
298 lines
11 KiB
Go
package loop
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Rule — a proactive rule. Rules are CODE, not a DSL config — until ~30 rules
|
|
// and you feel the pain (per spec). A Rule has a name (ids it in nudges.outcome
|
|
// for the feedback loop), a Severity, a pure Predicate, and a cooldown.
|
|
//
|
|
// The Predicate answers "should this rule want to fire given the State?" —
|
|
// only the check against the snapshot. It is pure, no I/O. The GATE answers
|
|
// "are we allowed to fire it right now?" (quiet hours, cooldown, etc) —
|
|
// applied by the loop, never per-rule.
|
|
//
|
|
// Cooldown is the BASE duration between same-rule nudges. The feedback
|
|
// auto-tuner scales it over time (mostly ignored → lengthen; acted → leave).
|
|
// Bounds belong at the daemon-config level; here we just carry the base.
|
|
type Rule struct {
|
|
Name string
|
|
Severity Severity
|
|
Cooldown // base cooldown + bounded-duration envelope for the auto-tuner
|
|
Predicate func(State) bool
|
|
|
|
// DigestIdentity names one semantic occurrence while this rule is held by
|
|
// quiet hours, presence, or calendar restraint. The durable digest checks
|
|
// its fingerprint before asking the phraser for prose, so it must change
|
|
// when the candidate's meaning changes and remain stable while only Now
|
|
// advances. Only rules eligible for that digest need to declare it.
|
|
DigestIdentity DigestIdentity
|
|
|
|
// InertWhenNoData — most rules should be silent when their substrate key is
|
|
// missing (since(key)==null → don't fire). If the predicate already encodes
|
|
// that check itself, leave this empty. Otherwise set to the key(s) the rule
|
|
// needs and the gate will skip the rule when any are missing.
|
|
InertWhenNoData []string
|
|
|
|
// WantPrefixes — key prefixes whose whole family the gatherer must load.
|
|
// InertWhenNoData names keys that exist at wiring time; a rule over a key
|
|
// set that is only known at read time (one fact per kuma monitor) declares
|
|
// the prefix here instead. Prefixes never make a rule inert: an empty
|
|
// family is the predicate's own "no data" case.
|
|
WantPrefixes []string
|
|
|
|
// StillTrue — is the CONDITION still true, ignoring whether it is worth
|
|
// saying again? Distinct from Predicate on purpose, and the distinction is
|
|
// the whole reason this field exists (Vikunja #535).
|
|
//
|
|
// Predicate answers "should this fire now", which folds in edge-triggering:
|
|
// ServiceDownRule ends in !s.NudgedSince(...), so it reads false the instant
|
|
// a nudge goes out even though the service is still down. A repeat loop that
|
|
// consulted Predicate would cancel every alarm one tick after raising it,
|
|
// which is exactly backwards.
|
|
//
|
|
// Only a rule whose alarm repeats needs this. nil means "I cannot tell you",
|
|
// and the caller must then fall back to a bound it can enforce without the
|
|
// rule's help. nil must never be read as "the condition cleared": a rule
|
|
// that says nothing about its condition is not a rule that resolved.
|
|
StillTrue func(State) bool
|
|
}
|
|
|
|
// Cooldown — tunable bounded by the envelope so a weird week (auto-tuned) can't
|
|
// mutate maven silent or stalker. Base is what the rule ships with; Min/Max
|
|
// bound the feedback-driven adjustments persisted as `facts (source=feedback)`.
|
|
type Cooldown struct {
|
|
Base time.Duration
|
|
Min time.Duration
|
|
Max time.Duration
|
|
}
|
|
|
|
// Canonical care/ops rules — NOT a config DSL. Code, so the predicate is
|
|
// inspectable and unit-tested. Daemon wires these up; the loop just iterates.
|
|
|
|
// WaterRule — sev1 care: if it's been ≥3h since a `water` fact, fire.
|
|
// Inert when no water fact exists at all (shuts up when uncertain).
|
|
func WaterRule() Rule {
|
|
return Rule{
|
|
Name: "water",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 30 * time.Minute, Min: 15 * time.Minute, Max: 6 * time.Hour},
|
|
InertWhenNoData: []string{"water"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("water")
|
|
if !ok {
|
|
return false // no data → shut up
|
|
}
|
|
return d >= 3*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// MealRule — sev1 care: if ≥6h since an `meal` fact, fire. Inert without data.
|
|
func MealRule() Rule {
|
|
return Rule{
|
|
Name: "meal",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 60 * time.Minute, Min: 30 * time.Minute, Max: 8 * time.Hour},
|
|
InertWhenNoData: []string{"meal"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("meal")
|
|
if !ok {
|
|
return false
|
|
}
|
|
return d >= 6*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// BreakRule — sev2 care: ≥90min of continuous desk activity without a break.
|
|
// Reads both `desk_active` (fresh input ⇒ at desk) and `break` (last taken).
|
|
// Inert unless both exist — can't claim continuous activity without both anchors.
|
|
func BreakRule() Rule {
|
|
return Rule{
|
|
Name: "break",
|
|
Severity: Sev2,
|
|
Cooldown: Cooldown{Base: 45 * time.Minute, Min: 20 * time.Minute, Max: 4 * time.Hour},
|
|
InertWhenNoData: []string{"desk_active", "break"},
|
|
// The last completed break is the semantic anchor for this stretch.
|
|
// desk_active is freshness evidence for the predicate and may be
|
|
// refreshed every poll; treating each refresh as a new occurrence would
|
|
// defeat durable dedupe even though the unmet need did not change.
|
|
DigestIdentity: func(s State) []byte {
|
|
lastBreak, ok := s.Fact("break")
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return factDigestIdentity(lastBreak)
|
|
},
|
|
Predicate: func(s State) bool {
|
|
dDesk, ok1 := s.Since("desk_active")
|
|
dBreak, ok2 := s.Since("break")
|
|
if !ok1 || !ok2 {
|
|
return false // shut up until we have both anchors
|
|
}
|
|
// at desk (fresh input within 2min) AND no break for ≥90min.
|
|
return dDesk <= 2*time.Minute && dBreak >= 90*time.Minute
|
|
},
|
|
}
|
|
}
|
|
|
|
// ServiceDownPrefix — mavpoll writes one fact per kuma monitor under this
|
|
// prefix, `service_down:<monitor name>`. The suffix is the name he hears.
|
|
const ServiceDownPrefix = "service_down:"
|
|
|
|
// ServiceDownSource — kuma is the source of truth for service up/down. The
|
|
// rule is provenance-scoped: a poller writing under a different source cannot
|
|
// forge the trigger.
|
|
const ServiceDownSource = "poll:uptimekuma"
|
|
|
|
// DownServices — the monitors currently reading "down", by name, in key order.
|
|
//
|
|
// Pure, and the rule and the phraser both call it, so the message can never
|
|
// name a service the predicate did not fire on.
|
|
func DownServices(s State) []string {
|
|
var out []string
|
|
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
|
if f.Source == ServiceDownSource && f.Value == `"down"` {
|
|
out = append(out, strings.TrimPrefix(f.Key, ServiceDownPrefix))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// MinDownAge — how long a monitor must have read "down" before it is worth
|
|
// waking him (Vikunja #536).
|
|
//
|
|
// Technitium read down on one kuma poll and up on the next, sixty seconds
|
|
// apart, and the alarm arrived after the service was already back. mavpoll
|
|
// writes a service_down fact only when the state CHANGES, so the fact's
|
|
// timestamp is the instant the monitor went down and its age is how long it
|
|
// has stayed there. That is the whole debounce: no history to keep, no counter
|
|
// to persist.
|
|
//
|
|
// Ninety seconds is one poll interval plus room for jitter, so a monitor must
|
|
// survive at least one further poll as down. The cost is up to ninety seconds
|
|
// of alarm latency on a real outage, against never being paged for a blip.
|
|
//
|
|
// It bounds the alarm, not the truth: DownServices still reports a monitor the
|
|
// instant it goes down, because /dash showing a fresh outage is right even
|
|
// when phoning him about it is not.
|
|
const MinDownAge = 90 * time.Second
|
|
|
|
// downLongEnough — the newest down fact that has aged past MinDownAge, or the
|
|
// zero time when no monitor has. The rule fires off this and not off the
|
|
// newest down fact outright.
|
|
func downLongEnough(s State, now time.Time) time.Time {
|
|
var newest time.Time
|
|
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
|
if f.Source != ServiceDownSource || f.Value != `"down"` {
|
|
continue
|
|
}
|
|
if now.Sub(f.Ts) < MinDownAge {
|
|
continue
|
|
}
|
|
if f.Ts.After(newest) {
|
|
newest = f.Ts
|
|
}
|
|
}
|
|
return newest
|
|
}
|
|
|
|
// ServiceDownRule — sev4 ops hard: at least one kuma monitor reads "down".
|
|
//
|
|
// It used to read one aggregate `service_down` fact, which is why it was
|
|
// disabled in deploy: the nudge could say that something on homesrv was down
|
|
// but never which thing. Per-monitor facts fix that, and pausing a monitor in
|
|
// kuma now silences that monitor rather than nothing.
|
|
//
|
|
// Edge-triggered — see State.NudgedSince. Without it a service that stays down
|
|
// for a day qualifies on every tick and cooldown alone is the only brake.
|
|
func ServiceDownRule() Rule {
|
|
return Rule{
|
|
Name: "service_down",
|
|
Severity: Sev4,
|
|
Cooldown: Cooldown{Base: 15 * time.Minute, Min: 5 * time.Minute, Max: 1 * time.Hour},
|
|
WantPrefixes: []string{ServiceDownPrefix},
|
|
Predicate: func(s State) bool {
|
|
newest := downLongEnough(s, s.Now)
|
|
if newest.IsZero() {
|
|
// Nothing down, no data at all, or nothing down long enough
|
|
// to be more than a flap → shut up. See MinDownAge.
|
|
return false
|
|
}
|
|
return !s.NudgedSince("service_down", newest)
|
|
},
|
|
// The condition without the edge trigger. DownServices is the same
|
|
// helper the predicate and the phraser read, so the repeat stops on
|
|
// exactly the monitors he was told about.
|
|
StillTrue: func(s State) bool { return len(DownServices(s)) > 0 },
|
|
}
|
|
}
|
|
|
|
// NetdataCriticalRule — sev3 ops soft: netdata has a CRITICAL alarm active
|
|
// (disk/mem/cert/temp). The `netdata_alarm` aggregate fact (mavpoll, source
|
|
// poll:netdata) reads "critical". Sev3 not sev4: netdata resource alarms are
|
|
// "look soon", not "wake me" — a full disk matters, but kuma's service_down is
|
|
// the hard page. Provenance-scoped to poll:netdata.
|
|
func NetdataCriticalRule() Rule {
|
|
return Rule{
|
|
Name: "netdata_critical",
|
|
Severity: Sev3,
|
|
Cooldown: Cooldown{Base: 20 * time.Minute, Min: 10 * time.Minute, Max: 2 * time.Hour},
|
|
InertWhenNoData: []string{"netdata_alarm"},
|
|
Predicate: func(s State) bool {
|
|
f, ok := s.Fact("netdata_alarm")
|
|
if !ok || f.Ts.IsZero() {
|
|
return false
|
|
}
|
|
return f.Source == "poll:netdata" && f.Value == `"critical"`
|
|
},
|
|
}
|
|
}
|
|
|
|
// RulesExcept returns DefaultRules minus the named ones (config's
|
|
// `disabled_rules`). Config subtracts from the canonical set; it never adds to
|
|
// it and never reorders it, so the "rules are code" line above still holds.
|
|
//
|
|
// Names are matched exactly and an unknown one is ignored, on purpose: a
|
|
// config that still lists a rule someone deleted must not stop the daemon from
|
|
// booting. The logging of what was actually dropped belongs to the caller,
|
|
// which knows whether anyone is listening.
|
|
func RulesExcept(disabled []string) ([]Rule, []string) {
|
|
all := DefaultRules()
|
|
if len(disabled) == 0 {
|
|
return all, nil
|
|
}
|
|
off := make(map[string]bool, len(disabled))
|
|
for _, n := range disabled {
|
|
if n = strings.TrimSpace(n); n != "" {
|
|
off[n] = true
|
|
}
|
|
}
|
|
out := make([]Rule, 0, len(all))
|
|
var dropped []string
|
|
for _, r := range all {
|
|
if off[r.Name] {
|
|
dropped = append(dropped, r.Name)
|
|
continue
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out, dropped
|
|
}
|
|
|
|
// DefaultRules — the canonical set the daemon wires. Add more as code, not config.
|
|
// Order here is NOT load-bearing — the loop picks max severity, ties broken by
|
|
// (severity desc, name asc) for deterministic output.
|
|
func DefaultRules() []Rule {
|
|
return []Rule{
|
|
WaterRule(),
|
|
MealRule(),
|
|
BreakRule(),
|
|
ServiceDownRule(),
|
|
NetdataCriticalRule(),
|
|
}
|
|
}
|