8a13d189bb
It repeated every five minutes for over two hours. WasAcked was the only stop condition and nothing reachable from telegram can mark a nudge acked — the only ack is a voice 'готово' on a box that runs no voice loop. Two endings now. The condition cleared, which the rule answers through the new Rule.StillTrue — deliberately not Predicate, which is edge-triggered and reads false one tick after the alarm is raised, so building the stop on it would cancel every alarm immediately. Or the alarm got old, which is the bound that needs no cooperation from the rule. A rule with no StillTrue is never read as resolved and stops only on age. Covered by tests including the flap case, since none of it can be reproduced by hand without waiting hours. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
248 lines
9.1 KiB
Go
248 lines
9.1 KiB
Go
package loop
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Rule — a proactive rule. Rules are CODE, not a DSL config — until ~30 rules
|
|
// and you feel the pain (per spec). A Rule has a name (ids it in nudges.outcome
|
|
// for the feedback loop), a Severity, a pure Predicate, and a cooldown.
|
|
//
|
|
// The Predicate answers "should this rule want to fire given the State?" —
|
|
// only the check against the snapshot. It is pure, no I/O. The GATE answers
|
|
// "are we allowed to fire it right now?" (quiet hours, cooldown, etc) —
|
|
// applied by the loop, never per-rule.
|
|
//
|
|
// Cooldown is the BASE duration between same-rule nudges. The feedback
|
|
// auto-tuner scales it over time (mostly ignored → lengthen; acted → leave).
|
|
// Bounds belong at the daemon-config level; here we just carry the base.
|
|
type Rule struct {
|
|
Name string
|
|
Severity Severity
|
|
Cooldown // base cooldown + bounded-duration envelope for the auto-tuner
|
|
Predicate func(State) bool
|
|
|
|
// InertWhenNoData — most rules should be silent when their substrate key is
|
|
// missing (since(key)==null → don't fire). If the predicate already encodes
|
|
// that check itself, leave this empty. Otherwise set to the key(s) the rule
|
|
// needs and the gate will skip the rule when any are missing.
|
|
InertWhenNoData []string
|
|
|
|
// WantPrefixes — key prefixes whose whole family the gatherer must load.
|
|
// InertWhenNoData names keys that exist at wiring time; a rule over a key
|
|
// set that is only known at read time (one fact per kuma monitor) declares
|
|
// the prefix here instead. Prefixes never make a rule inert: an empty
|
|
// family is the predicate's own "no data" case.
|
|
WantPrefixes []string
|
|
|
|
// StillTrue — is the CONDITION still true, ignoring whether it is worth
|
|
// saying again? Distinct from Predicate on purpose, and the distinction is
|
|
// the whole reason this field exists (Vikunja #535).
|
|
//
|
|
// Predicate answers "should this fire now", which folds in edge-triggering:
|
|
// ServiceDownRule ends in !s.NudgedSince(...), so it reads false the instant
|
|
// a nudge goes out even though the service is still down. A repeat loop that
|
|
// consulted Predicate would cancel every alarm one tick after raising it,
|
|
// which is exactly backwards.
|
|
//
|
|
// Only a rule whose alarm repeats needs this. nil means "I cannot tell you",
|
|
// and the caller must then fall back to a bound it can enforce without the
|
|
// rule's help. nil must never be read as "the condition cleared": a rule
|
|
// that says nothing about its condition is not a rule that resolved.
|
|
StillTrue func(State) bool
|
|
}
|
|
|
|
// Cooldown — tunable bounded by the envelope so a weird week (auto-tuned) can't
|
|
// mutate maven silent or stalker. Base is what the rule ships with; Min/Max
|
|
// bound the feedback-driven adjustments persisted as `facts (source=feedback)`.
|
|
type Cooldown struct {
|
|
Base time.Duration
|
|
Min time.Duration
|
|
Max time.Duration
|
|
}
|
|
|
|
// Canonical care/ops rules — NOT a config DSL. Code, so the predicate is
|
|
// inspectable and unit-tested. Daemon wires these up; the loop just iterates.
|
|
|
|
// WaterRule — sev1 care: if it's been ≥3h since a `water` fact, fire.
|
|
// Inert when no water fact exists at all (shuts up when uncertain).
|
|
func WaterRule() Rule {
|
|
return Rule{
|
|
Name: "water",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 30 * time.Minute, Min: 15 * time.Minute, Max: 6 * time.Hour},
|
|
InertWhenNoData: []string{"water"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("water")
|
|
if !ok {
|
|
return false // no data → shut up
|
|
}
|
|
return d >= 3*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// MealRule — sev1 care: if ≥6h since an `meal` fact, fire. Inert without data.
|
|
func MealRule() Rule {
|
|
return Rule{
|
|
Name: "meal",
|
|
Severity: Sev1,
|
|
Cooldown: Cooldown{Base: 60 * time.Minute, Min: 30 * time.Minute, Max: 8 * time.Hour},
|
|
InertWhenNoData: []string{"meal"},
|
|
Predicate: func(s State) bool {
|
|
d, ok := s.Since("meal")
|
|
if !ok {
|
|
return false
|
|
}
|
|
return d >= 6*time.Hour
|
|
},
|
|
}
|
|
}
|
|
|
|
// BreakRule — sev2 care: ≥90min of continuous desk activity without a break.
|
|
// Reads both `desk_active` (fresh input ⇒ at desk) and `break` (last taken).
|
|
// Inert unless both exist — can't claim continuous activity without both anchors.
|
|
func BreakRule() Rule {
|
|
return Rule{
|
|
Name: "break",
|
|
Severity: Sev2,
|
|
Cooldown: Cooldown{Base: 45 * time.Minute, Min: 20 * time.Minute, Max: 4 * time.Hour},
|
|
InertWhenNoData: []string{"desk_active", "break"},
|
|
Predicate: func(s State) bool {
|
|
dDesk, ok1 := s.Since("desk_active")
|
|
dBreak, ok2 := s.Since("break")
|
|
if !ok1 || !ok2 {
|
|
return false // shut up until we have both anchors
|
|
}
|
|
// at desk (fresh input within 2min) AND no break for ≥90min.
|
|
return dDesk <= 2*time.Minute && dBreak >= 90*time.Minute
|
|
},
|
|
}
|
|
}
|
|
|
|
// ServiceDownPrefix — mavpoll writes one fact per kuma monitor under this
|
|
// prefix, `service_down:<monitor name>`. The suffix is the name he hears.
|
|
const ServiceDownPrefix = "service_down:"
|
|
|
|
// ServiceDownSource — kuma is the source of truth for service up/down. The
|
|
// rule is provenance-scoped: a poller writing under a different source cannot
|
|
// forge the trigger.
|
|
const ServiceDownSource = "poll:uptimekuma"
|
|
|
|
// DownServices — the monitors currently reading "down", by name, in key order.
|
|
//
|
|
// Pure, and the rule and the phraser both call it, so the message can never
|
|
// name a service the predicate did not fire on.
|
|
func DownServices(s State) []string {
|
|
var out []string
|
|
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
|
if f.Source == ServiceDownSource && f.Value == `"down"` {
|
|
out = append(out, strings.TrimPrefix(f.Key, ServiceDownPrefix))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ServiceDownRule — sev4 ops hard: at least one kuma monitor reads "down".
|
|
//
|
|
// It used to read one aggregate `service_down` fact, which is why it was
|
|
// disabled in deploy: the nudge could say that something on homesrv was down
|
|
// but never which thing. Per-monitor facts fix that, and pausing a monitor in
|
|
// kuma now silences that monitor rather than nothing.
|
|
//
|
|
// Edge-triggered — see State.NudgedSince. Without it a service that stays down
|
|
// for a day qualifies on every tick and cooldown alone is the only brake.
|
|
func ServiceDownRule() Rule {
|
|
return Rule{
|
|
Name: "service_down",
|
|
Severity: Sev4,
|
|
Cooldown: Cooldown{Base: 15 * time.Minute, Min: 5 * time.Minute, Max: 1 * time.Hour},
|
|
WantPrefixes: []string{ServiceDownPrefix},
|
|
Predicate: func(s State) bool {
|
|
var newest time.Time
|
|
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
|
if f.Source != ServiceDownSource || f.Value != `"down"` {
|
|
continue
|
|
}
|
|
if f.Ts.After(newest) {
|
|
newest = f.Ts
|
|
}
|
|
}
|
|
if newest.IsZero() {
|
|
return false // nothing down, or no data at all → shut up
|
|
}
|
|
return !s.NudgedSince("service_down", newest)
|
|
},
|
|
// The condition without the edge trigger. DownServices is the same
|
|
// helper the predicate and the phraser read, so the repeat stops on
|
|
// exactly the monitors he was told about.
|
|
StillTrue: func(s State) bool { return len(DownServices(s)) > 0 },
|
|
}
|
|
}
|
|
|
|
// NetdataCriticalRule — sev3 ops soft: netdata has a CRITICAL alarm active
|
|
// (disk/mem/cert/temp). The `netdata_alarm` aggregate fact (mavpoll, source
|
|
// poll:netdata) reads "critical". Sev3 not sev4: netdata resource alarms are
|
|
// "look soon", not "wake me" — a full disk matters, but kuma's service_down is
|
|
// the hard page. Provenance-scoped to poll:netdata.
|
|
func NetdataCriticalRule() Rule {
|
|
return Rule{
|
|
Name: "netdata_critical",
|
|
Severity: Sev3,
|
|
Cooldown: Cooldown{Base: 20 * time.Minute, Min: 10 * time.Minute, Max: 2 * time.Hour},
|
|
InertWhenNoData: []string{"netdata_alarm"},
|
|
Predicate: func(s State) bool {
|
|
f, ok := s.Fact("netdata_alarm")
|
|
if !ok || f.Ts.IsZero() {
|
|
return false
|
|
}
|
|
return f.Source == "poll:netdata" && f.Value == `"critical"`
|
|
},
|
|
}
|
|
}
|
|
|
|
// RulesExcept returns DefaultRules minus the named ones (config's
|
|
// `disabled_rules`). Config subtracts from the canonical set; it never adds to
|
|
// it and never reorders it, so the "rules are code" line above still holds.
|
|
//
|
|
// Names are matched exactly and an unknown one is ignored, on purpose: a
|
|
// config that still lists a rule someone deleted must not stop the daemon from
|
|
// booting. The logging of what was actually dropped belongs to the caller,
|
|
// which knows whether anyone is listening.
|
|
func RulesExcept(disabled []string) ([]Rule, []string) {
|
|
all := DefaultRules()
|
|
if len(disabled) == 0 {
|
|
return all, nil
|
|
}
|
|
off := make(map[string]bool, len(disabled))
|
|
for _, n := range disabled {
|
|
if n = strings.TrimSpace(n); n != "" {
|
|
off[n] = true
|
|
}
|
|
}
|
|
out := make([]Rule, 0, len(all))
|
|
var dropped []string
|
|
for _, r := range all {
|
|
if off[r.Name] {
|
|
dropped = append(dropped, r.Name)
|
|
continue
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out, dropped
|
|
}
|
|
|
|
// DefaultRules — the canonical set the daemon wires. Add more as code, not config.
|
|
// Order here is NOT load-bearing — the loop picks max severity, ties broken by
|
|
// (severity desc, name asc) for deterministic output.
|
|
func DefaultRules() []Rule {
|
|
return []Rule{
|
|
WaterRule(),
|
|
MealRule(),
|
|
BreakRule(),
|
|
ServiceDownRule(),
|
|
NetdataCriticalRule(),
|
|
}
|
|
}
|