Merge pull request 'Kuma: a fact per monitor, so she can name the service that is down' (#147) from task/444-kuma-a-fact-per-monitor-so-she-can-name into master
This commit was merged in pull request #147.
This commit is contained in:
@@ -149,10 +149,11 @@ type Config struct {
|
||||
//
|
||||
// Rules are code, not config (see loop.DefaultRules), and that stays true:
|
||||
// this only subtracts. It exists because a rule can be right in principle
|
||||
// and useless in practice — kuma's service_down cannot name the service it
|
||||
// is nudging about (Vikunja #444), so being told "a service on homesrv is
|
||||
// down" every fifteen minutes is noise with no action attached. Turning it
|
||||
// off beats learning to ignore her.
|
||||
// and useless in practice. service_down was the case that forced it: it
|
||||
// could not name the service it was nudging about, so being told "a service
|
||||
// on homesrv is down" every fifteen minutes was noise with no action
|
||||
// attached. That is fixed — one fact per kuma monitor — and the rule ships
|
||||
// enabled again. The escape hatch stays.
|
||||
//
|
||||
// A disabled rule is never gathered for, never evaluated, and never
|
||||
// delivered on any channel. Unknown names are ignored, so removing a rule
|
||||
|
||||
@@ -103,7 +103,7 @@ func TestExplainGate_PresenceAway(t *testing.T) {
|
||||
func TestExplainGate_PresenceAwayOpsBypass(t *testing.T) {
|
||||
now := refTime()
|
||||
s := State{Now: now, Presence: store.Away,
|
||||
Facts: map[string]store.Fact{"service_down": factAt("service_down", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute))},
|
||||
Facts: map[string]store.Fact{"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute))},
|
||||
}
|
||||
r := ServiceDownRule() // Sev4 ops
|
||||
passed, blocked, d := ExplainGate(s, r)
|
||||
@@ -259,8 +259,8 @@ func TestExplainTick_WinnerRecorded(t *testing.T) {
|
||||
Now: now,
|
||||
Presence: store.Present,
|
||||
Facts: map[string]store.Fact{
|
||||
"water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)),
|
||||
"service_down": factAt("service_down", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)),
|
||||
"water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)),
|
||||
"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)),
|
||||
},
|
||||
}
|
||||
cand, trace := ExplainTick(s, DefaultRules())
|
||||
|
||||
@@ -198,12 +198,12 @@ func TestTickNeverDogpilesAndPicksLoudest(t *testing.T) {
|
||||
Now: now,
|
||||
Presence: store.Present,
|
||||
Facts: map[string]store.Fact{
|
||||
"water": ago("water", "tap:water", `"250ml"`, 5*time.Hour),
|
||||
"meal": ago("meal", "voice", `"lunch"`, 8*time.Hour),
|
||||
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
|
||||
"break": ago("break", "voice", `"walk"`, 3*time.Hour),
|
||||
"service_down": ago("service_down", "poll:uptimekuma", `"down"`, time.Minute),
|
||||
"netdata_alarm": ago("netdata_alarm", "poll:netdata", `"critical"`, time.Minute),
|
||||
"water": ago("water", "tap:water", `"250ml"`, 5*time.Hour),
|
||||
"meal": ago("meal", "voice", `"lunch"`, 8*time.Hour),
|
||||
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
|
||||
"break": ago("break", "voice", `"walk"`, 3*time.Hour),
|
||||
"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, time.Minute),
|
||||
"netdata_alarm": ago("netdata_alarm", "poll:netdata", `"critical"`, time.Minute),
|
||||
},
|
||||
}
|
||||
// sanity: every rule really does want to fire, so the pick is a real choice.
|
||||
|
||||
@@ -91,6 +91,20 @@ func (g *Gatherer) GatherState(ctx context.Context, now time.Time) (State, []sto
|
||||
return State{}, nil, err
|
||||
}
|
||||
|
||||
// prefix families — the keys a rule cannot name at wiring time (one fact
|
||||
// per kuma monitor). Loaded into the same map; State.FactsUnder reads them.
|
||||
for _, r := range g.rules {
|
||||
for _, p := range r.WantPrefixes {
|
||||
fam, err := g.store.LatestFactsByPrefix(ctx, p)
|
||||
if err != nil {
|
||||
return State{}, nil, err
|
||||
}
|
||||
for _, f := range fam {
|
||||
facts[f.Key] = f
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// last nudge per rule + cooldown-until derived from the active cooldown.
|
||||
// "active" = the feedback tuner's persisted base if one exists, else the
|
||||
// rule's static Base. LatestFactBySource is the trust-by-provenance read
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package loop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/store"
|
||||
)
|
||||
|
||||
// The gatherer is the only impure piece, and a rule over a prefix has no keys
|
||||
// to declare at wiring time. This is the end of that path: mavpoll's per-monitor
|
||||
// facts reach the snapshot, and the rule fires on the one that is down.
|
||||
func TestGatherStateLoadsPrefixFamilies(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := store.Open(ctx, filepath.Join(t.TempDir(), "loop_test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
for key, val := range map[string]string{
|
||||
"service_down:db": "down",
|
||||
"service_down:web": "up",
|
||||
} {
|
||||
if _, err := s.SetValue(ctx, store.KindEnv, key, ServiceDownSource, val, now.Add(-time.Minute)); err != nil {
|
||||
t.Fatalf("SetValue %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
rules := []Rule{ServiceDownRule()}
|
||||
st, _, err := NewGatherer(s, rules).GatherState(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("GatherState: %v", err)
|
||||
}
|
||||
if _, ok := st.Facts["service_down:db"]; !ok {
|
||||
t.Fatalf("prefix family not gathered: %v", st.Facts)
|
||||
}
|
||||
if got := DownServices(st); len(got) != 1 || got[0] != "db" {
|
||||
t.Fatalf("DownServices = %v, want [db]", got)
|
||||
}
|
||||
if !rules[0].Predicate(st) {
|
||||
t.Fatal("the rule must fire on a gathered per-monitor fact")
|
||||
}
|
||||
}
|
||||
@@ -82,7 +82,7 @@ func TestTickOpsHardSurvivesAwayAndQuiet(t *testing.T) {
|
||||
Presence: store.Away,
|
||||
QuietHours: true,
|
||||
Facts: map[string]store.Fact{
|
||||
"service_down": factAt("service_down", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)),
|
||||
"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)),
|
||||
},
|
||||
}
|
||||
got := Tick(s, DefaultRules())
|
||||
@@ -99,7 +99,7 @@ func TestTickServiceSourceTrustRefusesForgedTrigger(t *testing.T) {
|
||||
Now: now,
|
||||
Presence: store.Present,
|
||||
Facts: map[string]store.Fact{
|
||||
"service_down": factAt("service_down", "ambient", `"down"`, now.Add(-1*time.Minute)),
|
||||
"service_down:db": factAt("service_down:db", "ambient", `"down"`, now.Add(-1*time.Minute)),
|
||||
},
|
||||
}
|
||||
if got := Tick(s, DefaultRules()); got != nil {
|
||||
@@ -115,8 +115,8 @@ func TestTickOneNudgePerTickMaxSeverityWins(t *testing.T) {
|
||||
Now: now,
|
||||
Presence: store.Present,
|
||||
Facts: map[string]store.Fact{
|
||||
"water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)),
|
||||
"service_down": factAt("service_down", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)),
|
||||
"water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)),
|
||||
"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)),
|
||||
},
|
||||
}
|
||||
got := Tick(s, DefaultRules())
|
||||
|
||||
+55
-13
@@ -28,6 +28,13 @@ type Rule struct {
|
||||
// that check itself, leave this empty. Otherwise set to the key(s) the rule
|
||||
// needs and the gate will skip the rule when any are missing.
|
||||
InertWhenNoData []string
|
||||
|
||||
// WantPrefixes — key prefixes whose whole family the gatherer must load.
|
||||
// InertWhenNoData names keys that exist at wiring time; a rule over a key
|
||||
// set that is only known at read time (one fact per kuma monitor) declares
|
||||
// the prefix here instead. Prefixes never make a rule inert: an empty
|
||||
// family is the predicate's own "no data" case.
|
||||
WantPrefixes []string
|
||||
}
|
||||
|
||||
// Cooldown — tunable bounded by the envelope so a weird week (auto-tuned) can't
|
||||
@@ -98,23 +105,58 @@ func BreakRule() Rule {
|
||||
}
|
||||
}
|
||||
|
||||
// ServiceDownRule — sev4 ops hard: the `service_down` aggregate fact reads
|
||||
// "down". Source must be poll:uptimekuma — kuma is the source of truth for
|
||||
// service up/down (mavpoll writes this key). The predicate is provenance-scoped:
|
||||
// a compromised poller writing under a different source can't forge the trigger.
|
||||
// ServiceDownPrefix — mavpoll writes one fact per kuma monitor under this
|
||||
// prefix, `service_down:<monitor name>`. The suffix is the name he hears.
|
||||
const ServiceDownPrefix = "service_down:"
|
||||
|
||||
// ServiceDownSource — kuma is the source of truth for service up/down. The
|
||||
// rule is provenance-scoped: a poller writing under a different source cannot
|
||||
// forge the trigger.
|
||||
const ServiceDownSource = "poll:uptimekuma"
|
||||
|
||||
// DownServices — the monitors currently reading "down", by name, in key order.
|
||||
//
|
||||
// Pure, and the rule and the phraser both call it, so the message can never
|
||||
// name a service the predicate did not fire on.
|
||||
func DownServices(s State) []string {
|
||||
var out []string
|
||||
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
||||
if f.Source == ServiceDownSource && f.Value == `"down"` {
|
||||
out = append(out, strings.TrimPrefix(f.Key, ServiceDownPrefix))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ServiceDownRule — sev4 ops hard: at least one kuma monitor reads "down".
|
||||
//
|
||||
// It used to read one aggregate `service_down` fact, which is why it was
|
||||
// disabled in deploy: the nudge could say that something on homesrv was down
|
||||
// but never which thing. Per-monitor facts fix that, and pausing a monitor in
|
||||
// kuma now silences that monitor rather than nothing.
|
||||
//
|
||||
// Edge-triggered — see State.NudgedSince. Without it a service that stays down
|
||||
// for a day qualifies on every tick and cooldown alone is the only brake.
|
||||
func ServiceDownRule() Rule {
|
||||
return Rule{
|
||||
Name: "service_down",
|
||||
Severity: Sev4,
|
||||
Cooldown: Cooldown{Base: 15 * time.Minute, Min: 5 * time.Minute, Max: 1 * time.Hour},
|
||||
InertWhenNoData: []string{"service_down"},
|
||||
Name: "service_down",
|
||||
Severity: Sev4,
|
||||
Cooldown: Cooldown{Base: 15 * time.Minute, Min: 5 * time.Minute, Max: 1 * time.Hour},
|
||||
WantPrefixes: []string{ServiceDownPrefix},
|
||||
Predicate: func(s State) bool {
|
||||
f, ok := s.Fact("service_down")
|
||||
if !ok || f.Ts.IsZero() {
|
||||
return false
|
||||
var newest time.Time
|
||||
for _, f := range s.FactsUnder(ServiceDownPrefix) {
|
||||
if f.Source != ServiceDownSource || f.Value != `"down"` {
|
||||
continue
|
||||
}
|
||||
if f.Ts.After(newest) {
|
||||
newest = f.Ts
|
||||
}
|
||||
}
|
||||
// value is json `"down"`; trivial check keyed off source provenance.
|
||||
return f.Source == "poll:uptimekuma" && f.Value == `"down"`
|
||||
if newest.IsZero() {
|
||||
return false // nothing down, or no data at all → shut up
|
||||
}
|
||||
return !s.NudgedSince("service_down", newest)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
+80
-15
@@ -193,13 +193,13 @@ func TestOpsRulePredicates(t *testing.T) {
|
||||
{
|
||||
name: "service_down fires on a kuma down fact",
|
||||
rule: ServiceDownRule(),
|
||||
facts: map[string]store.Fact{"service_down": ago("service_down", "poll:uptimekuma", `"down"`, time.Minute)},
|
||||
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, time.Minute)},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "service_down quiet when kuma says up",
|
||||
rule: ServiceDownRule(),
|
||||
facts: map[string]store.Fact{"service_down": ago("service_down", "poll:uptimekuma", `"up"`, time.Minute)},
|
||||
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"up"`, time.Minute)},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
@@ -211,38 +211,38 @@ func TestOpsRulePredicates(t *testing.T) {
|
||||
{
|
||||
name: "service_down quiet on a zero-timestamp fact",
|
||||
rule: ServiceDownRule(),
|
||||
facts: map[string]store.Fact{"service_down": {Key: "service_down", Source: "poll:uptimekuma", Value: `"down"`}},
|
||||
facts: map[string]store.Fact{"service_down:db": {Key: "service_down:db", Source: "poll:uptimekuma", Value: `"down"`}},
|
||||
want: false,
|
||||
},
|
||||
// forgery attempts — right value, wrong writer.
|
||||
{
|
||||
name: "service_down refuses a forgery from the netdata poller",
|
||||
rule: ServiceDownRule(),
|
||||
facts: map[string]store.Fact{"service_down": ago("service_down", "poll:netdata", `"down"`, time.Minute)},
|
||||
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:netdata", `"down"`, time.Minute)},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "service_down refuses a forgery from ambient audio",
|
||||
rule: ServiceDownRule(),
|
||||
facts: map[string]store.Fact{"service_down": ago("service_down", "ambient:other", `"down"`, time.Minute)},
|
||||
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "ambient:other", `"down"`, time.Minute)},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "service_down refuses a forgery from the user's own voice",
|
||||
rule: ServiceDownRule(),
|
||||
facts: map[string]store.Fact{"service_down": ago("service_down", "voice", `"down"`, time.Minute)},
|
||||
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "voice", `"down"`, time.Minute)},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "service_down refuses a source that only looks like kuma",
|
||||
rule: ServiceDownRule(),
|
||||
facts: map[string]store.Fact{"service_down": ago("service_down", "poll:uptimekuma-staging", `"down"`, time.Minute)},
|
||||
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma-staging", `"down"`, time.Minute)},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "service_down refuses an unquoted down value",
|
||||
rule: ServiceDownRule(),
|
||||
facts: map[string]store.Fact{"service_down": ago("service_down", "poll:uptimekuma", `down`, time.Minute)},
|
||||
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `down`, time.Minute)},
|
||||
want: false,
|
||||
},
|
||||
|
||||
@@ -312,9 +312,11 @@ func TestOpsRulePredicates(t *testing.T) {
|
||||
// a second no-data backstop, so a rule that forgets it loses the safety net
|
||||
// even if its predicate happens to check.
|
||||
func TestDefaultRulesDeclareInertKeys(t *testing.T) {
|
||||
// A rule over a key set that only exists at read time declares a prefix
|
||||
// instead — the gatherer still needs to be told what to load.
|
||||
for _, r := range DefaultRules() {
|
||||
if len(r.InertWhenNoData) == 0 {
|
||||
t.Errorf("rule %q declares no InertWhenNoData keys", r.Name)
|
||||
if len(r.InertWhenNoData) == 0 && len(r.WantPrefixes) == 0 {
|
||||
t.Errorf("rule %q declares neither InertWhenNoData keys nor WantPrefixes", r.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -377,11 +379,11 @@ func TestDefaultRuleCooldownsAreBounded(t *testing.T) {
|
||||
// hidden state, no clock reads.
|
||||
func TestPredicatesArePure(t *testing.T) {
|
||||
s := stateWith(map[string]store.Fact{
|
||||
"water": ago("water", "tap:water", `"250ml"`, 4*time.Hour),
|
||||
"meal": ago("meal", "voice", `"lunch"`, 7*time.Hour),
|
||||
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
|
||||
"break": ago("break", "voice", `"walk"`, 2*time.Hour),
|
||||
"service_down": ago("service_down", "poll:uptimekuma", `"down"`, time.Minute),
|
||||
"water": ago("water", "tap:water", `"250ml"`, 4*time.Hour),
|
||||
"meal": ago("meal", "voice", `"lunch"`, 7*time.Hour),
|
||||
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
|
||||
"break": ago("break", "voice", `"walk"`, 2*time.Hour),
|
||||
"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, time.Minute),
|
||||
})
|
||||
for _, r := range DefaultRules() {
|
||||
first := r.Predicate(s)
|
||||
@@ -434,3 +436,66 @@ func TestRulesExceptEmptyKeepsEverything(t *testing.T) {
|
||||
t.Errorf("rules = %v, dropped = %v", ruleNames(rules), dropped)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------- per-monitor service_down -----------------------
|
||||
|
||||
// The rule must name what fired on it, and the phraser reads the same helper,
|
||||
// so a service that is up can never be spoken as down.
|
||||
func TestDownServicesNamesOnlyTheDownOnes(t *testing.T) {
|
||||
s := State{
|
||||
Now: refTime(),
|
||||
Facts: map[string]store.Fact{
|
||||
"service_down:web": ago("service_down:web", ServiceDownSource, `"up"`, time.Minute),
|
||||
"service_down:db": ago("service_down:db", ServiceDownSource, `"down"`, time.Minute),
|
||||
"service_down:vault": ago("service_down:vault", ServiceDownSource, `"down"`, time.Minute),
|
||||
"service_down:paused": ago("service_down:paused", ServiceDownSource, `"maintenance"`, time.Minute),
|
||||
"service_down:forged": ago("service_down:forged", "voice", `"down"`, time.Minute),
|
||||
"service_down:missing": ago("service_down:missing", ServiceDownSource, `"unknown"`, time.Minute),
|
||||
},
|
||||
}
|
||||
got := DownServices(s)
|
||||
want := []string{"db", "vault"} // key order, so speech is stable
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("DownServices = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("DownServices = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A monitor paused in kuma must silence that monitor. Before per-monitor facts
|
||||
// the aggregate stayed "down" and pausing achieved nothing.
|
||||
func TestPausedMonitorSilencesOnlyItself(t *testing.T) {
|
||||
base := map[string]store.Fact{
|
||||
"service_down:db": ago("service_down:db", ServiceDownSource, `"maintenance"`, time.Minute),
|
||||
"service_down:web": ago("service_down:web", ServiceDownSource, `"down"`, time.Minute),
|
||||
}
|
||||
if !ServiceDownRule().Predicate(State{Now: refTime(), Facts: base}) {
|
||||
t.Fatal("web is still down, the rule must fire")
|
||||
}
|
||||
delete(base, "service_down:web")
|
||||
if ServiceDownRule().Predicate(State{Now: refTime(), Facts: base}) {
|
||||
t.Fatal("only a paused monitor is left, the rule must be quiet")
|
||||
}
|
||||
}
|
||||
|
||||
// Edge-triggered: he is told once per transition. A service that stays down
|
||||
// for a day used to qualify on every tick, with cooldown as the only brake.
|
||||
func TestServiceDownFiresOncePerTransition(t *testing.T) {
|
||||
down := ago("service_down:db", ServiceDownSource, `"down"`, time.Hour)
|
||||
s := State{Now: refTime(), Facts: map[string]store.Fact{"service_down:db": down}}
|
||||
if !ServiceDownRule().Predicate(s) {
|
||||
t.Fatal("first sight of the transition must fire")
|
||||
}
|
||||
s.LastNudge = map[string]store.Nudge{"service_down": {Ts: down.Ts.Add(time.Minute)}}
|
||||
if ServiceDownRule().Predicate(s) {
|
||||
t.Fatal("already told about this transition, must be quiet")
|
||||
}
|
||||
// A second service goes down after that nudge — a new edge, so it fires.
|
||||
s.Facts["service_down:web"] = ago("service_down:web", ServiceDownSource, `"down"`, time.Minute)
|
||||
if !ServiceDownRule().Predicate(s) {
|
||||
t.Fatal("a later transition must fire again")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@
|
||||
package loop
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/store"
|
||||
@@ -95,6 +97,32 @@ func (s State) Fact(key string) (store.Fact, bool) {
|
||||
return f, true
|
||||
}
|
||||
|
||||
// FactsUnder returns every gathered fact whose key starts with prefix, ordered
|
||||
// by key so a caller that names them speaks them in a stable order. Facts with
|
||||
// a zero Ts are skipped, the same "no data" rule Fact applies.
|
||||
func (s State) FactsUnder(prefix string) []store.Fact {
|
||||
var out []store.Fact
|
||||
for k, f := range s.Facts {
|
||||
if strings.HasPrefix(k, prefix) && !f.Ts.IsZero() {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
|
||||
return out
|
||||
}
|
||||
|
||||
// NudgedSince reports whether rule already sent a nudge at or after ts.
|
||||
//
|
||||
// It is what makes a rule edge-triggered. A polled fact is written only when
|
||||
// the value changes, so its Ts is the moment the service went down — but the
|
||||
// predicate reads the current value, so a service that stays down keeps
|
||||
// qualifying forever and cooldown alone only slows the repetition. Asking
|
||||
// whether he was already told about THIS transition stops it.
|
||||
func (s State) NudgedSince(rule string, ts time.Time) bool {
|
||||
n, ok := s.LastNudge[rule]
|
||||
return ok && !n.Ts.Before(ts)
|
||||
}
|
||||
|
||||
// Since returns the duration since the latest fact for key, or (0,false).
|
||||
// "false" ⇒ no data ⇒ shuts up when uncertain.
|
||||
func (s State) Since(key string) (time.Duration, bool) {
|
||||
|
||||
@@ -994,6 +994,9 @@ var fallbackNudges = map[string]string{
|
||||
}
|
||||
|
||||
func fallbackNudge(c loop.Candidate) string {
|
||||
if down := loop.DownServices(c.State); len(down) > 0 {
|
||||
return "Не отвечает: " + strings.Join(down, ", ") + "."
|
||||
}
|
||||
if s, ok := fallbackNudges[c.Rule.Name]; ok {
|
||||
return s
|
||||
}
|
||||
@@ -1009,6 +1012,11 @@ func buildNudgePrompt(c loop.Candidate) string {
|
||||
if f, ok := c.State.Facts[c.Rule.Name]; ok && f.Key != "" && f.Key != c.Rule.Name {
|
||||
ctxParts = append(ctxParts, "Что именно: "+f.Key)
|
||||
}
|
||||
if down := loop.DownServices(c.State); len(down) > 0 {
|
||||
// The names come from the same helper the rule fired on, so the model
|
||||
// is never handed a service that is actually up.
|
||||
ctxParts = append(ctxParts, "Какие сервисы лежат: "+strings.Join(down, ", "))
|
||||
}
|
||||
if d, ok := c.State.Since(c.Rule.Name); ok {
|
||||
ctxParts = append(ctxParts, "Прошло: "+ruDur(d))
|
||||
}
|
||||
|
||||
@@ -142,16 +142,21 @@ func phraseNudge(c loop.Candidate) (body, summary string) {
|
||||
}
|
||||
return body, "take a break"
|
||||
case "service_down":
|
||||
// the fact value is json `"down"`; the key carries the service name.
|
||||
body = "a service on homesrv is down — check journalctl."
|
||||
summary = "service down on homesrv"
|
||||
if f, ok := c.State.Fact("service_down"); ok {
|
||||
if f.Key != "" && f.Key != "service_down" {
|
||||
body = fmt.Sprintf("%s on homesrv is down — check journalctl.", f.Key)
|
||||
summary = fmt.Sprintf("%s down on homesrv", f.Key)
|
||||
}
|
||||
// One fact per kuma monitor, so the nudge names the service. The rule
|
||||
// and this share loop.DownServices, so the message cannot name a
|
||||
// service the predicate did not fire on.
|
||||
down := loop.DownServices(c.State)
|
||||
switch len(down) {
|
||||
case 0:
|
||||
return "a service on homesrv is down — check journalctl.", "service down on homesrv"
|
||||
case 1:
|
||||
return fmt.Sprintf("%s on homesrv is down — check journalctl.", down[0]),
|
||||
fmt.Sprintf("%s down on homesrv", down[0])
|
||||
default:
|
||||
list := strings.Join(down, ", ")
|
||||
return fmt.Sprintf("%s on homesrv are down — check journalctl.", list),
|
||||
fmt.Sprintf("%d services down on homesrv", len(down))
|
||||
}
|
||||
return body, summary
|
||||
default:
|
||||
// generic: name the rule + severity; the LLM impl replaces this with
|
||||
// a prompted phrase. the Stub never editorializes beyond the rule name.
|
||||
|
||||
@@ -78,12 +78,14 @@ func TestPhraseNudgeBreakDeskDuration(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPhraseNudgeServiceDownNamedService(t *testing.T) {
|
||||
// a service_down fact whose Key is the specific service name → the phrase
|
||||
// names the service, not just "service down".
|
||||
// one fact per kuma monitor → the phrase names the monitor that is down.
|
||||
now := time.Now().UTC()
|
||||
st := loop.State{
|
||||
Now: now,
|
||||
Facts: map[string]store.Fact{"service_down": {Key: "nginx", Ts: now, Source: "poll:healthcheck", Value: `"down"`}},
|
||||
Now: now,
|
||||
Facts: map[string]store.Fact{
|
||||
"service_down:nginx": {Key: "service_down:nginx", Ts: now, Source: loop.ServiceDownSource, Value: `"down"`},
|
||||
"service_down:db": {Key: "service_down:db", Ts: now, Source: loop.ServiceDownSource, Value: `"up"`},
|
||||
},
|
||||
}
|
||||
c := loop.Candidate{Rule: loop.ServiceDownRule(), Severity: loop.Sev4, State: st}
|
||||
pn, _ := NewStub().PhraseNudge(context.Background(), c)
|
||||
@@ -96,12 +98,12 @@ func TestPhraseNudgeServiceDownNamedService(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPhraseNudgeServiceDownGenericKey(t *testing.T) {
|
||||
// the rule key itself ("service_down") rather than a specific service →
|
||||
// the generic phrase, not a phantom "service_down down on homesrv".
|
||||
// the old aggregate key, still in the store from before the per-monitor
|
||||
// facts landed → the generic phrase, never a phantom "service_down down".
|
||||
now := time.Now().UTC()
|
||||
st := loop.State{
|
||||
Now: now,
|
||||
Facts: map[string]store.Fact{"service_down": {Key: "service_down", Ts: now, Source: "poll:healthcheck", Value: `"down"`}},
|
||||
Facts: map[string]store.Fact{"service_down": {Key: "service_down", Ts: now, Source: loop.ServiceDownSource, Value: `"down"`}},
|
||||
}
|
||||
c := loop.Candidate{Rule: loop.ServiceDownRule(), Severity: loop.Sev4, State: st}
|
||||
pn, _ := NewStub().PhraseNudge(context.Background(), c)
|
||||
@@ -110,6 +112,26 @@ func TestPhraseNudgeServiceDownGenericKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Two monitors down at once must both be named — he needs to know the blast
|
||||
// radius, and "a service is down" was the whole defect being fixed here.
|
||||
func TestPhraseNudgeServiceDownNamesEveryDownMonitor(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
st := loop.State{
|
||||
Now: now,
|
||||
Facts: map[string]store.Fact{
|
||||
"service_down:nginx": {Key: "service_down:nginx", Ts: now, Source: loop.ServiceDownSource, Value: `"down"`},
|
||||
"service_down:db": {Key: "service_down:db", Ts: now, Source: loop.ServiceDownSource, Value: `"down"`},
|
||||
},
|
||||
}
|
||||
c := loop.Candidate{Rule: loop.ServiceDownRule(), Severity: loop.Sev4, State: st}
|
||||
pn, _ := NewStub().PhraseNudge(context.Background(), c)
|
||||
for _, want := range []string{"nginx", "db"} {
|
||||
if !strings.Contains(pn.Body, want) {
|
||||
t.Fatalf("body should name %q, got %q", want, pn.Body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhraseNudgeUnknownRuleFallsBack(t *testing.T) {
|
||||
// a rule without a dedicated template — generic fallback names the rule +
|
||||
// severity gist. never empty.
|
||||
|
||||
@@ -254,6 +254,41 @@ func (s *Store) LatestFactBySource(ctx context.Context, key, source string) (Fac
|
||||
return scanFact(row)
|
||||
}
|
||||
|
||||
// LatestFactsByPrefix — the latest non-voided fact for every key that starts
|
||||
// with prefix, newest-per-key, ordered by key.
|
||||
//
|
||||
// The loop's gatherer loads the keys its rules declare, which works while the
|
||||
// key set is static. Kuma's monitors are not: one fact per monitor means the
|
||||
// keys are only known once the gauge is read, so the rule declares the prefix
|
||||
// and this read resolves it per tick. `_` and `%` are escaped — a monitor name
|
||||
// is user text and must not act as a LIKE wildcard.
|
||||
func (s *Store) LatestFactsByPrefix(ctx context.Context, prefix string) ([]Fact, error) {
|
||||
esc := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(prefix)
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, ts, kind, key, value, source, confidence, voids_id
|
||||
FROM facts f
|
||||
WHERE key LIKE ? ESCAPE '\'
|
||||
AND id NOT IN (SELECT voids_id FROM facts WHERE voids_id IS NOT NULL)
|
||||
AND id = (SELECT id FROM facts g
|
||||
WHERE g.key = f.key
|
||||
AND g.id NOT IN (SELECT voids_id FROM facts WHERE voids_id IS NOT NULL)
|
||||
ORDER BY g.ts DESC, g.id DESC LIMIT 1)
|
||||
ORDER BY key`, esc+"%")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("facts by prefix %q: %w", prefix, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Fact
|
||||
for rows.Next() {
|
||||
f, err := scanFact(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Since returns how long ago the latest non-voided fact for key landed, or
|
||||
// (0, ErrNoFact). Implements the `since(key)==null → don't fire` guard from
|
||||
// the spec — silence on no-data is "shuts up when uncertain".
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// One fact per kuma monitor means the loop cannot name its keys at wiring time,
|
||||
// so it asks for the family by prefix. The read must return the newest row per
|
||||
// key and stop at the prefix boundary.
|
||||
func TestLatestFactsByPrefix(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
write := func(key, val string, at time.Time) int64 {
|
||||
id, err := s.SetValue(ctx, KindEnv, key, "poll:uptimekuma", val, at)
|
||||
if err != nil {
|
||||
t.Fatalf("SetValue %s: %v", key, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
write("service_down:db", "up", now.Add(-2*time.Hour))
|
||||
write("service_down:db", "down", now.Add(-time.Hour)) // newer wins
|
||||
write("service_down:web", "up", now.Add(-time.Hour))
|
||||
write("service_downtime", "irrelevant", now) // no colon, not in the family
|
||||
write("water", "250ml", now)
|
||||
|
||||
got, err := s.LatestFactsByPrefix(ctx, "service_down:")
|
||||
if err != nil {
|
||||
t.Fatalf("LatestFactsByPrefix: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d facts, want 2: %+v", len(got), got)
|
||||
}
|
||||
if got[0].Key != "service_down:db" || got[0].Value != `"down"` {
|
||||
t.Errorf("first = %s=%s, want the newest db row", got[0].Key, got[0].Value)
|
||||
}
|
||||
if got[1].Key != "service_down:web" {
|
||||
t.Errorf("second = %s, want service_down:web", got[1].Key)
|
||||
}
|
||||
}
|
||||
|
||||
// A monitor name is user text. An underscore in it must match itself, not act
|
||||
// as a LIKE wildcard and drag in every other monitor.
|
||||
func TestLatestFactsByPrefixEscapesWildcards(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
for _, k := range []string{"a_b:one", "axb:two"} {
|
||||
if _, err := s.SetValue(ctx, KindEnv, k, "poll:uptimekuma", "down", now); err != nil {
|
||||
t.Fatalf("SetValue %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
got, err := s.LatestFactsByPrefix(ctx, "a_b:")
|
||||
if err != nil {
|
||||
t.Fatalf("LatestFactsByPrefix: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key != "a_b:one" {
|
||||
t.Fatalf("got %+v, want only a_b:one", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user