diff --git a/cmd/mavend/tick_test.go b/cmd/mavend/tick_test.go index 7290ece..c2c4ac9 100644 --- a/cmd/mavend/tick_test.go +++ b/cmd/mavend/tick_test.go @@ -584,7 +584,7 @@ func TestDigestSev4BypassesQueue(t *testing.T) { ctx := context.Background() now := refNow() markPresent(t, st, ctx, now) - if _, err := st.SetValue(ctx, store.KindSelf, "service_down", "poll:uptimekuma", "down", now); err != nil { + if _, err := st.SetValue(ctx, store.KindSelf, "service_down:db", "poll:uptimekuma", "down", now); err != nil { t.Fatalf("seed service_down: %v", err) } sink := &fakeSink{} diff --git a/internal/loop/explain_test.go b/internal/loop/explain_test.go index 0acbf30..1110106 100644 --- a/internal/loop/explain_test.go +++ b/internal/loop/explain_test.go @@ -103,7 +103,7 @@ func TestExplainGate_PresenceAway(t *testing.T) { func TestExplainGate_PresenceAwayOpsBypass(t *testing.T) { now := refTime() s := State{Now: now, Presence: store.Away, - Facts: map[string]store.Fact{"service_down": factAt("service_down", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute))}, + Facts: map[string]store.Fact{"service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute))}, } r := ServiceDownRule() // Sev4 ops passed, blocked, d := ExplainGate(s, r) @@ -259,8 +259,8 @@ func TestExplainTick_WinnerRecorded(t *testing.T) { Now: now, Presence: store.Present, Facts: map[string]store.Fact{ - "water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)), - "service_down": factAt("service_down", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)), + "water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)), + "service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)), }, } cand, trace := ExplainTick(s, DefaultRules()) diff --git a/internal/loop/gate_test.go b/internal/loop/gate_test.go index eb613d4..4ad6e90 100644 --- a/internal/loop/gate_test.go +++ b/internal/loop/gate_test.go @@ -198,12 +198,12 @@ func TestTickNeverDogpilesAndPicksLoudest(t *testing.T) { Now: now, Presence: store.Present, Facts: map[string]store.Fact{ - "water": ago("water", "tap:water", `"250ml"`, 5*time.Hour), - "meal": ago("meal", "voice", `"lunch"`, 8*time.Hour), - "desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second), - "break": ago("break", "voice", `"walk"`, 3*time.Hour), - "service_down": ago("service_down", "poll:uptimekuma", `"down"`, time.Minute), - "netdata_alarm": ago("netdata_alarm", "poll:netdata", `"critical"`, time.Minute), + "water": ago("water", "tap:water", `"250ml"`, 5*time.Hour), + "meal": ago("meal", "voice", `"lunch"`, 8*time.Hour), + "desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second), + "break": ago("break", "voice", `"walk"`, 3*time.Hour), + "service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, time.Minute), + "netdata_alarm": ago("netdata_alarm", "poll:netdata", `"critical"`, time.Minute), }, } // sanity: every rule really does want to fire, so the pick is a real choice. diff --git a/internal/loop/gather_test.go b/internal/loop/gather_test.go new file mode 100644 index 0000000..c854eba --- /dev/null +++ b/internal/loop/gather_test.go @@ -0,0 +1,47 @@ +package loop + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/kami/maven/internal/store" +) + +// The gatherer is the only impure piece, and a rule over a prefix has no keys +// to declare at wiring time. This is the end of that path: mavpoll's per-monitor +// facts reach the snapshot, and the rule fires on the one that is down. +func TestGatherStateLoadsPrefixFamilies(t *testing.T) { + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "loop_test.db")) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + + now := time.Now().UTC().Truncate(time.Millisecond) + for key, val := range map[string]string{ + "service_down:db": "down", + "service_down:web": "up", + } { + if _, err := s.SetValue(ctx, store.KindEnv, key, ServiceDownSource, val, now.Add(-time.Minute)); err != nil { + t.Fatalf("SetValue %s: %v", key, err) + } + } + + rules := []Rule{ServiceDownRule()} + st, _, err := NewGatherer(s, rules).GatherState(ctx, now) + if err != nil { + t.Fatalf("GatherState: %v", err) + } + if _, ok := st.Facts["service_down:db"]; !ok { + t.Fatalf("prefix family not gathered: %v", st.Facts) + } + if got := DownServices(st); len(got) != 1 || got[0] != "db" { + t.Fatalf("DownServices = %v, want [db]", got) + } + if !rules[0].Predicate(st) { + t.Fatal("the rule must fire on a gathered per-monitor fact") + } +} diff --git a/internal/loop/loop_test.go b/internal/loop/loop_test.go index d6837c8..0f5cf1a 100644 --- a/internal/loop/loop_test.go +++ b/internal/loop/loop_test.go @@ -82,7 +82,7 @@ func TestTickOpsHardSurvivesAwayAndQuiet(t *testing.T) { Presence: store.Away, QuietHours: true, Facts: map[string]store.Fact{ - "service_down": factAt("service_down", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)), + "service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)), }, } got := Tick(s, DefaultRules()) @@ -99,7 +99,7 @@ func TestTickServiceSourceTrustRefusesForgedTrigger(t *testing.T) { Now: now, Presence: store.Present, Facts: map[string]store.Fact{ - "service_down": factAt("service_down", "ambient", `"down"`, now.Add(-1*time.Minute)), + "service_down:db": factAt("service_down:db", "ambient", `"down"`, now.Add(-1*time.Minute)), }, } if got := Tick(s, DefaultRules()); got != nil { @@ -115,8 +115,8 @@ func TestTickOneNudgePerTickMaxSeverityWins(t *testing.T) { Now: now, Presence: store.Present, Facts: map[string]store.Fact{ - "water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)), - "service_down": factAt("service_down", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)), + "water": factAt("water", "tap:water", `"250ml"`, now.Add(-4*time.Hour)), + "service_down:db": factAt("service_down:db", "poll:uptimekuma", `"down"`, now.Add(-1*time.Minute)), }, } got := Tick(s, DefaultRules()) diff --git a/internal/loop/rules_test.go b/internal/loop/rules_test.go index 3bec077..70cc11c 100644 --- a/internal/loop/rules_test.go +++ b/internal/loop/rules_test.go @@ -193,13 +193,13 @@ func TestOpsRulePredicates(t *testing.T) { { name: "service_down fires on a kuma down fact", rule: ServiceDownRule(), - facts: map[string]store.Fact{"service_down": ago("service_down", "poll:uptimekuma", `"down"`, time.Minute)}, + facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, time.Minute)}, want: true, }, { name: "service_down quiet when kuma says up", rule: ServiceDownRule(), - facts: map[string]store.Fact{"service_down": ago("service_down", "poll:uptimekuma", `"up"`, time.Minute)}, + facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"up"`, time.Minute)}, want: false, }, { @@ -211,38 +211,38 @@ func TestOpsRulePredicates(t *testing.T) { { name: "service_down quiet on a zero-timestamp fact", rule: ServiceDownRule(), - facts: map[string]store.Fact{"service_down": {Key: "service_down", Source: "poll:uptimekuma", Value: `"down"`}}, + facts: map[string]store.Fact{"service_down:db": {Key: "service_down:db", Source: "poll:uptimekuma", Value: `"down"`}}, want: false, }, // forgery attempts — right value, wrong writer. { name: "service_down refuses a forgery from the netdata poller", rule: ServiceDownRule(), - facts: map[string]store.Fact{"service_down": ago("service_down", "poll:netdata", `"down"`, time.Minute)}, + facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:netdata", `"down"`, time.Minute)}, want: false, }, { name: "service_down refuses a forgery from ambient audio", rule: ServiceDownRule(), - facts: map[string]store.Fact{"service_down": ago("service_down", "ambient:other", `"down"`, time.Minute)}, + facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "ambient:other", `"down"`, time.Minute)}, want: false, }, { name: "service_down refuses a forgery from the user's own voice", rule: ServiceDownRule(), - facts: map[string]store.Fact{"service_down": ago("service_down", "voice", `"down"`, time.Minute)}, + facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "voice", `"down"`, time.Minute)}, want: false, }, { name: "service_down refuses a source that only looks like kuma", rule: ServiceDownRule(), - facts: map[string]store.Fact{"service_down": ago("service_down", "poll:uptimekuma-staging", `"down"`, time.Minute)}, + facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma-staging", `"down"`, time.Minute)}, want: false, }, { name: "service_down refuses an unquoted down value", rule: ServiceDownRule(), - facts: map[string]store.Fact{"service_down": ago("service_down", "poll:uptimekuma", `down`, time.Minute)}, + facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `down`, time.Minute)}, want: false, }, @@ -312,9 +312,11 @@ func TestOpsRulePredicates(t *testing.T) { // a second no-data backstop, so a rule that forgets it loses the safety net // even if its predicate happens to check. func TestDefaultRulesDeclareInertKeys(t *testing.T) { + // A rule over a key set that only exists at read time declares a prefix + // instead — the gatherer still needs to be told what to load. for _, r := range DefaultRules() { - if len(r.InertWhenNoData) == 0 { - t.Errorf("rule %q declares no InertWhenNoData keys", r.Name) + if len(r.InertWhenNoData) == 0 && len(r.WantPrefixes) == 0 { + t.Errorf("rule %q declares neither InertWhenNoData keys nor WantPrefixes", r.Name) } } } @@ -377,11 +379,11 @@ func TestDefaultRuleCooldownsAreBounded(t *testing.T) { // hidden state, no clock reads. func TestPredicatesArePure(t *testing.T) { s := stateWith(map[string]store.Fact{ - "water": ago("water", "tap:water", `"250ml"`, 4*time.Hour), - "meal": ago("meal", "voice", `"lunch"`, 7*time.Hour), - "desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second), - "break": ago("break", "voice", `"walk"`, 2*time.Hour), - "service_down": ago("service_down", "poll:uptimekuma", `"down"`, time.Minute), + "water": ago("water", "tap:water", `"250ml"`, 4*time.Hour), + "meal": ago("meal", "voice", `"lunch"`, 7*time.Hour), + "desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second), + "break": ago("break", "voice", `"walk"`, 2*time.Hour), + "service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, time.Minute), }) for _, r := range DefaultRules() { first := r.Predicate(s) @@ -434,3 +436,66 @@ func TestRulesExceptEmptyKeepsEverything(t *testing.T) { t.Errorf("rules = %v, dropped = %v", ruleNames(rules), dropped) } } + +// ---------------------------- per-monitor service_down ----------------------- + +// The rule must name what fired on it, and the phraser reads the same helper, +// so a service that is up can never be spoken as down. +func TestDownServicesNamesOnlyTheDownOnes(t *testing.T) { + s := State{ + Now: refTime(), + Facts: map[string]store.Fact{ + "service_down:web": ago("service_down:web", ServiceDownSource, `"up"`, time.Minute), + "service_down:db": ago("service_down:db", ServiceDownSource, `"down"`, time.Minute), + "service_down:vault": ago("service_down:vault", ServiceDownSource, `"down"`, time.Minute), + "service_down:paused": ago("service_down:paused", ServiceDownSource, `"maintenance"`, time.Minute), + "service_down:forged": ago("service_down:forged", "voice", `"down"`, time.Minute), + "service_down:missing": ago("service_down:missing", ServiceDownSource, `"unknown"`, time.Minute), + }, + } + got := DownServices(s) + want := []string{"db", "vault"} // key order, so speech is stable + if len(got) != len(want) { + t.Fatalf("DownServices = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("DownServices = %v, want %v", got, want) + } + } +} + +// A monitor paused in kuma must silence that monitor. Before per-monitor facts +// the aggregate stayed "down" and pausing achieved nothing. +func TestPausedMonitorSilencesOnlyItself(t *testing.T) { + base := map[string]store.Fact{ + "service_down:db": ago("service_down:db", ServiceDownSource, `"maintenance"`, time.Minute), + "service_down:web": ago("service_down:web", ServiceDownSource, `"down"`, time.Minute), + } + if !ServiceDownRule().Predicate(State{Now: refTime(), Facts: base}) { + t.Fatal("web is still down, the rule must fire") + } + delete(base, "service_down:web") + if ServiceDownRule().Predicate(State{Now: refTime(), Facts: base}) { + t.Fatal("only a paused monitor is left, the rule must be quiet") + } +} + +// Edge-triggered: he is told once per transition. A service that stays down +// for a day used to qualify on every tick, with cooldown as the only brake. +func TestServiceDownFiresOncePerTransition(t *testing.T) { + down := ago("service_down:db", ServiceDownSource, `"down"`, time.Hour) + s := State{Now: refTime(), Facts: map[string]store.Fact{"service_down:db": down}} + if !ServiceDownRule().Predicate(s) { + t.Fatal("first sight of the transition must fire") + } + s.LastNudge = map[string]store.Nudge{"service_down": {Ts: down.Ts.Add(time.Minute)}} + if ServiceDownRule().Predicate(s) { + t.Fatal("already told about this transition, must be quiet") + } + // A second service goes down after that nudge — a new edge, so it fires. + s.Facts["service_down:web"] = ago("service_down:web", ServiceDownSource, `"down"`, time.Minute) + if !ServiceDownRule().Predicate(s) { + t.Fatal("a later transition must fire again") + } +} diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index 2c5e28d..8d6c04f 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -975,6 +975,9 @@ var fallbackNudges = map[string]string{ } func fallbackNudge(c loop.Candidate) string { + if down := loop.DownServices(c.State); len(down) > 0 { + return "Не отвечает: " + strings.Join(down, ", ") + "." + } if s, ok := fallbackNudges[c.Rule.Name]; ok { return s } @@ -990,6 +993,11 @@ func buildNudgePrompt(c loop.Candidate) string { if f, ok := c.State.Facts[c.Rule.Name]; ok && f.Key != "" && f.Key != c.Rule.Name { ctxParts = append(ctxParts, "Что именно: "+f.Key) } + if down := loop.DownServices(c.State); len(down) > 0 { + // The names come from the same helper the rule fired on, so the model + // is never handed a service that is actually up. + ctxParts = append(ctxParts, "Какие сервисы лежат: "+strings.Join(down, ", ")) + } if d, ok := c.State.Since(c.Rule.Name); ok { ctxParts = append(ctxParts, "Прошло: "+ruDur(d)) } diff --git a/internal/phraser/phraser.go b/internal/phraser/phraser.go index 79488d2..cb3fa9d 100644 --- a/internal/phraser/phraser.go +++ b/internal/phraser/phraser.go @@ -142,16 +142,21 @@ func phraseNudge(c loop.Candidate) (body, summary string) { } return body, "take a break" case "service_down": - // the fact value is json `"down"`; the key carries the service name. - body = "a service on homesrv is down — check journalctl." - summary = "service down on homesrv" - if f, ok := c.State.Fact("service_down"); ok { - if f.Key != "" && f.Key != "service_down" { - body = fmt.Sprintf("%s on homesrv is down — check journalctl.", f.Key) - summary = fmt.Sprintf("%s down on homesrv", f.Key) - } + // One fact per kuma monitor, so the nudge names the service. The rule + // and this share loop.DownServices, so the message cannot name a + // service the predicate did not fire on. + down := loop.DownServices(c.State) + switch len(down) { + case 0: + return "a service on homesrv is down — check journalctl.", "service down on homesrv" + case 1: + return fmt.Sprintf("%s on homesrv is down — check journalctl.", down[0]), + fmt.Sprintf("%s down on homesrv", down[0]) + default: + list := strings.Join(down, ", ") + return fmt.Sprintf("%s on homesrv are down — check journalctl.", list), + fmt.Sprintf("%d services down on homesrv", len(down)) } - return body, summary default: // generic: name the rule + severity; the LLM impl replaces this with // a prompted phrase. the Stub never editorializes beyond the rule name. diff --git a/internal/phraser/phraser_test.go b/internal/phraser/phraser_test.go index 23cddfc..69bc2ea 100644 --- a/internal/phraser/phraser_test.go +++ b/internal/phraser/phraser_test.go @@ -78,12 +78,14 @@ func TestPhraseNudgeBreakDeskDuration(t *testing.T) { } func TestPhraseNudgeServiceDownNamedService(t *testing.T) { - // a service_down fact whose Key is the specific service name → the phrase - // names the service, not just "service down". + // one fact per kuma monitor → the phrase names the monitor that is down. now := time.Now().UTC() st := loop.State{ - Now: now, - Facts: map[string]store.Fact{"service_down": {Key: "nginx", Ts: now, Source: "poll:healthcheck", Value: `"down"`}}, + Now: now, + Facts: map[string]store.Fact{ + "service_down:nginx": {Key: "service_down:nginx", Ts: now, Source: loop.ServiceDownSource, Value: `"down"`}, + "service_down:db": {Key: "service_down:db", Ts: now, Source: loop.ServiceDownSource, Value: `"up"`}, + }, } c := loop.Candidate{Rule: loop.ServiceDownRule(), Severity: loop.Sev4, State: st} pn, _ := NewStub().PhraseNudge(context.Background(), c) @@ -96,12 +98,12 @@ func TestPhraseNudgeServiceDownNamedService(t *testing.T) { } func TestPhraseNudgeServiceDownGenericKey(t *testing.T) { - // the rule key itself ("service_down") rather than a specific service → - // the generic phrase, not a phantom "service_down down on homesrv". + // the old aggregate key, still in the store from before the per-monitor + // facts landed → the generic phrase, never a phantom "service_down down". now := time.Now().UTC() st := loop.State{ Now: now, - Facts: map[string]store.Fact{"service_down": {Key: "service_down", Ts: now, Source: "poll:healthcheck", Value: `"down"`}}, + Facts: map[string]store.Fact{"service_down": {Key: "service_down", Ts: now, Source: loop.ServiceDownSource, Value: `"down"`}}, } c := loop.Candidate{Rule: loop.ServiceDownRule(), Severity: loop.Sev4, State: st} pn, _ := NewStub().PhraseNudge(context.Background(), c) @@ -110,6 +112,26 @@ func TestPhraseNudgeServiceDownGenericKey(t *testing.T) { } } +// Two monitors down at once must both be named — he needs to know the blast +// radius, and "a service is down" was the whole defect being fixed here. +func TestPhraseNudgeServiceDownNamesEveryDownMonitor(t *testing.T) { + now := time.Now().UTC() + st := loop.State{ + Now: now, + Facts: map[string]store.Fact{ + "service_down:nginx": {Key: "service_down:nginx", Ts: now, Source: loop.ServiceDownSource, Value: `"down"`}, + "service_down:db": {Key: "service_down:db", Ts: now, Source: loop.ServiceDownSource, Value: `"down"`}, + }, + } + c := loop.Candidate{Rule: loop.ServiceDownRule(), Severity: loop.Sev4, State: st} + pn, _ := NewStub().PhraseNudge(context.Background(), c) + for _, want := range []string{"nginx", "db"} { + if !strings.Contains(pn.Body, want) { + t.Fatalf("body should name %q, got %q", want, pn.Body) + } + } +} + func TestPhraseNudgeUnknownRuleFallsBack(t *testing.T) { // a rule without a dedicated template — generic fallback names the rule + // severity gist. never empty.