claude
0560684b35
Technitium read down on one poll and up on the next, sixty seconds apart, and the sev4 arrived after the service was already back. mavpoll writes a service_down fact only when the state changes, so the fact's timestamp IS the moment the monitor went down and its age is how long it has stayed there. The debounce is that age against MinDownAge, 90s — one poll interval plus jitter. No history to keep and no counter to persist. It bounds the alarm and not the truth: DownServices still reports a monitor the instant it goes down, because /dash showing a fresh outage is right even when phoning him about it is not. Existing fixtures that seeded a one-minute-old down fact now seed five, which is what they always meant.
542 lines
18 KiB
Go
542 lines
18 KiB
Go
package loop
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/kami/maven/internal/store"
|
|
)
|
|
|
|
// Direct tests for the five default rule predicates.
|
|
//
|
|
// A predicate is pure — (State) -> bool, no I/O — so these need no store and no
|
|
// daemon. They test the predicate ALONE: the restraint gate is tested in
|
|
// loop_test.go and gate_test.go, never here.
|
|
//
|
|
// Every rule gets the same three questions plus its own edges:
|
|
// - does it fire when it should?
|
|
// - does it stay quiet when it should?
|
|
// - is it silent when the key it needs has no data at all?
|
|
//
|
|
// The last one is load-bearing. docs/design.md: "since(key)==null → don't fire.
|
|
// Silence on no-data is 'shuts up when uncertain'."
|
|
|
|
// stateWith builds a snapshot at refTime() holding just the given facts.
|
|
// Presence and the env flags are left zero — the predicate must not read them.
|
|
func stateWith(facts map[string]store.Fact) State {
|
|
return State{Now: refTime(), Facts: facts}
|
|
}
|
|
|
|
// ago is a fact for key written `d` before refTime().
|
|
func ago(key, source, value string, d time.Duration) store.Fact {
|
|
return factAt(key, source, value, refTime().Add(-d))
|
|
}
|
|
|
|
// ---------------------------- since-based care rules -------------------------
|
|
|
|
// The three care rules share one shape: "fire when it has been at least N since
|
|
// the last fact for key". One table drives all of them.
|
|
func TestCareRulePredicates(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
rule Rule
|
|
facts map[string]store.Fact
|
|
want bool
|
|
}{
|
|
// water — threshold 3h.
|
|
{
|
|
name: "water fires at 4h",
|
|
rule: WaterRule(),
|
|
facts: map[string]store.Fact{"water": ago("water", "tap:water", `"250ml"`, 4*time.Hour)},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "water fires exactly at the 3h threshold",
|
|
rule: WaterRule(),
|
|
facts: map[string]store.Fact{"water": ago("water", "tap:water", `"250ml"`, 3*time.Hour)},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "water quiet just under 3h",
|
|
rule: WaterRule(),
|
|
facts: map[string]store.Fact{"water": ago("water", "tap:water", `"250ml"`, 3*time.Hour-time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "water quiet on no data",
|
|
rule: WaterRule(),
|
|
facts: nil,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "water quiet on a zero-timestamp fact",
|
|
rule: WaterRule(),
|
|
facts: map[string]store.Fact{"water": {Key: "water", Source: "tap:water", Value: `"250ml"`}},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "water quiet when the only fact is for another key",
|
|
rule: WaterRule(),
|
|
facts: map[string]store.Fact{"meal": ago("meal", "voice", `"lunch"`, 9*time.Hour)},
|
|
want: false,
|
|
},
|
|
|
|
// meal — threshold 6h.
|
|
{
|
|
name: "meal fires at 7h",
|
|
rule: MealRule(),
|
|
facts: map[string]store.Fact{"meal": ago("meal", "voice", `"lunch"`, 7*time.Hour)},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "meal fires exactly at the 6h threshold",
|
|
rule: MealRule(),
|
|
facts: map[string]store.Fact{"meal": ago("meal", "voice", `"lunch"`, 6*time.Hour)},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "meal quiet just under 6h",
|
|
rule: MealRule(),
|
|
facts: map[string]store.Fact{"meal": ago("meal", "voice", `"lunch"`, 6*time.Hour-time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "meal quiet on no data",
|
|
rule: MealRule(),
|
|
facts: nil,
|
|
want: false,
|
|
},
|
|
|
|
// break — needs BOTH anchors: at the desk now, and no break for 90min.
|
|
{
|
|
name: "break fires when at desk and no break for 2h",
|
|
rule: BreakRule(),
|
|
facts: map[string]store.Fact{
|
|
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
|
|
"break": ago("break", "voice", `"walk"`, 2*time.Hour),
|
|
},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "break fires exactly at both thresholds",
|
|
rule: BreakRule(),
|
|
facts: map[string]store.Fact{
|
|
"desk_active": ago("desk_active", "infer:hyprland", "1", 2*time.Minute),
|
|
"break": ago("break", "voice", `"walk"`, 90*time.Minute),
|
|
},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "break quiet when the desk signal is stale (user left)",
|
|
rule: BreakRule(),
|
|
facts: map[string]store.Fact{
|
|
"desk_active": ago("desk_active", "infer:hyprland", "1", 10*time.Minute),
|
|
"break": ago("break", "voice", `"walk"`, 2*time.Hour),
|
|
},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "break quiet when the last break was recent",
|
|
rule: BreakRule(),
|
|
facts: map[string]store.Fact{
|
|
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
|
|
"break": ago("break", "voice", `"walk"`, 20*time.Minute),
|
|
},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "break quiet with only the desk anchor",
|
|
rule: BreakRule(),
|
|
facts: map[string]store.Fact{
|
|
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
|
|
},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "break quiet with only the break anchor",
|
|
rule: BreakRule(),
|
|
facts: map[string]store.Fact{
|
|
"break": ago("break", "voice", `"walk"`, 2*time.Hour),
|
|
},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "break quiet on no data",
|
|
rule: BreakRule(),
|
|
facts: nil,
|
|
want: false,
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
if got := c.rule.Predicate(stateWith(c.facts)); got != c.want {
|
|
t.Fatalf("%s predicate: want %v, got %v", c.rule.Name, c.want, got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// ---------------------------- ops rules --------------------------------------
|
|
|
|
// The two ops rules match on a value AND on which poller wrote it. docs/design.md:
|
|
// "a compromised poller must not be able to forge a trigger." Half of this
|
|
// table is forgery attempts; all of them must be refused.
|
|
func TestOpsRulePredicates(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
rule Rule
|
|
facts map[string]store.Fact
|
|
want bool
|
|
}{
|
|
// service_down — only poll:uptimekuma may say a service is down.
|
|
{
|
|
name: "service_down fires on a kuma down fact",
|
|
rule: ServiceDownRule(),
|
|
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, 5*time.Minute)},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "service_down quiet when kuma says up",
|
|
rule: ServiceDownRule(),
|
|
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `"up"`, 5*time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "service_down quiet on no data",
|
|
rule: ServiceDownRule(),
|
|
facts: nil,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "service_down quiet on a zero-timestamp fact",
|
|
rule: ServiceDownRule(),
|
|
facts: map[string]store.Fact{"service_down:db": {Key: "service_down:db", Source: "poll:uptimekuma", Value: `"down"`}},
|
|
want: false,
|
|
},
|
|
// forgery attempts — right value, wrong writer.
|
|
{
|
|
name: "service_down refuses a forgery from the netdata poller",
|
|
rule: ServiceDownRule(),
|
|
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:netdata", `"down"`, 5*time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "service_down refuses a forgery from ambient audio",
|
|
rule: ServiceDownRule(),
|
|
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "ambient:other", `"down"`, 5*time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "service_down refuses a forgery from the user's own voice",
|
|
rule: ServiceDownRule(),
|
|
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "voice", `"down"`, 5*time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "service_down refuses a source that only looks like kuma",
|
|
rule: ServiceDownRule(),
|
|
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma-staging", `"down"`, 5*time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "service_down refuses an unquoted down value",
|
|
rule: ServiceDownRule(),
|
|
facts: map[string]store.Fact{"service_down:db": ago("service_down:db", "poll:uptimekuma", `down`, 5*time.Minute)},
|
|
want: false,
|
|
},
|
|
|
|
// netdata_critical — only poll:netdata may raise a critical alarm.
|
|
{
|
|
name: "netdata_critical fires on a netdata critical alarm",
|
|
rule: NetdataCriticalRule(),
|
|
facts: map[string]store.Fact{"netdata_alarm": ago("netdata_alarm", "poll:netdata", `"critical"`, time.Minute)},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "netdata_critical quiet on a warning alarm",
|
|
rule: NetdataCriticalRule(),
|
|
facts: map[string]store.Fact{"netdata_alarm": ago("netdata_alarm", "poll:netdata", `"warning"`, time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "netdata_critical quiet on a cleared alarm",
|
|
rule: NetdataCriticalRule(),
|
|
facts: map[string]store.Fact{"netdata_alarm": ago("netdata_alarm", "poll:netdata", `"clear"`, time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "netdata_critical quiet on no data",
|
|
rule: NetdataCriticalRule(),
|
|
facts: nil,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "netdata_critical quiet on a zero-timestamp fact",
|
|
rule: NetdataCriticalRule(),
|
|
facts: map[string]store.Fact{"netdata_alarm": {Key: "netdata_alarm", Source: "poll:netdata", Value: `"critical"`}},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "netdata_critical refuses a forgery from the kuma poller",
|
|
rule: NetdataCriticalRule(),
|
|
facts: map[string]store.Fact{"netdata_alarm": ago("netdata_alarm", "poll:uptimekuma", `"critical"`, time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "netdata_critical refuses a forgery from ambient audio",
|
|
rule: NetdataCriticalRule(),
|
|
facts: map[string]store.Fact{"netdata_alarm": ago("netdata_alarm", "ambient:other", `"critical"`, time.Minute)},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "netdata_critical reads netdata_alarm, not netdata_critical",
|
|
rule: NetdataCriticalRule(),
|
|
facts: map[string]store.Fact{"netdata_critical": ago("netdata_critical", "poll:netdata", `"critical"`, time.Minute)},
|
|
want: false,
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
if got := c.rule.Predicate(stateWith(c.facts)); got != c.want {
|
|
t.Fatalf("%s predicate: want %v, got %v", c.rule.Name, c.want, got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// ---------------------------- rule metadata ----------------------------------
|
|
|
|
// Every default rule must declare the keys it needs. The gate uses that list as
|
|
// a second no-data backstop, so a rule that forgets it loses the safety net
|
|
// even if its predicate happens to check.
|
|
func TestDefaultRulesDeclareInertKeys(t *testing.T) {
|
|
// A rule over a key set that only exists at read time declares a prefix
|
|
// instead — the gatherer still needs to be told what to load.
|
|
for _, r := range DefaultRules() {
|
|
if len(r.InertWhenNoData) == 0 && len(r.WantPrefixes) == 0 {
|
|
t.Errorf("rule %q declares neither InertWhenNoData keys nor WantPrefixes", r.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A no-data snapshot must make EVERY default rule quiet, predicate alone, with
|
|
// the gate out of the picture. This is the whole-set version of the per-rule
|
|
// no-data cases above.
|
|
func TestNoDefaultRuleFiresOnEmptyState(t *testing.T) {
|
|
empty := stateWith(nil)
|
|
for _, r := range DefaultRules() {
|
|
if r.Predicate(empty) {
|
|
t.Errorf("rule %q fires on an empty snapshot", r.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Severities are the delivery contract (docs/design.md § Delivery / channel
|
|
// routing): care is sev1-2 and drops when away, ops is sev3-4 and holds. Pin
|
|
// them so a change to a rule's insistence has to be deliberate.
|
|
func TestDefaultRuleSeverities(t *testing.T) {
|
|
want := map[string]Severity{
|
|
"water": Sev1,
|
|
"meal": Sev1,
|
|
"break": Sev2,
|
|
"service_down": Sev4,
|
|
"netdata_critical": Sev3,
|
|
}
|
|
got := map[string]Severity{}
|
|
for _, r := range DefaultRules() {
|
|
got[r.Name] = r.Severity
|
|
}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("rule count changed: want %d, got %d", len(want), len(got))
|
|
}
|
|
for name, sev := range want {
|
|
if got[name] != sev {
|
|
t.Errorf("rule %q severity: want %d, got %d", name, sev, got[name])
|
|
}
|
|
}
|
|
}
|
|
|
|
// Cooldown bounds keep the feedback tuner honest — docs/design.md wants
|
|
// `cooldown in [min,max]` "so a weird week can't mutate Maven silent or
|
|
// stalker". A base outside its own envelope would make that meaningless.
|
|
func TestDefaultRuleCooldownsAreBounded(t *testing.T) {
|
|
for _, r := range DefaultRules() {
|
|
c := r.Cooldown
|
|
if c.Min <= 0 || c.Base <= 0 || c.Max <= 0 {
|
|
t.Errorf("rule %q has a non-positive cooldown: %+v", r.Name, c)
|
|
continue
|
|
}
|
|
if c.Base < c.Min || c.Base > c.Max {
|
|
t.Errorf("rule %q base %v outside envelope [%v, %v]", r.Name, c.Base, c.Min, c.Max)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A predicate must read only the snapshot it is handed. Same snapshot twice
|
|
// (and a snapshot shared between two rules) must give the same answer — no
|
|
// hidden state, no clock reads.
|
|
func TestPredicatesArePure(t *testing.T) {
|
|
s := stateWith(map[string]store.Fact{
|
|
"water": ago("water", "tap:water", `"250ml"`, 4*time.Hour),
|
|
"meal": ago("meal", "voice", `"lunch"`, 7*time.Hour),
|
|
"desk_active": ago("desk_active", "infer:hyprland", "1", 30*time.Second),
|
|
"break": ago("break", "voice", `"walk"`, 2*time.Hour),
|
|
"service_down:db": ago("service_down:db", "poll:uptimekuma", `"down"`, 5*time.Minute),
|
|
})
|
|
for _, r := range DefaultRules() {
|
|
first := r.Predicate(s)
|
|
for i := 0; i < 3; i++ {
|
|
if again := r.Predicate(s); again != first {
|
|
t.Fatalf("rule %q predicate is not pure: %v then %v", r.Name, first, again)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func ruleNames(rs []Rule) []string {
|
|
out := make([]string, len(rs))
|
|
for i, r := range rs {
|
|
out[i] = r.Name
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestRulesExceptDropsOnlyTheNamed(t *testing.T) {
|
|
rules, dropped := RulesExcept([]string{"service_down"})
|
|
if len(rules) != len(DefaultRules())-1 {
|
|
t.Fatalf("got %v", ruleNames(rules))
|
|
}
|
|
for _, r := range rules {
|
|
if r.Name == "service_down" {
|
|
t.Error("a disabled rule was still wired")
|
|
}
|
|
}
|
|
if len(dropped) != 1 || dropped[0] != "service_down" {
|
|
t.Errorf("dropped = %v", dropped)
|
|
}
|
|
}
|
|
|
|
// A config that names a rule nobody wrote must not stop the daemon booting,
|
|
// and must not quietly drop a real rule alongside it.
|
|
func TestRulesExceptIgnoresUnknownNames(t *testing.T) {
|
|
rules, dropped := RulesExcept([]string{"no_such_rule", " ", ""})
|
|
if len(rules) != len(DefaultRules()) {
|
|
t.Errorf("an unknown name removed something: %v", ruleNames(rules))
|
|
}
|
|
if len(dropped) != 0 {
|
|
t.Errorf("dropped = %v, want nothing", dropped)
|
|
}
|
|
}
|
|
|
|
func TestRulesExceptEmptyKeepsEverything(t *testing.T) {
|
|
rules, dropped := RulesExcept(nil)
|
|
if len(rules) != len(DefaultRules()) || dropped != nil {
|
|
t.Errorf("rules = %v, dropped = %v", ruleNames(rules), dropped)
|
|
}
|
|
}
|
|
|
|
// ---------------------------- per-monitor service_down -----------------------
|
|
|
|
// The rule must name what fired on it, and the phraser reads the same helper,
|
|
// so a service that is up can never be spoken as down.
|
|
func TestDownServicesNamesOnlyTheDownOnes(t *testing.T) {
|
|
s := State{
|
|
Now: refTime(),
|
|
Facts: map[string]store.Fact{
|
|
"service_down:web": ago("service_down:web", ServiceDownSource, `"up"`, 5*time.Minute),
|
|
"service_down:db": ago("service_down:db", ServiceDownSource, `"down"`, 5*time.Minute),
|
|
"service_down:vault": ago("service_down:vault", ServiceDownSource, `"down"`, 5*time.Minute),
|
|
"service_down:paused": ago("service_down:paused", ServiceDownSource, `"maintenance"`, 5*time.Minute),
|
|
"service_down:forged": ago("service_down:forged", "voice", `"down"`, 5*time.Minute),
|
|
"service_down:missing": ago("service_down:missing", ServiceDownSource, `"unknown"`, 5*time.Minute),
|
|
},
|
|
}
|
|
got := DownServices(s)
|
|
want := []string{"db", "vault"} // key order, so speech is stable
|
|
if len(got) != len(want) {
|
|
t.Fatalf("DownServices = %v, want %v", got, want)
|
|
}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Fatalf("DownServices = %v, want %v", got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A monitor paused in kuma must silence that monitor. Before per-monitor facts
|
|
// the aggregate stayed "down" and pausing achieved nothing.
|
|
func TestPausedMonitorSilencesOnlyItself(t *testing.T) {
|
|
base := map[string]store.Fact{
|
|
"service_down:db": ago("service_down:db", ServiceDownSource, `"maintenance"`, 5*time.Minute),
|
|
"service_down:web": ago("service_down:web", ServiceDownSource, `"down"`, 5*time.Minute),
|
|
}
|
|
if !ServiceDownRule().Predicate(State{Now: refTime(), Facts: base}) {
|
|
t.Fatal("web is still down, the rule must fire")
|
|
}
|
|
delete(base, "service_down:web")
|
|
if ServiceDownRule().Predicate(State{Now: refTime(), Facts: base}) {
|
|
t.Fatal("only a paused monitor is left, the rule must be quiet")
|
|
}
|
|
}
|
|
|
|
// Edge-triggered: he is told once per transition. A service that stays down
|
|
// for a day used to qualify on every tick, with cooldown as the only brake.
|
|
func TestServiceDownFiresOncePerTransition(t *testing.T) {
|
|
down := ago("service_down:db", ServiceDownSource, `"down"`, time.Hour)
|
|
s := State{Now: refTime(), Facts: map[string]store.Fact{"service_down:db": down}}
|
|
if !ServiceDownRule().Predicate(s) {
|
|
t.Fatal("first sight of the transition must fire")
|
|
}
|
|
s.LastNudge = map[string]store.Nudge{"service_down": {Ts: down.Ts.Add(time.Minute)}}
|
|
if ServiceDownRule().Predicate(s) {
|
|
t.Fatal("already told about this transition, must be quiet")
|
|
}
|
|
// A second service goes down after that nudge — a new edge, so it fires.
|
|
s.Facts["service_down:web"] = ago("service_down:web", ServiceDownSource, `"down"`, 5*time.Minute)
|
|
if !ServiceDownRule().Predicate(s) {
|
|
t.Fatal("a later transition must fire again")
|
|
}
|
|
}
|
|
|
|
// A monitor that reads down on one poll and up on the next raises nothing
|
|
// (Vikunja #536). Technitium did exactly that at 01:27:59 on 05-08-2026, and
|
|
// the alarm arrived after the service was already back.
|
|
func TestAFlappingMonitorRaisesNothing(t *testing.T) {
|
|
fresh := ago("service_down:dns", ServiceDownSource, `"down"`, 10*time.Second)
|
|
s := State{Now: refTime(), Facts: map[string]store.Fact{"service_down:dns": fresh}}
|
|
if ServiceDownRule().Predicate(s) {
|
|
t.Fatal("a monitor down for ten seconds must not wake him")
|
|
}
|
|
// It stays down. mavpoll writes only on change, so the fact does not move
|
|
// and its age is how long the outage has run.
|
|
s.Now = fresh.Ts.Add(MinDownAge)
|
|
if !ServiceDownRule().Predicate(s) {
|
|
t.Fatal("past MinDownAge this is a real outage and must fire")
|
|
}
|
|
}
|
|
|
|
// The debounce bounds the alarm, not the truth. /dash reads DownServices and
|
|
// showing a fresh outage there is right even when phoning him about it is not.
|
|
func TestDownServicesReportsAFreshOutage(t *testing.T) {
|
|
s := State{
|
|
Now: refTime(),
|
|
Facts: map[string]store.Fact{"service_down:dns": ago("service_down:dns", ServiceDownSource, `"down"`, 10*time.Second)},
|
|
}
|
|
if got := DownServices(s); len(got) != 1 || got[0] != "dns" {
|
|
t.Fatalf("DownServices = %v, want [dns]", got)
|
|
}
|
|
}
|
|
|
|
// One monitor flapping must not hide another that has been down for an hour.
|
|
func TestAFlapDoesNotMaskARealOutage(t *testing.T) {
|
|
s := State{Now: refTime(), Facts: map[string]store.Fact{
|
|
"service_down:dns": ago("service_down:dns", ServiceDownSource, `"down"`, 10*time.Second),
|
|
"service_down:db": ago("service_down:db", ServiceDownSource, `"down"`, time.Hour),
|
|
}}
|
|
if !ServiceDownRule().Predicate(s) {
|
|
t.Fatal("db has been down an hour, the rule must fire")
|
|
}
|
|
}
|