diff --git a/cmd/mavpoll/main.go b/cmd/mavpoll/main.go index 69c9eb4..f3f0702 100644 --- a/cmd/mavpoll/main.go +++ b/cmd/mavpoll/main.go @@ -140,6 +140,10 @@ type poller struct { wgIface string wgCmd string + // kumaSeen — monitor name → state as of the last poll, so a monitor that + // disappears from the gauge can be marked unknown instead of staying down. + kumaSeen map[string]string + // zen is nil unless a token file was configured — money tracking is a // capability, off by default like weather and telegram. zen *zenmoney.Client @@ -333,50 +337,96 @@ func maxSeverity(a netdataAlarms) string { return sev } -// ---- kuma: monitor_status gauge → aggregate service_down ------------------- +// ---- kuma: monitor_status gauge → one fact per monitor --------------------- // Kuma exposes Prometheus text: `monitor_status{...,monitor_name="X"} V` where -// V is 1=up 0=down 2=pending 3=maintenance. We reduce to one aggregate the -// existing ServiceDownRule consumes: "down" if ANY monitor reads 0, else "up". -// Per-service granularity is a later add (a fact per monitor) — the MVP nudge -// only needs "something is down". -var kumaLine = regexp.MustCompile(`^monitor_status\{([^}]*)\}\s+([0-9.eE+-]+)`) +// V is 1=up 0=down 2=pending 3=maintenance. We write one fact per monitor, +// keyed `service_down:`, because the nudge has to say WHICH +// service is down. The aggregate this used to write could not, which is why +// the rule shipped disabled. +var ( + kumaLine = regexp.MustCompile(`^monitor_status\{([^}]*)\}\s+([0-9.eE+-]+)`) + kumaName = regexp.MustCompile(`monitor_name="([^"]*)"`) +) func (p *poller) pollKuma(ctx context.Context, now time.Time) error { body, err := p.get(ctx, p.kumaURL, p.kumaKey) if err != nil { return err } - down, seen := kumaAnyDown(body) - if !seen { + states := kumaMonitors(body) + if len(states) == 0 { return fmt.Errorf("no monitor_status metrics (auth/endpoint wrong?)") } - val := "up" - if down { - val = "down" + var firstErr error + for name, val := range states { + if err := p.writeIfChanged(ctx, kumaFactKey(name), kumaSource, val, now); err != nil && firstErr == nil { + firstErr = err // one bad monitor must not blind the rest + } } - return p.writeIfChanged(ctx, "service_down", "poll:uptimekuma", val, now) + // A monitor deleted in kuma stops appearing in the gauge, and its last fact + // would otherwise read "down" forever. Mark it unknown, which no rule fires + // on. The seen-set is in memory, so a restart forgets it — harmless, since + // the next poll that still lacks the monitor says nothing new either. + for name := range p.kumaSeen { + if _, still := states[name]; !still { + if err := p.writeIfChanged(ctx, kumaFactKey(name), kumaSource, "unknown", now); err != nil && firstErr == nil { + firstErr = err + } + } + } + p.kumaSeen = states + return firstErr } -// kumaAnyDown parses kuma's Prometheus text: down=true if any monitor reads 0 -// (pending=2/maintenance=3 are not "down"). seen=false ⇒ no monitor_status -// lines matched at all (wrong endpoint or auth rejected before the body). -func kumaAnyDown(body []byte) (down, seen bool) { +// kumaSource — the provenance the loop rule requires. Written here, checked in +// loop.ServiceDownRule; a poller under any other source cannot fire it. +const kumaSource = "poll:uptimekuma" + +// kumaFactKey — the fact key for one monitor. The suffix is the name he hears, +// so it stays as kuma spells it rather than being slugged into something else. +func kumaFactKey(name string) string { return "service_down:" + name } + +// kumaMonitors parses kuma's Prometheus text into monitor name → state +// ("up"/"down"/"pending"/"maintenance"). An empty map means no monitor_status +// line matched at all (wrong endpoint, or auth rejected before the body). +// A line with no monitor_name label is skipped: a fact nobody can name is +// exactly the thing this replaced. +func kumaMonitors(body []byte) map[string]string { + out := make(map[string]string) for _, line := range strings.Split(string(body), "\n") { m := kumaLine.FindStringSubmatch(strings.TrimSpace(line)) if m == nil { continue } - seen = true + nm := kumaName.FindStringSubmatch(m[1]) + if nm == nil || strings.TrimSpace(nm[1]) == "" { + continue + } v, err := strconv.ParseFloat(m[2], 64) if err != nil { continue } - if v == 0 { - down = true - } + out[strings.TrimSpace(nm[1])] = kumaState(v) + } + return out +} + +// kumaState — the gauge's four values. pending and maintenance are not "down": +// a monitor paused in kuma should silence that monitor, not page him. +func kumaState(v float64) string { + switch v { + case 0: + return "down" + case 1: + return "up" + case 2: + return "pending" + case 3: + return "maintenance" + default: + return "unknown" } - return down, seen } // ---- helpers --------------------------------------------------------------- diff --git a/cmd/mavpoll/main_test.go b/cmd/mavpoll/main_test.go index 7004518..229dd19 100644 --- a/cmd/mavpoll/main_test.go +++ b/cmd/mavpoll/main_test.go @@ -35,22 +35,46 @@ func TestMaxSeverity(t *testing.T) { } } -func TestKumaAnyDown(t *testing.T) { +func TestKumaMonitorsNamesEveryOne(t *testing.T) { cases := []struct { - body string - down, seen bool + name string + body string + want map[string]string }{ - {"", false, false}, - {`monitor_status{monitor_name="web"} 1`, false, true}, - {`monitor_status{monitor_name="web"} 1` + "\n" + `monitor_status{monitor_name="db"} 0`, true, true}, - {`monitor_status{monitor_name="mnt"} 3`, false, true}, // maintenance ≠ down - {`# HELP monitor_status ...`, false, false}, + {"empty body", "", map[string]string{}}, + {"help line only", `# HELP monitor_status ...`, map[string]string{}}, + { + "one up one down", + `monitor_status{monitor_name="web"} 1` + "\n" + `monitor_status{monitor_name="db"} 0`, + map[string]string{"web": "up", "db": "down"}, + }, + {"maintenance is not down", `monitor_status{monitor_name="mnt"} 3`, map[string]string{"mnt": "maintenance"}}, + {"pending is not down", `monitor_status{monitor_name="p"} 2`, map[string]string{"p": "pending"}}, + { + "other labels do not hide the name", + `monitor_status{monitor_type="http",monitor_name="ci",monitor_url="x"} 0`, + map[string]string{"ci": "down"}, + }, + {"a nameless line is skipped", `monitor_status{monitor_type="http"} 0`, map[string]string{}}, } for _, c := range cases { - down, seen := kumaAnyDown([]byte(c.body)) - if down != c.down || seen != c.seen { - t.Errorf("kumaAnyDown(%q) = (%v,%v), want (%v,%v)", c.body, down, seen, c.down, c.seen) - } + t.Run(c.name, func(t *testing.T) { + got := kumaMonitors([]byte(c.body)) + if len(got) != len(c.want) { + t.Fatalf("kumaMonitors = %v, want %v", got, c.want) + } + for k, v := range c.want { + if got[k] != v { + t.Errorf("monitor %q = %q, want %q", k, got[k], v) + } + } + }) + } +} + +func TestKumaFactKeyCarriesTheName(t *testing.T) { + if got := kumaFactKey("nexus db"); got != "service_down:nexus db" { + t.Errorf("kumaFactKey = %q", got) } }