Merge pull request 'Kuma: a fact per monitor, so she can name the service that is down' (#147) from task/444-kuma-a-fact-per-monitor-so-she-can-name into master

2026-08-04 18:24:47 +02:00
parent db0752223f 6d8a95095a
commit bb8bd608da
19 changed files with 526 additions and 100 deletions
@@ -584,7 +584,7 @@ func TestDigestSev4BypassesQueue(t *testing.T) {
 	ctx := context.Background()
 	now := refNow()
 	markPresent(t, st, ctx, now)
-	if _, err := st.SetValue(ctx, store.KindSelf, "service_down", "poll:uptimekuma", "down", now); err != nil {
+	if _, err := st.SetValue(ctx, store.KindSelf, "service_down:db", "poll:uptimekuma", "down", now); err != nil {
 		t.Fatalf("seed service_down: %v", err)
 	}
 	sink := &fakeSink{}
@@ -140,6 +140,10 @@ type poller struct {
 	wgIface    string
 	wgCmd      string

+	// kumaSeen — monitor name → state as of the last poll, so a monitor that
+	// disappears from the gauge can be marked unknown instead of staying down.
+	kumaSeen map[string]string
+
 	// zen is nil unless a token file was configured — money tracking is a
 	// capability, off by default like weather and telegram.
 	zen      *zenmoney.Client
@@ -333,50 +337,96 @@ func maxSeverity(a netdataAlarms) string {
 	return sev
 }

-// ---- kuma: monitor_status gauge → aggregate service_down -------------------
+// ---- kuma: monitor_status gauge → one fact per monitor ---------------------

 // Kuma exposes Prometheus text: `monitor_status{...,monitor_name="X"} V` where
-// V is 1=up 0=down 2=pending 3=maintenance. We reduce to one aggregate the
-// existing ServiceDownRule consumes: "down" if ANY monitor reads 0, else "up".
-// Per-service granularity is a later add (a fact per monitor) — the MVP nudge
-// only needs "something is down".
-var kumaLine = regexp.MustCompile(`^monitor_status\{([^}]*)\}\s+([0-9.eE+-]+)`)
+// V is 1=up 0=down 2=pending 3=maintenance. We write one fact per monitor,
+// keyed `service_down:<monitor name>`, because the nudge has to say WHICH
+// service is down. The aggregate this used to write could not, which is why
+// the rule shipped disabled.
+var (
+	kumaLine = regexp.MustCompile(`^monitor_status\{([^}]*)\}\s+([0-9.eE+-]+)`)
+	kumaName = regexp.MustCompile(`monitor_name="([^"]*)"`)
+)

 func (p *poller) pollKuma(ctx context.Context, now time.Time) error {
 	body, err := p.get(ctx, p.kumaURL, p.kumaKey)
 	if err != nil {
 		return err
 	}
-	down, seen := kumaAnyDown(body)
-	if !seen {
+	states := kumaMonitors(body)
+	if len(states) == 0 {
 		return fmt.Errorf("no monitor_status metrics (auth/endpoint wrong?)")
 	}
-	val := "up"
-	if down {
-		val = "down"
+	var firstErr error
+	for name, val := range states {
+		if err := p.writeIfChanged(ctx, kumaFactKey(name), kumaSource, val, now); err != nil && firstErr == nil {
+			firstErr = err // one bad monitor must not blind the rest
+		}
 	}
-	return p.writeIfChanged(ctx, "service_down", "poll:uptimekuma", val, now)
+	// A monitor deleted in kuma stops appearing in the gauge, and its last fact
+	// would otherwise read "down" forever. Mark it unknown, which no rule fires
+	// on. The seen-set is in memory, so a restart forgets it — harmless, since
+	// the next poll that still lacks the monitor says nothing new either.
+	for name := range p.kumaSeen {
+		if _, still := states[name]; !still {
+			if err := p.writeIfChanged(ctx, kumaFactKey(name), kumaSource, "unknown", now); err != nil && firstErr == nil {
+				firstErr = err
+			}
+		}
+	}
+	p.kumaSeen = states
+	return firstErr
 }

-// kumaAnyDown parses kuma's Prometheus text: down=true if any monitor reads 0
-// (pending=2/maintenance=3 are not "down"). seen=false ⇒ no monitor_status
-// lines matched at all (wrong endpoint or auth rejected before the body).
-func kumaAnyDown(body []byte) (down, seen bool) {
+// kumaSource — the provenance the loop rule requires. Written here, checked in
+// loop.ServiceDownRule; a poller under any other source cannot fire it.
+const kumaSource = "poll:uptimekuma"
+
+// kumaFactKey — the fact key for one monitor. The suffix is the name he hears,
+// so it stays as kuma spells it rather than being slugged into something else.
+func kumaFactKey(name string) string { return "service_down:" + name }
+
+// kumaMonitors parses kuma's Prometheus text into monitor name → state
+// ("up"/"down"/"pending"/"maintenance"). An empty map means no monitor_status
+// line matched at all (wrong endpoint, or auth rejected before the body).
+// A line with no monitor_name label is skipped: a fact nobody can name is
+// exactly the thing this replaced.
+func kumaMonitors(body []byte) map[string]string {
+	out := make(map[string]string)
 	for _, line := range strings.Split(string(body), "\n") {
 		m := kumaLine.FindStringSubmatch(strings.TrimSpace(line))
 		if m == nil {
 			continue
 		}
-		seen = true
+		nm := kumaName.FindStringSubmatch(m[1])
+		if nm == nil || strings.TrimSpace(nm[1]) == "" {
+			continue
+		}
 		v, err := strconv.ParseFloat(m[2], 64)
 		if err != nil {
 			continue
 		}
-		if v == 0 {
-			down = true
-		}
+		out[strings.TrimSpace(nm[1])] = kumaState(v)
+	}
+	return out
+}
+
+// kumaState — the gauge's four values. pending and maintenance are not "down":
+// a monitor paused in kuma should silence that monitor, not page him.
+func kumaState(v float64) string {
+	switch v {
+	case 0:
+		return "down"
+	case 1:
+		return "up"
+	case 2:
+		return "pending"
+	case 3:
+		return "maintenance"
+	default:
+		return "unknown"
 	}
-	return down, seen
 }

 // ---- helpers ---------------------------------------------------------------
@@ -35,22 +35,46 @@ func TestMaxSeverity(t *testing.T) {
 	}
 }

-func TestKumaAnyDown(t *testing.T) {
+func TestKumaMonitorsNamesEveryOne(t *testing.T) {
 	cases := []struct {
-		body       string
-		down, seen bool
+		name string
+		body string
+		want map[string]string
 	}{
-		{"", false, false},
-		{`monitor_status{monitor_name="web"} 1`, false, true},
-		{`monitor_status{monitor_name="web"} 1` + "\n" + `monitor_status{monitor_name="db"} 0`, true, true},
-		{`monitor_status{monitor_name="mnt"} 3`, false, true}, // maintenance ≠ down
-		{`# HELP monitor_status ...`, false, false},
+		{"empty body", "", map[string]string{}},
+		{"help line only", `# HELP monitor_status ...`, map[string]string{}},
+		{
+			"one up one down",
+			`monitor_status{monitor_name="web"} 1` + "\n" + `monitor_status{monitor_name="db"} 0`,
+			map[string]string{"web": "up", "db": "down"},
+		},
+		{"maintenance is not down", `monitor_status{monitor_name="mnt"} 3`, map[string]string{"mnt": "maintenance"}},
+		{"pending is not down", `monitor_status{monitor_name="p"} 2`, map[string]string{"p": "pending"}},
+		{
+			"other labels do not hide the name",
+			`monitor_status{monitor_type="http",monitor_name="ci",monitor_url="x"} 0`,
+			map[string]string{"ci": "down"},
+		},
+		{"a nameless line is skipped", `monitor_status{monitor_type="http"} 0`, map[string]string{}},
 	}
 	for _, c := range cases {
-		down, seen := kumaAnyDown([]byte(c.body))
-		if down != c.down || seen != c.seen {
-			t.Errorf("kumaAnyDown(%q) = (%v,%v), want (%v,%v)", c.body, down, seen, c.down, c.seen)
-		}
+		t.Run(c.name, func(t *testing.T) {
+			got := kumaMonitors([]byte(c.body))
+			if len(got) != len(c.want) {
+				t.Fatalf("kumaMonitors = %v, want %v", got, c.want)
+			}
+			for k, v := range c.want {
+				if got[k] != v {
+					t.Errorf("monitor %q = %q, want %q", k, got[k], v)
+				}
+			}
+		})
+	}
+}
+
+func TestKumaFactKeyCarriesTheName(t *testing.T) {
+	if got := kumaFactKey("nexus db"); got != "service_down:nexus db" {
+		t.Errorf("kumaFactKey = %q", got)
 	}
 }