internal/router/eval/reach_test.go

@@ -0,0 +1,228 @@
+package eval
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/kami/maven/internal/router"
+)
+
+func TestLoadReachFixture(t *testing.T) {
+	f, err := LoadReach()
+	if err != nil {
+		t.Fatalf("LoadReach: %v", err)
+	}
+	if _, err := f.Now(); err != nil {
+		t.Fatalf("Now: %v", err)
+	}
+	valid := map[Service]bool{ServiceNone: true, ServicePraxis: true, ServiceHexis: true}
+	seen := map[string]bool{}
+	byService := map[Service]int{}
+	for _, c := range f.Cases {
+		if c.ID == "" || seen[c.ID] {
+			t.Errorf("case %q: empty or duplicate id", c.ID)
+		}
+		seen[c.ID] = true
+		if !valid[c.WantService] {
+			t.Errorf("%s: want_service %q, want none|praxis|hexis", c.ID, c.WantService)
+		}
+		if c.Lang != "ru" && c.Lang != "en" {
+			t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang)
+		}
+		// A capability only means something on the Praxis side, and it must
+		// name an arm that exists — otherwise the case asserts a target the
+		// dispatch can never hit.
+		if c.WantCapability != "" {
+			if c.WantService != ServicePraxis {
+				t.Errorf("%s: want_capability on %q", c.ID, c.WantService)
+			}
+			if PraxisAliases[c.WantCapability] != c.WantCapability {
+				t.Errorf("%s: capability %q is not a praxis arm", c.ID, c.WantCapability)
+			}
+		}
+		byService[c.WantService]++
+	}
+	// Coverage floor. The negative cases matter most: without them a router
+	// that sent everything to Hexis would score perfectly.
+	for s := range valid {
+		if byService[s] < 5 {
+			t.Errorf("want_service %q has %d cases, want >= 5", s, byService[s])
+		}
+	}
+}
+
+// TestReachFixtureIsHeldOut — same rule as TestFixtureIsHeldOut. The classifier
+// routes by similarity to frozen seeds, so a case copied from models/seeds
+// would measure memorisation rather than reach.
+func TestReachFixtureIsHeldOut(t *testing.T) {
+	f, err := LoadReach()
+	if err != nil {
+		t.Fatalf("LoadReach: %v", err)
+	}
+	seeds := loadSeeds(t)
+	for _, c := range f.Cases {
+		if src, ok := seeds[normalize(c.Utterance)]; ok {
+			t.Errorf("%s: %q is verbatim in %s — not held out", c.ID, c.Utterance, src)
+		}
+	}
+}
+
+// TestReachDerivation pins the gate order that ScoreReach depends on. These are
+// the four branches of actionAct plus the clarify pre-empt, asserted directly
+// so a change to cmd/mavend/actions_act.go that this package no longer mirrors
+// fails here rather than silently moving the score.
+func TestReachDerivation(t *testing.T) {
+	m := router.DefaultActMatcher{Fns: actFns}
+	cases := []struct {
+		name       string
+		dec        router.Decision
+		want       Service
+		capability string
+	}{
+		{
+			name: "act with a praxis alias in the fn slot",
+			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "list_attention", HasFn: true}},
+			want: ServicePraxis, capability: "list_attention",
+		},
+		{
+			name: "act with a russian praxis alias",
+			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "готово", HasFn: true}},
+			want: ServicePraxis, capability: "resolve_item",
+		},
+		{
+			name: "act with an entity but no fn earns one from the matcher",
+			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Text: "выключи свет в спальне"}},
+			want: ServiceHexis,
+		},
+		{
+			name: "act with a non-praxis fn and text goes to hexis",
+			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "restart", HasFn: true, Text: "перезапусти гитею"}},
+			want: ServiceHexis,
+		},
+		{
+			name: "act with a fn and no text stays local",
+			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "restart", HasFn: true}},
+			want: ServiceNone,
+		},
+		{
+			name: "a clarified act with text still reaches hexis",
+			dec:  router.Decision{Intent: router.IntentAct, Clarify: true, Slots: router.Slots{Text: "выключи это"}},
+			want: ServiceHexis,
+		},
+		{
+			name: "a clarified act that already has a fn does not",
+			dec:  router.Decision{Intent: router.IntentAct, Clarify: true, Slots: router.Slots{Fn: "restart", HasFn: true, Text: "перезапусти"}},
+			want: ServiceNone,
+		},
+		{
+			name: "no intent but act reaches nothing",
+			dec:  router.Decision{Intent: router.IntentQuery, Slots: router.Slots{Fn: "list_attention", HasFn: true, Text: "что требует внимания"}},
+			want: ServiceNone,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, capability := Reach(tc.dec, m)
+			if got != tc.want {
+				t.Errorf("Reach = %q, want %q", got, tc.want)
+			}
+			if capability != tc.capability {
+				t.Errorf("capability = %q, want %q", capability, tc.capability)
+			}
+		})
+	}
+}
+
+// TestPraxisAliasesShape — the copy of praxisCapabilities in reach.go is a
+// drift risk (see its comment). This does not close it, but it does catch the
+// cheap half: an arm losing all its aliases, or an alias pointing at an arm
+// that no longer has a canonical name.
+func TestPraxisAliasesShape(t *testing.T) {
+	arms := map[string]int{}
+	for alias, capability := range PraxisAliases {
+		if alias == "" || capability == "" {
+			t.Errorf("empty alias or capability: %q → %q", alias, capability)
+		}
+		arms[capability]++
+	}
+	for _, want := range []string{
+		"list_attention", "acknowledge_item", "resolve_item",
+		"ignore_item", "pin_item", "list_changes", "entity_attention",
+	} {
+		if arms[want] == 0 {
+			t.Errorf("praxis arm %q has no aliases", want)
+		}
+		// Every arm must be reachable by its own name, which is the alias the
+		// LLM router's fn slot actually emits.
+		if PraxisAliases[want] != want {
+			t.Errorf("arm %q does not alias to itself", want)
+		}
+	}
+}
+
+// TestReachBaselineHash — the deterministic ratchet. Same deal as
+// TestClassifierBaseline: the hash embedder never clears the confidence gate on
+// paraphrases, so almost everything lands on clarify, and the number this
+// asserts is the overreach count rather than the accuracy.
+//
+// Overreach is the direction worth a hard assertion. An utterance that should
+// stay inside Maven and instead resolves an entity through Nexus is one Hexis
+// capability away from executing something nobody asked for, and unlike a miss
+// he never gets asked about it.
+func TestReachBaselineHash(t *testing.T) {
+	f, err := LoadReach()
+	if err != nil {
+		t.Fatalf("LoadReach: %v", err)
+	}
+	m := router.DefaultActMatcher{Fns: actFns}
+	rep, err := ScoreReach(context.Background(), "reach: classifier+hash",
+		newBaselineRouter(t, router.NewHashEmbedder(1024), nil), m, f)
+	if err != nil {
+		t.Fatalf("ScoreReach: %v", err)
+	}
+	t.Log("\n" + rep.String() + rep.Failures())
+
+	if rep.Overreach > 4 {
+		t.Errorf("%d utterances reached a service they should not have, want <= 4:\n%s",
+			rep.Overreach, rep.Failures())
+	}
+}
+
+// TestReachBaselineONNX — the deployed configuration: the cascade with the
+// multilingual e5 embedder. Opt-in via MAVEN_ONNX_LIB, same as TestONNXBaseline,
+// because deps/ is gitignored. `make eval-reach` points it at the vendored copy.
+//
+// Reports rather than asserts. This is the number Vikunja #405 asked for, and a
+// threshold invented alongside the first measurement is not a ratchet, it is a
+// guess written down twice.
+func TestReachBaselineONNX(t *testing.T) {
+	lib := os.Getenv("MAVEN_ONNX_LIB")
+	if lib == "" {
+		t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
+	}
+	model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx")
+	tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json")
+	for _, p := range []string{lib, model, tok} {
+		if _, err := os.Stat(p); err != nil {
+			t.Skipf("missing %s: %v", p, err)
+		}
+	}
+	emb, err := router.NewONNXEmbedder(model, tok, lib)
+	if err != nil {
+		t.Skipf("onnx embedder unavailable: %v", err)
+	}
+	defer emb.Close()
+
+	f, err := LoadReach()
+	if err != nil {
+		t.Fatalf("LoadReach: %v", err)
+	}
+	rep, err := ScoreReach(context.Background(), "reach: classifier+onnx",
+		newBaselineRouter(t, emb, nil), router.DefaultActMatcher{Fns: actFns}, f)
+	if err != nil {
+		t.Fatalf("ScoreReach: %v", err)
+	}
+	t.Log("\n" + rep.String() + rep.Failures())
+}