Add held-out RU routing fixture and scorer (Vikunja #319)

#319 asks for a measurement before #320 flips the route decider from the
classifier cascade to the resident model. There was nothing to measure
against: the only routing tests assert single utterances, and the
classifier's seed corpus is its own training set — scoring it there
measures memorisation of frozen centroids, which is the illusion that hid
the weak RU query handling in the first place.

internal/router/eval is a separate package so both paths can be scored
from outside router (including cmd/mavend, where the real llama-server
client lives). The fixture is embedded; the scorer takes a Router
interface, so *router.Router and a bare LLM stage both go through the same
76 cases.

The fixture is a CONTRACT, not a snapshot: cases the cascade fails today
stay in the file and fail loudly. TestFixtureIsHeldOut enforces that no
utterance appears verbatim in models/seeds/*.txt.

Baseline, hash embedder at the deployed 0.55 gate: 9/76 (11.8%), 63 false
clarifies, 0 missed clarifies, p50 9µs. Almost everything falls to the
confidence gate — the documented floor behaviour, not a new bug. The
number worth comparing is TestONNXBaseline's (skipped without
MAVEN_ONNX_LIB); the assertions here are a regression ratchet plus a tight
bound on the dangerous direction: ambiguous utterances must not start
being routed confidently.

Seeding is order-fixed on purpose — a few phrases appear under two intents
and map iteration handed them to a different centroid each run, which made
the score jitter between 9 and 10.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01X5JApcrCRVGmqrxnhynSik

internal/router/eval/eval_test.go

@@ -0,0 +1,315 @@
+package eval
+
+import (
+	"bufio"
+	"context"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/kami/maven/internal/config"
+	"github.com/kami/maven/internal/router"
+)
+
+const seedDir = "../../../models/seeds"
+
+// actFns — the verbs the baseline act matcher accepts. In the daemon the
+// allowlist is exactly the enabled tool names (see buildRouter in
+// cmd/mavend/voice.go); here it stands in for a deployment's tools so the
+// fixture's want_fn cases are satisfiable at all. DefaultActMatcher is
+// verb-prefix only, so these are verbs, not full commands.
+var actFns = []string{
+	"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
+	"закрой", "открой", "сделай", "поставь",
+	"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
+}
+
+func TestLoadFixture(t *testing.T) {
+	f, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if _, err := f.Now(); err != nil {
+		t.Fatalf("Now: %v", err)
+	}
+	valid := map[router.Intent]bool{
+		router.IntentAct: true, router.IntentReminder: true, router.IntentFact: true,
+		router.IntentNote: true, router.IntentQuery: true, router.IntentChat: true,
+		router.IntentSystem: true,
+	}
+	seen := map[string]bool{}
+	for _, c := range f.Cases {
+		if c.ID == "" || seen[c.ID] {
+			t.Errorf("case %q: empty or duplicate id", c.ID)
+		}
+		seen[c.ID] = true
+		if strings.TrimSpace(c.Utterance) == "" {
+			t.Errorf("%s: empty utterance", c.ID)
+		}
+		if c.Lang != "ru" && c.Lang != "en" {
+			t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang)
+		}
+		// Intent is empty exactly when the case expects a refusal — otherwise
+		// a typo'd intent would score as an unreachable target forever.
+		if c.WantClarify {
+			if c.Intent != "" {
+				t.Errorf("%s: want_clarify with intent %q — pick one", c.ID, c.Intent)
+			}
+			if c.WantTime || c.WantFn || c.WantFactKey != "" {
+				t.Errorf("%s: want_clarify with slot expectations", c.ID)
+			}
+			continue
+		}
+		if !valid[c.Intent] {
+			t.Errorf("%s: intent %q not in the seven", c.ID, c.Intent)
+		}
+		// Slot expectations must match the intent that owns that slot, or the
+		// case asserts something Extract never fills.
+		if c.WantTime && c.Intent != router.IntentReminder {
+			t.Errorf("%s: want_time on intent %q", c.ID, c.Intent)
+		}
+		if c.WantFn && c.Intent != router.IntentAct {
+			t.Errorf("%s: want_fn on intent %q", c.ID, c.Intent)
+		}
+		if c.WantFactKey != "" && c.Intent != router.IntentFact {
+			t.Errorf("%s: want_fact_key on intent %q", c.ID, c.Intent)
+		}
+	}
+	// Coverage floor: every intent plus the refusal lane must be represented,
+	// or a path can regress to zero without the number moving.
+	byIntent := map[router.Intent]int{}
+	clarify := 0
+	for _, c := range f.Cases {
+		if c.WantClarify {
+			clarify++
+			continue
+		}
+		byIntent[c.Intent]++
+	}
+	for in := range valid {
+		if byIntent[in] < 5 {
+			t.Errorf("intent %q has %d cases, want >= 5", in, byIntent[in])
+		}
+	}
+	if clarify < 5 {
+		t.Errorf("%d clarify cases, want >= 5", clarify)
+	}
+}
+
+// TestFixtureIsHeldOut — the fixture's whole claim to measuring anything. The
+// classifier routes by similarity to frozen seed phrases, so a fixture sharing
+// utterances with models/seeds/*.txt would score its own training set. Verbatim
+// match is the line drawn: paraphrases are the point of the fixture, copies are
+// the failure.
+func TestFixtureIsHeldOut(t *testing.T) {
+	f, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	seeds := loadSeeds(t)
+	for _, c := range f.Cases {
+		if src, ok := seeds[normalize(c.Utterance)]; ok {
+			t.Errorf("%s: %q is verbatim in %s — not held out", c.ID, c.Utterance, src)
+		}
+	}
+}
+
+// TestClassifierBaseline — the number Vikunja #319 compares against. This is
+// the committed stopgap path (stage 0 grammar → nearest-centroid classifier
+// over the real models/seeds corpus), scored on held-out utterances.
+//
+// It runs on HashEmbedder, not the ONNX multilingual embedder: the hash floor
+// is deterministic, so this baseline is reproducible in CI. The production
+// ONNX number will be higher; measure both against the same fixture before
+// flipping the default (#320), and never compare a hash-embedder run to an
+// ONNX one.
+//
+// The assertion is a ratchet, not a target — it only catches a regression
+// below what the cascade already does. The gap between it and 100% is the
+// backlog.
+func TestClassifierBaseline(t *testing.T) {
+	f, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	// dim 1024: the hash embedder is bag-of-words, so a narrower space would
+	// collide tokens across intents and measure the hash, not the centroids.
+	rep, err := Score(context.Background(), "classifier+hash", newBaselineRouter(t, router.NewHashEmbedder(1024)), f)
+	if err != nil {
+		t.Fatalf("Score: %v", err)
+	}
+	t.Log("\n" + rep.String() + rep.Failures())
+
+	// 0.10 is under the observed 0.118, not a target. Almost every case here
+	// falls to clarify because the hash embedder's cosine never clears the
+	// 0.55 gate on paraphrases — which is the documented floor behaviour
+	// (AGENTS.md: "Russian recall rarely clears the confidence gate"), not a
+	// bug this fixture is asking anyone to fix. The number worth moving is
+	// TestONNXBaseline's.
+	const floor = 0.10
+	if rep.Accuracy() < floor {
+		t.Errorf("accuracy %.3f below ratchet %.2f — routing regressed", rep.Accuracy(), floor)
+	}
+	// The dangerous direction is asserted separately and tightly: a confident
+	// route for "сделай это" is a destructive guess, and unlike a miss the user
+	// never gets asked. Clarify cases must not silently start deciding.
+	if rep.MissedClarify > 2 {
+		t.Errorf("%d ambiguous utterances routed confidently, want <= 2:\n%s", rep.MissedClarify, rep.Failures())
+	}
+}
+
+// TestONNXBaseline — the number that actually belongs in Vikunja #319: the
+// deployed cascade with the multilingual ONNX embedder, the configuration
+// homesrv runs. Skipped unless the runtime is present, because the .so is a
+// gitignored ~200MB download and CI has the hash ratchet above instead.
+//
+//	MAVEN_ONNX_LIB=/usr/local/lib/libonnxruntime.so go test -run ONNXBaseline ./internal/router/eval/
+//
+// Reports rather than asserts: the score is an input to the #320 flip decision,
+// and a threshold invented here would just be a second opinion about the same
+// unmeasured thing. Compare it to an LLM-router run over the SAME fixture —
+// accuracy and p50/p95 latency both, since the resident model pays CPU seconds
+// per turn that the classifier's ~10µs does not.
+func TestONNXBaseline(t *testing.T) {
+	lib := os.Getenv("MAVEN_ONNX_LIB")
+	if lib == "" {
+		t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
+	}
+	model := filepath.Join("../../..", "models/embedder/model.onnx")
+	tok := filepath.Join("../../..", "models/embedder/tokenizer.json")
+	for _, p := range []string{lib, model, tok} {
+		if _, err := os.Stat(p); err != nil {
+			t.Skipf("missing %s: %v", p, err)
+		}
+	}
+	emb, err := router.NewONNXEmbedder(model, tok, lib)
+	if err != nil {
+		t.Skipf("onnx embedder unavailable: %v", err)
+	}
+	defer emb.Close()
+
+	f, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	rep, err := Score(context.Background(), "classifier+onnx", newBaselineRouter(t, emb), f)
+	if err != nil {
+		t.Fatalf("Score: %v", err)
+	}
+	t.Log("\n" + rep.String() + rep.Failures())
+}
+
+// newBaselineRouter mirrors buildRouter in cmd/mavend/voice.go — same grammar
+// set, same extractor floors, same seed corpus, same confidence gate — so the
+// score reflects the deployed cascade and not a test-local approximation. Only
+// the embedder varies, and that is the axis being measured.
+func newBaselineRouter(t *testing.T, emb router.Embedder) *router.Router {
+	t.Helper()
+	acts := router.DefaultActMatcher{Fns: actFns}
+	cls := router.NewClassifier(emb)
+	ctx := context.Background()
+	seeds := seedsWithIntent(t)
+	texts := make([]string, 0, len(seeds))
+	for text := range seeds {
+		texts = append(texts, text)
+	}
+	// Sorted: centroids are order-independent, but tie-breaking in Classify's
+	// result sort is not, and a jittering baseline can't be a ratchet.
+	sort.Strings(texts)
+	for _, text := range texts {
+		if err := cls.AddExample(ctx, seeds[text], text); err != nil {
+			t.Fatalf("seed %q: %v", text, err)
+		}
+	}
+	grammars := router.DefaultGrammars(acts)
+	grammars = append(grammars, router.SystemTimeDateGrammars()...)
+	grammars = append(grammars, router.ReminderGrammar())
+	return router.New(router.Config{
+		Grammars:   grammars,
+		Classifier: cls,
+		Extractor: router.Extractor{
+			Time:  router.StubDateTimeParser{},
+			Acts:  acts,
+			Facts: router.DefaultFactParser{},
+		},
+		// The deployed gate, not a test-local one: a fixture scored at a looser
+		// threshold reports an accuracy no real turn would see.
+		Threshold: config.DefaultRouterThreshold,
+	})
+}
+
+// seedOrder — fixed iteration order over the corpus. Not cosmetic: a few
+// phrases appear under two intents ("как дела у сервера" is in both query.txt
+// and system.txt), and ranging over a map would hand the duplicate to a
+// different centroid on every run, which makes the baseline score jitter and
+// the ratchet meaningless.
+var seedOrder = []router.Intent{
+	router.IntentAct, router.IntentReminder, router.IntentFact,
+	router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
+}
+
+// loadSeeds maps normalized seed text → the file it came from.
+func loadSeeds(t *testing.T) map[string]string {
+	t.Helper()
+	files := readSeedFiles(t)
+	out := map[string]string{}
+	for _, intent := range seedOrder {
+		for _, l := range files[intent] {
+			out[normalize(l)] = string(intent) + ".txt"
+		}
+	}
+	return out
+}
+
+// seedsWithIntent flattens the corpus to text→intent. A phrase appearing under
+// two intents collapses to one example — the classifier would otherwise train
+// two centroids to fight over the same vector. First intent in seedOrder wins,
+// deterministically.
+func seedsWithIntent(t *testing.T) map[string]router.Intent {
+	t.Helper()
+	files := readSeedFiles(t)
+	out := map[string]router.Intent{}
+	for _, intent := range seedOrder {
+		for _, l := range files[intent] {
+			if _, dup := out[l]; dup {
+				continue
+			}
+			out[l] = intent
+		}
+	}
+	return out
+}
+
+func readSeedFiles(t *testing.T) map[router.Intent][]string {
+	t.Helper()
+	out := map[router.Intent][]string{}
+	for _, in := range seedOrder {
+		path := filepath.Join(seedDir, string(in)+".txt")
+		fh, err := os.Open(path)
+		if err != nil {
+			t.Fatalf("open %s: %v", path, err)
+		}
+		sc := bufio.NewScanner(fh)
+		for sc.Scan() {
+			line := strings.TrimSpace(sc.Text())
+			if line == "" || strings.HasPrefix(line, "#") {
+				continue
+			}
+			out[in] = append(out[in], line)
+		}
+		err = sc.Err()
+		fh.Close()
+		if err != nil {
+			t.Fatalf("read %s: %v", path, err)
+		}
+	}
+	return out
+}
+
+// normalize — held-out comparison is on lowercased, whitespace-collapsed text
+// so a copied seed can't sneak in behind capitalisation.
+func normalize(s string) string {
+	return strings.Join(strings.Fields(strings.ToLower(s)), " ")
+}