Add held-out RU routing fixture and scorer (Vikunja #319)

#319 asks for a measurement before #320 flips the route decider from the
classifier cascade to the resident model. There was nothing to measure
against: the only routing tests assert single utterances, and the
classifier's seed corpus is its own training set — scoring it there
measures memorisation of frozen centroids, which is the illusion that hid
the weak RU query handling in the first place.

internal/router/eval is a separate package so both paths can be scored
from outside router (including cmd/mavend, where the real llama-server
client lives). The fixture is embedded; the scorer takes a Router
interface, so *router.Router and a bare LLM stage both go through the same
76 cases.

The fixture is a CONTRACT, not a snapshot: cases the cascade fails today
stay in the file and fail loudly. TestFixtureIsHeldOut enforces that no
utterance appears verbatim in models/seeds/*.txt.

Baseline, hash embedder at the deployed 0.55 gate: 9/76 (11.8%), 63 false
clarifies, 0 missed clarifies, p50 9µs. Almost everything falls to the
confidence gate — the documented floor behaviour, not a new bug. The
number worth comparing is TestONNXBaseline's (skipped without
MAVEN_ONNX_LIB); the assertions here are a regression ratchet plus a tight
bound on the dangerous direction: ambiguous utterances must not start
being routed confidently.

Seeding is order-fixed on purpose — a few phrases appear under two intents
and map iteration handed them to a different centroid each run, which made
the score jitter between 9 and 10.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01X5JApcrCRVGmqrxnhynSik

internal/router/eval/eval.go

@@ -0,0 +1,340 @@
+// Package eval is the held-out routing contract — the fixture Vikunja #319
+// measures against before #320 flips the default route decider.
+//
+// Why it is a separate package from router: the fixture must be scorable by
+// BOTH paths (today's classifier cascade and the resident model's LLM router)
+// from outside the router package, including from cmd/mavend where the real
+// llama-server client lives. A _test.go file in router can't be imported, and
+// testdata isn't reachable from another package's working directory — so the
+// fixture is embedded here and the scorer takes a Router interface.
+//
+// The fixture is HELD OUT from models/seeds/*.txt on purpose: a classifier
+// scored on its own seed phrases measures memorisation of frozen centroids,
+// which is exactly the illusion that hid the weak RU query handling. See
+// TestFixtureIsHeldOut, which enforces it.
+package eval
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/kami/maven/internal/router"
+)
+
+//go:embed ru_routing_v1.json
+var fixtureJSON []byte
+
+// Case — one utterance and the route it must produce. Slot expectations are
+// deliberately coarse (see the fixture's notes): want_fn is a boolean because
+// the fn allowlist lives in deploy config, and want_fact_key names the loop's
+// rule keys because a fact under the wrong key starves its predicate.
+//
+// Intent is empty exactly when WantClarify is set: the contract there is that
+// the router refuses instead of guessing.
+type Case struct {
+	ID          string        `json:"id"`
+	Utterance   string        `json:"utterance"`
+	Lang        string        `json:"lang"`
+	Intent      router.Intent `json:"intent"`
+	WantTime    bool          `json:"want_time"`
+	WantFn      bool          `json:"want_fn"`
+	WantFactKey string        `json:"want_fact_key"`
+	WantClarify bool          `json:"want_clarify"`
+	Tags        []string      `json:"tags"`
+	Note        string        `json:"note"`
+}
+
+// Fixture — the versioned envelope, same shape as
+// cmd/mavend/testdata/system_safety_scenarios.json. SchemaVersion gates the
+// loader so an older binary refuses a fixture it would misread rather than
+// scoring it wrong and reporting a number.
+type Fixture struct {
+	SchemaVersion int      `json:"schema_version"`
+	Name          string   `json:"name"`
+	ReferenceNow  string   `json:"reference_now"`
+	Notes         []string `json:"notes"`
+	Cases         []Case   `json:"cases"`
+}
+
+// SchemaVersion — the version this package understands.
+const SchemaVersion = 1
+
+// Load returns the embedded fixture.
+func Load() (Fixture, error) {
+	var f Fixture
+	if err := json.Unmarshal(fixtureJSON, &f); err != nil {
+		return Fixture{}, fmt.Errorf("parse fixture: %w", err)
+	}
+	if f.SchemaVersion != SchemaVersion {
+		return Fixture{}, fmt.Errorf("fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion)
+	}
+	if len(f.Cases) == 0 {
+		return Fixture{}, fmt.Errorf("fixture has no cases")
+	}
+	return f, nil
+}
+
+// Now — the fixture's reference clock. Relative reminder slots ("через
+// полчаса") resolve against it, so a scoring run is reproducible regardless of
+// when it runs.
+func (f Fixture) Now() (time.Time, error) {
+	t, err := time.Parse(time.RFC3339, f.ReferenceNow)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("parse reference_now %q: %w", f.ReferenceNow, err)
+	}
+	return t, nil
+}
+
+// Router — the one thing a route decider must do to be scorable. *router.Router
+// satisfies it directly; an LLM-only path wraps its Route in RouterFunc.
+type Router interface {
+	Route(ctx context.Context, utterance string, now time.Time) (router.Decision, error)
+}
+
+// RouterFunc adapts a bare function to Router — for scoring a single stage
+// (e.g. *router.LLMRouter, whose Route returns an extra ok bool) without
+// standing up the whole cascade.
+type RouterFunc func(ctx context.Context, utterance string, now time.Time) (router.Decision, error)
+
+// Route implements Router.
+func (f RouterFunc) Route(ctx context.Context, utterance string, now time.Time) (router.Decision, error) {
+	return f(ctx, utterance, now)
+}
+
+// Outcome — one scored case. Reasons is empty exactly when Pass is true.
+type Outcome struct {
+	Case     Case
+	Decision router.Decision
+	Err      error
+	Latency  time.Duration
+	Pass     bool
+	// IntentOK is tracked separately from Pass: a case can land the right
+	// intent and still fail on a slot, and #319 needs those two numbers apart
+	// (a slot gap is a parser fix; a wrong intent is a router fix).
+	IntentOK bool
+	Reasons  []string
+}
+
+// Report — the aggregate. Accuracy is the headline; the rest exists so a
+// regression names itself instead of just moving a percentage.
+type Report struct {
+	Name      string
+	Total     int
+	Passed    int
+	IntentHit int
+	// FalseClarify — the router asked when the fixture expected a decision.
+	// A gap, recoverable by asking again.
+	FalseClarify int
+	// MissedClarify — the router decided confidently where the fixture
+	// expected a refusal. The dangerous direction: "сделай это" routed to an
+	// act is a confident destructive guess.
+	MissedClarify int
+	Errors        int
+	Outcomes      []Outcome
+	// Confusion counts want→got intent pairs, decided cases only.
+	Confusion map[string]int
+	// ByTag accuracy for the fixture's tags ("hard", "homelab", …).
+	ByTag map[string]TagStat
+	// ByLang accuracy — the RU/EN split is the whole reason this fixture
+	// exists.
+	ByLang map[string]TagStat
+	P50    time.Duration
+	P95    time.Duration
+	Max    time.Duration
+}
+
+// TagStat — passed/total for one slice of the fixture.
+type TagStat struct{ Passed, Total int }
+
+// Accuracy — fraction of cases fully satisfied (intent AND slots AND the
+// clarify contract).
+func (r Report) Accuracy() float64 {
+	if r.Total == 0 {
+		return 0
+	}
+	return float64(r.Passed) / float64(r.Total)
+}
+
+// IntentAccuracy — fraction with the right intent, ignoring slot fills.
+func (r Report) IntentAccuracy() float64 {
+	if r.Total == 0 {
+		return 0
+	}
+	return float64(r.IntentHit) / float64(r.Total)
+}
+
+// Score runs every case through r and aggregates. It never fails the run on a
+// route error — an erroring case scores as a miss and is counted in Errors,
+// because "the model was down" and "the model was wrong" are different numbers
+// and #319 needs to tell them apart.
+//
+// Latency is wall-clock per Route call, including any llama-server round trip.
+// That is the point on the CPU-only target: a correctness win the resident
+// model pays for with seconds per turn is not a win.
+func Score(ctx context.Context, name string, r Router, f Fixture) (Report, error) {
+	now, err := f.Now()
+	if err != nil {
+		return Report{}, err
+	}
+	rep := Report{
+		Name:      name,
+		Total:     len(f.Cases),
+		Confusion: map[string]int{},
+		ByTag:     map[string]TagStat{},
+		ByLang:    map[string]TagStat{},
+	}
+	lat := make([]time.Duration, 0, len(f.Cases))
+
+	for _, c := range f.Cases {
+		start := time.Now()
+		d, err := r.Route(ctx, c.Utterance, now)
+		o := Outcome{Case: c, Decision: d, Err: err, Latency: time.Since(start)}
+		lat = append(lat, o.Latency)
+
+		switch {
+		case err != nil:
+			rep.Errors++
+			o.Reasons = append(o.Reasons, fmt.Sprintf("route error: %v", err))
+		case c.WantClarify:
+			// Only the refusal matters here; whatever intent the cascade
+			// guessed underneath is irrelevant if it gated.
+			o.IntentOK = d.Clarify
+			if !d.Clarify {
+				rep.MissedClarify++
+				o.Reasons = append(o.Reasons, fmt.Sprintf("decided %q confidently (%.3f), want clarify", d.Intent, d.Confidence))
+			}
+		default:
+			o.IntentOK = d.Intent == c.Intent && !d.Clarify
+			if d.Clarify {
+				rep.FalseClarify++
+				o.Reasons = append(o.Reasons, fmt.Sprintf("clarified (%.3f), want intent %q", d.Confidence, c.Intent))
+			} else if d.Intent != c.Intent {
+				rep.Confusion[string(c.Intent)+"→"+string(d.Intent)]++
+				o.Reasons = append(o.Reasons, fmt.Sprintf("intent %q, want %q (%.3f)", d.Intent, c.Intent, d.Confidence))
+			}
+			if c.WantTime && !d.Slots.HasTime {
+				o.Reasons = append(o.Reasons, "no time slot, want one")
+			}
+			if c.WantFn && !d.Slots.HasFn {
+				o.Reasons = append(o.Reasons, "no fn slot, want an allowlist match")
+			}
+			if c.WantFactKey != "" && d.Slots.Key != c.WantFactKey {
+				o.Reasons = append(o.Reasons, fmt.Sprintf("fact key %q, want %q", d.Slots.Key, c.WantFactKey))
+			}
+		}
+
+		o.Pass = len(o.Reasons) == 0
+		if o.Pass {
+			rep.Passed++
+		}
+		if o.IntentOK {
+			rep.IntentHit++
+		}
+		bump(rep.ByLang, c.Lang, o.Pass)
+		for _, tag := range c.Tags {
+			bump(rep.ByTag, tag, o.Pass)
+		}
+		rep.Outcomes = append(rep.Outcomes, o)
+	}
+
+	sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] })
+	rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95)
+	if len(lat) > 0 {
+		rep.Max = lat[len(lat)-1]
+	}
+	return rep, nil
+}
+
+func bump(m map[string]TagStat, key string, pass bool) {
+	if key == "" {
+		return
+	}
+	s := m[key]
+	s.Total++
+	if pass {
+		s.Passed++
+	}
+	m[key] = s
+}
+
+// percentile — nearest-rank on a pre-sorted slice. No interpolation: with ~80
+// samples an interpolated p95 invents a latency no turn actually took.
+func percentile(sorted []time.Duration, p float64) time.Duration {
+	if len(sorted) == 0 {
+		return 0
+	}
+	i := int(p * float64(len(sorted)))
+	if i >= len(sorted) {
+		i = len(sorted) - 1
+	}
+	return sorted[i]
+}
+
+// String renders the report as the comparison table #319 pastes into the task —
+// headline accuracy, the two clarify directions apart, latency, and the slices
+// that name where a path is weak.
+func (r Report) String() string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "%s: %d/%d cases (%.1f%% full, %.1f%% intent-only)\n",
+		r.Name, r.Passed, r.Total, 100*r.Accuracy(), 100*r.IntentAccuracy())
+	fmt.Fprintf(&b, "  clarify: %d false (asked, shouldn't) / %d missed (guessed, shouldn't) | errors: %d\n",
+		r.FalseClarify, r.MissedClarify, r.Errors)
+	fmt.Fprintf(&b, "  latency: p50 %s  p95 %s  max %s\n", r.P50, r.P95, r.Max)
+	fmt.Fprintf(&b, "  by lang: %s\n", renderStats(r.ByLang))
+	fmt.Fprintf(&b, "  by tag:  %s\n", renderStats(r.ByTag))
+	if len(r.Confusion) > 0 {
+		fmt.Fprintf(&b, "  confusion: %s\n", renderCounts(r.Confusion))
+	}
+	return b.String()
+}
+
+// Failures — the per-case detail, sorted by ID so two runs diff cleanly.
+func (r Report) Failures() string {
+	var b strings.Builder
+	out := append([]Outcome(nil), r.Outcomes...)
+	sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID })
+	for _, o := range out {
+		if o.Pass {
+			continue
+		}
+		fmt.Fprintf(&b, "  %s %q: %s\n", o.Case.ID, o.Case.Utterance, strings.Join(o.Reasons, "; "))
+	}
+	return b.String()
+}
+
+func renderStats(m map[string]TagStat) string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	parts := make([]string, 0, len(keys))
+	for _, k := range keys {
+		s := m[k]
+		parts = append(parts, fmt.Sprintf("%s %d/%d", k, s.Passed, s.Total))
+	}
+	return strings.Join(parts, "  ")
+}
+
+func renderCounts(m map[string]int) string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		if m[keys[i]] != m[keys[j]] {
+			return m[keys[i]] > m[keys[j]]
+		}
+		return keys[i] < keys[j]
+	})
+	parts := make([]string, 0, len(keys))
+	for _, k := range keys {
+		parts = append(parts, fmt.Sprintf("%s ×%d", k, m[k]))
+	}
+	return strings.Join(parts, "  ")
+}