From c7c44229a244e3888a870afae3836d671a883ade Mon Sep 17 00:00:00 2001 From: kami Date: Fri, 31 Jul 2026 00:28:44 +0400 Subject: [PATCH] Add held-out RU routing fixture and scorer (Vikunja #319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #319 asks for a measurement before #320 flips the route decider from the classifier cascade to the resident model. There was nothing to measure against: the only routing tests assert single utterances, and the classifier's seed corpus is its own training set — scoring it there measures memorisation of frozen centroids, which is the illusion that hid the weak RU query handling in the first place. internal/router/eval is a separate package so both paths can be scored from outside router (including cmd/mavend, where the real llama-server client lives). The fixture is embedded; the scorer takes a Router interface, so *router.Router and a bare LLM stage both go through the same 76 cases. The fixture is a CONTRACT, not a snapshot: cases the cascade fails today stay in the file and fail loudly. TestFixtureIsHeldOut enforces that no utterance appears verbatim in models/seeds/*.txt. Baseline, hash embedder at the deployed 0.55 gate: 9/76 (11.8%), 63 false clarifies, 0 missed clarifies, p50 9µs. Almost everything falls to the confidence gate — the documented floor behaviour, not a new bug. The number worth comparing is TestONNXBaseline's (skipped without MAVEN_ONNX_LIB); the assertions here are a regression ratchet plus a tight bound on the dangerous direction: ambiguous utterances must not start being routed confidently. Seeding is order-fixed on purpose — a few phrases appear under two intents and map iteration handed them to a different centroid each run, which made the score jitter between 9 and 10. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01X5JApcrCRVGmqrxnhynSik --- Makefile | 9 +- internal/router/eval/eval.go | 340 ++++++++++++++++++++++++ internal/router/eval/eval_test.go | 315 ++++++++++++++++++++++ internal/router/eval/ru_routing_v1.json | 96 +++++++ 4 files changed, 759 insertions(+), 1 deletion(-) create mode 100644 internal/router/eval/eval.go create mode 100644 internal/router/eval/eval_test.go create mode 100644 internal/router/eval/ru_routing_v1.json diff --git a/Makefile b/Makefile index 1cdf048..a7ab586 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data -.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go +.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go eval-router all: build @@ -73,6 +73,13 @@ test: CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ $(GO) test -race -coverprofile=coverage.out ./internal/... ./cmd/... +# eval-router — score the held-out RU routing fixture (internal/router/eval). +# Verbose so the report table lands in the terminal; MAVEN_ONNX_LIB additionally +# runs the prod-representative ONNX baseline (skipped without it). This is the +# measurement Vikunja #319 compares before #320 flips the route decider. +eval-router: + MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/ + run-stt: build-stt LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ ./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL) diff --git a/internal/router/eval/eval.go b/internal/router/eval/eval.go new file mode 100644 index 0000000..35fb623 --- /dev/null +++ b/internal/router/eval/eval.go @@ -0,0 +1,340 @@ +// Package eval is the held-out routing contract — the fixture Vikunja #319 +// measures against before #320 flips the default route decider. +// +// Why it is a separate package from router: the fixture must be scorable by +// BOTH paths (today's classifier cascade and the resident model's LLM router) +// from outside the router package, including from cmd/mavend where the real +// llama-server client lives. A _test.go file in router can't be imported, and +// testdata isn't reachable from another package's working directory — so the +// fixture is embedded here and the scorer takes a Router interface. +// +// The fixture is HELD OUT from models/seeds/*.txt on purpose: a classifier +// scored on its own seed phrases measures memorisation of frozen centroids, +// which is exactly the illusion that hid the weak RU query handling. See +// TestFixtureIsHeldOut, which enforces it. +package eval + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/kami/maven/internal/router" +) + +//go:embed ru_routing_v1.json +var fixtureJSON []byte + +// Case — one utterance and the route it must produce. Slot expectations are +// deliberately coarse (see the fixture's notes): want_fn is a boolean because +// the fn allowlist lives in deploy config, and want_fact_key names the loop's +// rule keys because a fact under the wrong key starves its predicate. +// +// Intent is empty exactly when WantClarify is set: the contract there is that +// the router refuses instead of guessing. +type Case struct { + ID string `json:"id"` + Utterance string `json:"utterance"` + Lang string `json:"lang"` + Intent router.Intent `json:"intent"` + WantTime bool `json:"want_time"` + WantFn bool `json:"want_fn"` + WantFactKey string `json:"want_fact_key"` + WantClarify bool `json:"want_clarify"` + Tags []string `json:"tags"` + Note string `json:"note"` +} + +// Fixture — the versioned envelope, same shape as +// cmd/mavend/testdata/system_safety_scenarios.json. SchemaVersion gates the +// loader so an older binary refuses a fixture it would misread rather than +// scoring it wrong and reporting a number. +type Fixture struct { + SchemaVersion int `json:"schema_version"` + Name string `json:"name"` + ReferenceNow string `json:"reference_now"` + Notes []string `json:"notes"` + Cases []Case `json:"cases"` +} + +// SchemaVersion — the version this package understands. +const SchemaVersion = 1 + +// Load returns the embedded fixture. +func Load() (Fixture, error) { + var f Fixture + if err := json.Unmarshal(fixtureJSON, &f); err != nil { + return Fixture{}, fmt.Errorf("parse fixture: %w", err) + } + if f.SchemaVersion != SchemaVersion { + return Fixture{}, fmt.Errorf("fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion) + } + if len(f.Cases) == 0 { + return Fixture{}, fmt.Errorf("fixture has no cases") + } + return f, nil +} + +// Now — the fixture's reference clock. Relative reminder slots ("через +// полчаса") resolve against it, so a scoring run is reproducible regardless of +// when it runs. +func (f Fixture) Now() (time.Time, error) { + t, err := time.Parse(time.RFC3339, f.ReferenceNow) + if err != nil { + return time.Time{}, fmt.Errorf("parse reference_now %q: %w", f.ReferenceNow, err) + } + return t, nil +} + +// Router — the one thing a route decider must do to be scorable. *router.Router +// satisfies it directly; an LLM-only path wraps its Route in RouterFunc. +type Router interface { + Route(ctx context.Context, utterance string, now time.Time) (router.Decision, error) +} + +// RouterFunc adapts a bare function to Router — for scoring a single stage +// (e.g. *router.LLMRouter, whose Route returns an extra ok bool) without +// standing up the whole cascade. +type RouterFunc func(ctx context.Context, utterance string, now time.Time) (router.Decision, error) + +// Route implements Router. +func (f RouterFunc) Route(ctx context.Context, utterance string, now time.Time) (router.Decision, error) { + return f(ctx, utterance, now) +} + +// Outcome — one scored case. Reasons is empty exactly when Pass is true. +type Outcome struct { + Case Case + Decision router.Decision + Err error + Latency time.Duration + Pass bool + // IntentOK is tracked separately from Pass: a case can land the right + // intent and still fail on a slot, and #319 needs those two numbers apart + // (a slot gap is a parser fix; a wrong intent is a router fix). + IntentOK bool + Reasons []string +} + +// Report — the aggregate. Accuracy is the headline; the rest exists so a +// regression names itself instead of just moving a percentage. +type Report struct { + Name string + Total int + Passed int + IntentHit int + // FalseClarify — the router asked when the fixture expected a decision. + // A gap, recoverable by asking again. + FalseClarify int + // MissedClarify — the router decided confidently where the fixture + // expected a refusal. The dangerous direction: "сделай это" routed to an + // act is a confident destructive guess. + MissedClarify int + Errors int + Outcomes []Outcome + // Confusion counts want→got intent pairs, decided cases only. + Confusion map[string]int + // ByTag accuracy for the fixture's tags ("hard", "homelab", …). + ByTag map[string]TagStat + // ByLang accuracy — the RU/EN split is the whole reason this fixture + // exists. + ByLang map[string]TagStat + P50 time.Duration + P95 time.Duration + Max time.Duration +} + +// TagStat — passed/total for one slice of the fixture. +type TagStat struct{ Passed, Total int } + +// Accuracy — fraction of cases fully satisfied (intent AND slots AND the +// clarify contract). +func (r Report) Accuracy() float64 { + if r.Total == 0 { + return 0 + } + return float64(r.Passed) / float64(r.Total) +} + +// IntentAccuracy — fraction with the right intent, ignoring slot fills. +func (r Report) IntentAccuracy() float64 { + if r.Total == 0 { + return 0 + } + return float64(r.IntentHit) / float64(r.Total) +} + +// Score runs every case through r and aggregates. It never fails the run on a +// route error — an erroring case scores as a miss and is counted in Errors, +// because "the model was down" and "the model was wrong" are different numbers +// and #319 needs to tell them apart. +// +// Latency is wall-clock per Route call, including any llama-server round trip. +// That is the point on the CPU-only target: a correctness win the resident +// model pays for with seconds per turn is not a win. +func Score(ctx context.Context, name string, r Router, f Fixture) (Report, error) { + now, err := f.Now() + if err != nil { + return Report{}, err + } + rep := Report{ + Name: name, + Total: len(f.Cases), + Confusion: map[string]int{}, + ByTag: map[string]TagStat{}, + ByLang: map[string]TagStat{}, + } + lat := make([]time.Duration, 0, len(f.Cases)) + + for _, c := range f.Cases { + start := time.Now() + d, err := r.Route(ctx, c.Utterance, now) + o := Outcome{Case: c, Decision: d, Err: err, Latency: time.Since(start)} + lat = append(lat, o.Latency) + + switch { + case err != nil: + rep.Errors++ + o.Reasons = append(o.Reasons, fmt.Sprintf("route error: %v", err)) + case c.WantClarify: + // Only the refusal matters here; whatever intent the cascade + // guessed underneath is irrelevant if it gated. + o.IntentOK = d.Clarify + if !d.Clarify { + rep.MissedClarify++ + o.Reasons = append(o.Reasons, fmt.Sprintf("decided %q confidently (%.3f), want clarify", d.Intent, d.Confidence)) + } + default: + o.IntentOK = d.Intent == c.Intent && !d.Clarify + if d.Clarify { + rep.FalseClarify++ + o.Reasons = append(o.Reasons, fmt.Sprintf("clarified (%.3f), want intent %q", d.Confidence, c.Intent)) + } else if d.Intent != c.Intent { + rep.Confusion[string(c.Intent)+"→"+string(d.Intent)]++ + o.Reasons = append(o.Reasons, fmt.Sprintf("intent %q, want %q (%.3f)", d.Intent, c.Intent, d.Confidence)) + } + if c.WantTime && !d.Slots.HasTime { + o.Reasons = append(o.Reasons, "no time slot, want one") + } + if c.WantFn && !d.Slots.HasFn { + o.Reasons = append(o.Reasons, "no fn slot, want an allowlist match") + } + if c.WantFactKey != "" && d.Slots.Key != c.WantFactKey { + o.Reasons = append(o.Reasons, fmt.Sprintf("fact key %q, want %q", d.Slots.Key, c.WantFactKey)) + } + } + + o.Pass = len(o.Reasons) == 0 + if o.Pass { + rep.Passed++ + } + if o.IntentOK { + rep.IntentHit++ + } + bump(rep.ByLang, c.Lang, o.Pass) + for _, tag := range c.Tags { + bump(rep.ByTag, tag, o.Pass) + } + rep.Outcomes = append(rep.Outcomes, o) + } + + sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] }) + rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95) + if len(lat) > 0 { + rep.Max = lat[len(lat)-1] + } + return rep, nil +} + +func bump(m map[string]TagStat, key string, pass bool) { + if key == "" { + return + } + s := m[key] + s.Total++ + if pass { + s.Passed++ + } + m[key] = s +} + +// percentile — nearest-rank on a pre-sorted slice. No interpolation: with ~80 +// samples an interpolated p95 invents a latency no turn actually took. +func percentile(sorted []time.Duration, p float64) time.Duration { + if len(sorted) == 0 { + return 0 + } + i := int(p * float64(len(sorted))) + if i >= len(sorted) { + i = len(sorted) - 1 + } + return sorted[i] +} + +// String renders the report as the comparison table #319 pastes into the task — +// headline accuracy, the two clarify directions apart, latency, and the slices +// that name where a path is weak. +func (r Report) String() string { + var b strings.Builder + fmt.Fprintf(&b, "%s: %d/%d cases (%.1f%% full, %.1f%% intent-only)\n", + r.Name, r.Passed, r.Total, 100*r.Accuracy(), 100*r.IntentAccuracy()) + fmt.Fprintf(&b, " clarify: %d false (asked, shouldn't) / %d missed (guessed, shouldn't) | errors: %d\n", + r.FalseClarify, r.MissedClarify, r.Errors) + fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max) + fmt.Fprintf(&b, " by lang: %s\n", renderStats(r.ByLang)) + fmt.Fprintf(&b, " by tag: %s\n", renderStats(r.ByTag)) + if len(r.Confusion) > 0 { + fmt.Fprintf(&b, " confusion: %s\n", renderCounts(r.Confusion)) + } + return b.String() +} + +// Failures — the per-case detail, sorted by ID so two runs diff cleanly. +func (r Report) Failures() string { + var b strings.Builder + out := append([]Outcome(nil), r.Outcomes...) + sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID }) + for _, o := range out { + if o.Pass { + continue + } + fmt.Fprintf(&b, " %s %q: %s\n", o.Case.ID, o.Case.Utterance, strings.Join(o.Reasons, "; ")) + } + return b.String() +} + +func renderStats(m map[string]TagStat) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + s := m[k] + parts = append(parts, fmt.Sprintf("%s %d/%d", k, s.Passed, s.Total)) + } + return strings.Join(parts, " ") +} + +func renderCounts(m map[string]int) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + if m[keys[i]] != m[keys[j]] { + return m[keys[i]] > m[keys[j]] + } + return keys[i] < keys[j] + }) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s ×%d", k, m[k])) + } + return strings.Join(parts, " ") +} diff --git a/internal/router/eval/eval_test.go b/internal/router/eval/eval_test.go new file mode 100644 index 0000000..e6290be --- /dev/null +++ b/internal/router/eval/eval_test.go @@ -0,0 +1,315 @@ +package eval + +import ( + "bufio" + "context" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/router" +) + +const seedDir = "../../../models/seeds" + +// actFns — the verbs the baseline act matcher accepts. In the daemon the +// allowlist is exactly the enabled tool names (see buildRouter in +// cmd/mavend/voice.go); here it stands in for a deployment's tools so the +// fixture's want_fn cases are satisfiable at all. DefaultActMatcher is +// verb-prefix only, so these are verbs, not full commands. +var actFns = []string{ + "перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти", + "закрой", "открой", "сделай", "поставь", + "restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close", +} + +func TestLoadFixture(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + if _, err := f.Now(); err != nil { + t.Fatalf("Now: %v", err) + } + valid := map[router.Intent]bool{ + router.IntentAct: true, router.IntentReminder: true, router.IntentFact: true, + router.IntentNote: true, router.IntentQuery: true, router.IntentChat: true, + router.IntentSystem: true, + } + seen := map[string]bool{} + for _, c := range f.Cases { + if c.ID == "" || seen[c.ID] { + t.Errorf("case %q: empty or duplicate id", c.ID) + } + seen[c.ID] = true + if strings.TrimSpace(c.Utterance) == "" { + t.Errorf("%s: empty utterance", c.ID) + } + if c.Lang != "ru" && c.Lang != "en" { + t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang) + } + // Intent is empty exactly when the case expects a refusal — otherwise + // a typo'd intent would score as an unreachable target forever. + if c.WantClarify { + if c.Intent != "" { + t.Errorf("%s: want_clarify with intent %q — pick one", c.ID, c.Intent) + } + if c.WantTime || c.WantFn || c.WantFactKey != "" { + t.Errorf("%s: want_clarify with slot expectations", c.ID) + } + continue + } + if !valid[c.Intent] { + t.Errorf("%s: intent %q not in the seven", c.ID, c.Intent) + } + // Slot expectations must match the intent that owns that slot, or the + // case asserts something Extract never fills. + if c.WantTime && c.Intent != router.IntentReminder { + t.Errorf("%s: want_time on intent %q", c.ID, c.Intent) + } + if c.WantFn && c.Intent != router.IntentAct { + t.Errorf("%s: want_fn on intent %q", c.ID, c.Intent) + } + if c.WantFactKey != "" && c.Intent != router.IntentFact { + t.Errorf("%s: want_fact_key on intent %q", c.ID, c.Intent) + } + } + // Coverage floor: every intent plus the refusal lane must be represented, + // or a path can regress to zero without the number moving. + byIntent := map[router.Intent]int{} + clarify := 0 + for _, c := range f.Cases { + if c.WantClarify { + clarify++ + continue + } + byIntent[c.Intent]++ + } + for in := range valid { + if byIntent[in] < 5 { + t.Errorf("intent %q has %d cases, want >= 5", in, byIntent[in]) + } + } + if clarify < 5 { + t.Errorf("%d clarify cases, want >= 5", clarify) + } +} + +// TestFixtureIsHeldOut — the fixture's whole claim to measuring anything. The +// classifier routes by similarity to frozen seed phrases, so a fixture sharing +// utterances with models/seeds/*.txt would score its own training set. Verbatim +// match is the line drawn: paraphrases are the point of the fixture, copies are +// the failure. +func TestFixtureIsHeldOut(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + seeds := loadSeeds(t) + for _, c := range f.Cases { + if src, ok := seeds[normalize(c.Utterance)]; ok { + t.Errorf("%s: %q is verbatim in %s — not held out", c.ID, c.Utterance, src) + } + } +} + +// TestClassifierBaseline — the number Vikunja #319 compares against. This is +// the committed stopgap path (stage 0 grammar → nearest-centroid classifier +// over the real models/seeds corpus), scored on held-out utterances. +// +// It runs on HashEmbedder, not the ONNX multilingual embedder: the hash floor +// is deterministic, so this baseline is reproducible in CI. The production +// ONNX number will be higher; measure both against the same fixture before +// flipping the default (#320), and never compare a hash-embedder run to an +// ONNX one. +// +// The assertion is a ratchet, not a target — it only catches a regression +// below what the cascade already does. The gap between it and 100% is the +// backlog. +func TestClassifierBaseline(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + // dim 1024: the hash embedder is bag-of-words, so a narrower space would + // collide tokens across intents and measure the hash, not the centroids. + rep, err := Score(context.Background(), "classifier+hash", newBaselineRouter(t, router.NewHashEmbedder(1024)), f) + if err != nil { + t.Fatalf("Score: %v", err) + } + t.Log("\n" + rep.String() + rep.Failures()) + + // 0.10 is under the observed 0.118, not a target. Almost every case here + // falls to clarify because the hash embedder's cosine never clears the + // 0.55 gate on paraphrases — which is the documented floor behaviour + // (AGENTS.md: "Russian recall rarely clears the confidence gate"), not a + // bug this fixture is asking anyone to fix. The number worth moving is + // TestONNXBaseline's. + const floor = 0.10 + if rep.Accuracy() < floor { + t.Errorf("accuracy %.3f below ratchet %.2f — routing regressed", rep.Accuracy(), floor) + } + // The dangerous direction is asserted separately and tightly: a confident + // route for "сделай это" is a destructive guess, and unlike a miss the user + // never gets asked. Clarify cases must not silently start deciding. + if rep.MissedClarify > 2 { + t.Errorf("%d ambiguous utterances routed confidently, want <= 2:\n%s", rep.MissedClarify, rep.Failures()) + } +} + +// TestONNXBaseline — the number that actually belongs in Vikunja #319: the +// deployed cascade with the multilingual ONNX embedder, the configuration +// homesrv runs. Skipped unless the runtime is present, because the .so is a +// gitignored ~200MB download and CI has the hash ratchet above instead. +// +// MAVEN_ONNX_LIB=/usr/local/lib/libonnxruntime.so go test -run ONNXBaseline ./internal/router/eval/ +// +// Reports rather than asserts: the score is an input to the #320 flip decision, +// and a threshold invented here would just be a second opinion about the same +// unmeasured thing. Compare it to an LLM-router run over the SAME fixture — +// accuracy and p50/p95 latency both, since the resident model pays CPU seconds +// per turn that the classifier's ~10µs does not. +func TestONNXBaseline(t *testing.T) { + lib := os.Getenv("MAVEN_ONNX_LIB") + if lib == "" { + t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") + } + model := filepath.Join("../../..", "models/embedder/model.onnx") + tok := filepath.Join("../../..", "models/embedder/tokenizer.json") + for _, p := range []string{lib, model, tok} { + if _, err := os.Stat(p); err != nil { + t.Skipf("missing %s: %v", p, err) + } + } + emb, err := router.NewONNXEmbedder(model, tok, lib) + if err != nil { + t.Skipf("onnx embedder unavailable: %v", err) + } + defer emb.Close() + + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + rep, err := Score(context.Background(), "classifier+onnx", newBaselineRouter(t, emb), f) + if err != nil { + t.Fatalf("Score: %v", err) + } + t.Log("\n" + rep.String() + rep.Failures()) +} + +// newBaselineRouter mirrors buildRouter in cmd/mavend/voice.go — same grammar +// set, same extractor floors, same seed corpus, same confidence gate — so the +// score reflects the deployed cascade and not a test-local approximation. Only +// the embedder varies, and that is the axis being measured. +func newBaselineRouter(t *testing.T, emb router.Embedder) *router.Router { + t.Helper() + acts := router.DefaultActMatcher{Fns: actFns} + cls := router.NewClassifier(emb) + ctx := context.Background() + seeds := seedsWithIntent(t) + texts := make([]string, 0, len(seeds)) + for text := range seeds { + texts = append(texts, text) + } + // Sorted: centroids are order-independent, but tie-breaking in Classify's + // result sort is not, and a jittering baseline can't be a ratchet. + sort.Strings(texts) + for _, text := range texts { + if err := cls.AddExample(ctx, seeds[text], text); err != nil { + t.Fatalf("seed %q: %v", text, err) + } + } + grammars := router.DefaultGrammars(acts) + grammars = append(grammars, router.SystemTimeDateGrammars()...) + grammars = append(grammars, router.ReminderGrammar()) + return router.New(router.Config{ + Grammars: grammars, + Classifier: cls, + Extractor: router.Extractor{ + Time: router.StubDateTimeParser{}, + Acts: acts, + Facts: router.DefaultFactParser{}, + }, + // The deployed gate, not a test-local one: a fixture scored at a looser + // threshold reports an accuracy no real turn would see. + Threshold: config.DefaultRouterThreshold, + }) +} + +// seedOrder — fixed iteration order over the corpus. Not cosmetic: a few +// phrases appear under two intents ("как дела у сервера" is in both query.txt +// and system.txt), and ranging over a map would hand the duplicate to a +// different centroid on every run, which makes the baseline score jitter and +// the ratchet meaningless. +var seedOrder = []router.Intent{ + router.IntentAct, router.IntentReminder, router.IntentFact, + router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem, +} + +// loadSeeds maps normalized seed text → the file it came from. +func loadSeeds(t *testing.T) map[string]string { + t.Helper() + files := readSeedFiles(t) + out := map[string]string{} + for _, intent := range seedOrder { + for _, l := range files[intent] { + out[normalize(l)] = string(intent) + ".txt" + } + } + return out +} + +// seedsWithIntent flattens the corpus to text→intent. A phrase appearing under +// two intents collapses to one example — the classifier would otherwise train +// two centroids to fight over the same vector. First intent in seedOrder wins, +// deterministically. +func seedsWithIntent(t *testing.T) map[string]router.Intent { + t.Helper() + files := readSeedFiles(t) + out := map[string]router.Intent{} + for _, intent := range seedOrder { + for _, l := range files[intent] { + if _, dup := out[l]; dup { + continue + } + out[l] = intent + } + } + return out +} + +func readSeedFiles(t *testing.T) map[router.Intent][]string { + t.Helper() + out := map[router.Intent][]string{} + for _, in := range seedOrder { + path := filepath.Join(seedDir, string(in)+".txt") + fh, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + sc := bufio.NewScanner(fh) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out[in] = append(out[in], line) + } + err = sc.Err() + fh.Close() + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + } + return out +} + +// normalize — held-out comparison is on lowercased, whitespace-collapsed text +// so a copied seed can't sneak in behind capitalisation. +func normalize(s string) string { + return strings.Join(strings.Fields(strings.ToLower(s)), " ") +} diff --git a/internal/router/eval/ru_routing_v1.json b/internal/router/eval/ru_routing_v1.json new file mode 100644 index 0000000..3cfb6a0 --- /dev/null +++ b/internal/router/eval/ru_routing_v1.json @@ -0,0 +1,96 @@ +{ + "schema_version": 1, + "name": "ru_routing_v1", + "reference_now": "2026-07-30T12:00:00Z", + "notes": [ + "Held-out routing contract. Every utterance here is absent from models/seeds/*.txt (TestFixtureIsHeldOut enforces it verbatim) — scoring a classifier on its own seed phrases measures memorisation, not routing.", + "This is a CONTRACT, not a snapshot of current behaviour. Cases the classifier cascade fails today are expected to stay in the file and fail loudly; that failure count is the number Vikunja #319 compares against the LLM router before #320 flips the default.", + "Slot expectations are deployment-independent on purpose. want_fn is a boolean (the act must resolve to SOME allowlisted fn) because the allowlist lives in deploy config, not here. want_fact_key names the loop's rule keys (water/meal/sleep/break/shower) — a fact that lands under the wrong key silently starves the predicate that reads it.", + "want_clarify cases carry intent \"\": the contract is that the router refuses rather than guesses. A confident answer there is a worse failure than a miss." + ], + "cases": [ + { "id": "ru-query-001", "utterance": "сколько воды я выпил с утра", "lang": "ru", "intent": "query", "tags": ["aggregate"] }, + { "id": "ru-query-002", "utterance": "я сегодня вообще пил воду", "lang": "ru", "intent": "query", "tags": ["hard", "fact-shaped"], "note": "past-tense fact lexicon in a question — the classifier's fact centroid pulls this hard" }, + { "id": "ru-query-003", "utterance": "во сколько я лёг вчера", "lang": "ru", "intent": "query", "tags": ["temporal"] }, + { "id": "ru-query-004", "utterance": "давно я не тренировался", "lang": "ru", "intent": "query", "tags": ["hard", "no-question-word"] }, + { "id": "ru-query-005", "utterance": "напоминания на завтра есть", "lang": "ru", "intent": "query", "tags": ["hard", "reminder-shaped"], "note": "asks about reminders, does not create one" }, + { "id": "ru-query-006", "utterance": "что я записывал про кота", "lang": "ru", "intent": "query", "tags": ["recall"] }, + { "id": "ru-query-007", "utterance": "сколько раз я ел вчера", "lang": "ru", "intent": "query", "tags": ["aggregate", "hard"] }, + { "id": "ru-query-008", "utterance": "мой вес за последний месяц", "lang": "ru", "intent": "query", "tags": ["no-verb"] }, + { "id": "ru-query-009", "utterance": "когда я в последний раз принимал витамины", "lang": "ru", "intent": "query", "tags": ["temporal"] }, + { "id": "ru-query-010", "utterance": "есть новости по бэкапу базы", "lang": "ru", "intent": "query", "tags": ["homelab"] }, + { "id": "ru-query-011", "utterance": "почему сервер тормозит", "lang": "ru", "intent": "query", "tags": ["homelab", "hard"], "note": "diagnostic question, not a chat opener" }, + { "id": "ru-query-012", "utterance": "какие заметки я оставил про полив", "lang": "ru", "intent": "query", "tags": ["recall"] }, + { "id": "ru-query-013", "utterance": "во сколько у меня встреча", "lang": "ru", "intent": "query", "tags": ["calendar"] }, + { "id": "ru-query-014", "utterance": "я успеваю до дедлайна", "lang": "ru", "intent": "query", "tags": ["hard", "no-question-word"] }, + { "id": "ru-query-015", "utterance": "сколько я прошёл шагов", "lang": "ru", "intent": "query", "tags": ["aggregate"] }, + { "id": "ru-query-016", "utterance": "покажи давление за неделю", "lang": "ru", "intent": "query", "tags": ["hard", "imperative"], "note": "imperative form but a read — must not route to act" }, + { "id": "ru-query-017", "utterance": "чем я занимался в среду", "lang": "ru", "intent": "query", "tags": ["hard", "chat-shaped"] }, + { "id": "ru-query-018", "utterance": "хватает ли места под новые бэкапы", "lang": "ru", "intent": "query", "tags": ["homelab"] }, + { "id": "en-query-001", "utterance": "did I take my vitamins today", "lang": "en", "intent": "query", "tags": ["fact-shaped"] }, + { "id": "en-query-002", "utterance": "how long since the last backup finished", "lang": "en", "intent": "query", "tags": ["temporal"] }, + { "id": "en-query-003", "utterance": "show me this week's weight", "lang": "en", "intent": "query", "tags": ["imperative"] }, + + { "id": "ru-fact-001", "utterance": "только что выпил кружку воды", "lang": "ru", "intent": "fact", "want_fact_key": "water" }, + { "id": "ru-fact-002", "utterance": "воды попил наконец", "lang": "ru", "intent": "fact", "want_fact_key": "water", "tags": ["inverted"] }, + { "id": "ru-fact-003", "utterance": "поужинал", "lang": "ru", "intent": "fact", "want_fact_key": "meal", "tags": ["single-word"] }, + { "id": "ru-fact-004", "utterance": "отметь что я позавтракал овсянкой", "lang": "ru", "intent": "fact", "want_fact_key": "meal" }, + { "id": "ru-fact-005", "utterance": "поспал часов пять", "lang": "ru", "intent": "fact", "want_fact_key": "sleep" }, + { "id": "ru-fact-006", "utterance": "сходил в душ", "lang": "ru", "intent": "fact", "want_fact_key": "shower" }, + { "id": "ru-fact-007", "utterance": "отдохнул минут двадцать", "lang": "ru", "intent": "fact", "want_fact_key": "break" }, + { "id": "ru-fact-008", "utterance": "запиши вес 74 килограмма", "lang": "ru", "intent": "fact", "tags": ["unparsed-key"], "note": "no recognizer for weight yet — intent must still be fact; HasKey false is allowed" }, + { "id": "ru-fact-009", "utterance": "отметь что я выпил таблетки утром", "lang": "ru", "intent": "fact", "tags": ["unparsed-key"] }, + { "id": "ru-fact-010", "utterance": "сделал зарядку двадцать минут", "lang": "ru", "intent": "fact", "tags": ["unparsed-key"] }, + { "id": "en-fact-001", "utterance": "just drank a glass of water", "lang": "en", "intent": "fact", "want_fact_key": "water" }, + { "id": "en-fact-002", "utterance": "slept about seven hours", "lang": "en", "intent": "fact", "want_fact_key": "sleep" }, + + { "id": "ru-rem-001", "utterance": "напомни через сорок минут выключить свет", "lang": "ru", "intent": "reminder", "want_time": true }, + { "id": "ru-rem-002", "utterance": "напомни завтра в 7 позвонить в клинику", "lang": "ru", "intent": "reminder", "want_time": true }, + { "id": "ru-rem-003", "utterance": "не забудь напомнить мне про счёт за свет", "lang": "ru", "intent": "reminder", "tags": ["no-time"], "note": "no datetime — HasTime false is correct; the daemon asks for a time" }, + { "id": "ru-rem-004", "utterance": "поставь напоминание через полчаса", "lang": "ru", "intent": "reminder", "want_time": true }, + { "id": "ru-rem-005", "utterance": "разбуди меня в 6:30", "lang": "ru", "intent": "reminder", "want_time": true, "tags": ["hard"], "note": "wake-me phrasing, no напомни stem" }, + { "id": "ru-rem-006", "utterance": "напомни послезавтра в 12 забрать заказ", "lang": "ru", "intent": "reminder", "want_time": true }, + { "id": "ru-rem-007", "utterance": "через два часа напомни проверить бэкап", "lang": "ru", "intent": "reminder", "want_time": true, "tags": ["inverted"] }, + { "id": "en-rem-001", "utterance": "remind me in 45 minutes to stretch", "lang": "en", "intent": "reminder", "want_time": true }, + { "id": "en-rem-002", "utterance": "wake me at 6:15", "lang": "en", "intent": "reminder", "want_time": true, "tags": ["hard"] }, + + { "id": "ru-act-001", "utterance": "перезапусти докер", "lang": "ru", "intent": "act", "want_fn": true, "tags": ["homelab", "destructive"] }, + { "id": "ru-act-002", "utterance": "выключи свет в спальне", "lang": "ru", "intent": "act", "want_fn": true }, + { "id": "ru-act-003", "utterance": "maven, останови бэкап", "lang": "ru", "intent": "act", "want_fn": true, "tags": ["wake-token"] }, + { "id": "ru-act-004", "utterance": "включи вытяжку", "lang": "ru", "intent": "act", "want_fn": true }, + { "id": "ru-act-005", "utterance": "запусти бэкап сейчас", "lang": "ru", "intent": "act", "want_fn": true }, + { "id": "ru-act-006", "utterance": "закрой жалюзи", "lang": "ru", "intent": "act", "want_fn": true }, + { "id": "en-act-001", "utterance": "maven, restart the media server", "lang": "en", "intent": "act", "want_fn": true, "tags": ["wake-token"] }, + { "id": "en-act-002", "utterance": "turn off the kitchen light", "lang": "en", "intent": "act", "want_fn": true }, + + { "id": "ru-note-001", "utterance": "заметка: продлить домен в августе", "lang": "ru", "intent": "note" }, + { "id": "ru-note-002", "utterance": "запиши что кран на кухне снова капает", "lang": "ru", "intent": "note" }, + { "id": "ru-note-003", "utterance": "заметка про настройку vlan на свитче", "lang": "ru", "intent": "note", "tags": ["homelab"] }, + { "id": "ru-note-004", "utterance": "запиши идею: гидропоника на балконе", "lang": "ru", "intent": "note" }, + { "id": "ru-note-005", "utterance": "запиши что сосед просил номер электрика", "lang": "ru", "intent": "note" }, + { "id": "en-note-001", "utterance": "note: rotate the kuma api key", "lang": "en", "intent": "note", "tags": ["homelab"] }, + + { "id": "ru-sys-001", "utterance": "сколько сейчас времени в киеве", "lang": "ru", "intent": "system", "tags": ["time"] }, + { "id": "ru-sys-002", "utterance": "какое число завтра", "lang": "ru", "intent": "system", "tags": ["date"] }, + { "id": "ru-sys-003", "utterance": "переходи в тихий режим", "lang": "ru", "intent": "system", "tags": ["quiet"] }, + { "id": "ru-sys-004", "utterance": "хватит тихого режима", "lang": "ru", "intent": "system", "tags": ["quiet", "hard"] }, + { "id": "ru-sys-005", "utterance": "какой день недели послезавтра", "lang": "ru", "intent": "system", "tags": ["date"] }, + { "id": "en-sys-001", "utterance": "what time is it now", "lang": "en", "intent": "system", "tags": ["time"] }, + { "id": "en-sys-002", "utterance": "turn quiet mode back on", "lang": "en", "intent": "system", "tags": ["quiet"] }, + + { "id": "ru-chat-001", "utterance": "мне немного грустно сегодня", "lang": "ru", "intent": "chat" }, + { "id": "ru-chat-002", "utterance": "что думаешь про переезд на другую квартиру", "lang": "ru", "intent": "chat" }, + { "id": "ru-chat-003", "utterance": "расскажи анекдот про программистов", "lang": "ru", "intent": "chat" }, + { "id": "ru-chat-004", "utterance": "доброе утро", "lang": "ru", "intent": "chat", "tags": ["greeting"] }, + { "id": "ru-chat-005", "utterance": "спасибо тебе", "lang": "ru", "intent": "chat" }, + { "id": "en-chat-001", "utterance": "good morning", "lang": "en", "intent": "chat", "tags": ["greeting"] }, + { "id": "en-chat-002", "utterance": "i had a rough day", "lang": "en", "intent": "chat" }, + + { "id": "amb-001", "utterance": "вода", "lang": "ru", "want_clarify": true, "tags": ["ambiguous"], "note": "fact write or a query? one noun decides nothing — ask" }, + { "id": "amb-002", "utterance": "бэкап", "lang": "ru", "want_clarify": true, "tags": ["ambiguous"], "note": "run it, or report on it? a confident act here is destructive" }, + { "id": "amb-003", "utterance": "ну это", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "filler"] }, + { "id": "amb-004", "utterance": "сделай это", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "anaphora"], "note": "unresolved anaphora with an imperative — must not guess an fn" }, + { "id": "amb-005", "utterance": "потом", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "filler"] }, + { "id": "amb-006", "utterance": "the thing from earlier", "lang": "en", "want_clarify": true, "tags": ["ambiguous", "anaphora"] } + ] +}