Add held-out RU routing fixture and scorer (Vikunja #319)
#319 asks for a measurement before #320 flips the route decider from the classifier cascade to the resident model. There was nothing to measure against: the only routing tests assert single utterances, and the classifier's seed corpus is its own training set — scoring it there measures memorisation of frozen centroids, which is the illusion that hid the weak RU query handling in the first place. internal/router/eval is a separate package so both paths can be scored from outside router (including cmd/mavend, where the real llama-server client lives). The fixture is embedded; the scorer takes a Router interface, so *router.Router and a bare LLM stage both go through the same 76 cases. The fixture is a CONTRACT, not a snapshot: cases the cascade fails today stay in the file and fail loudly. TestFixtureIsHeldOut enforces that no utterance appears verbatim in models/seeds/*.txt. Baseline, hash embedder at the deployed 0.55 gate: 9/76 (11.8%), 63 false clarifies, 0 missed clarifies, p50 9µs. Almost everything falls to the confidence gate — the documented floor behaviour, not a new bug. The number worth comparing is TestONNXBaseline's (skipped without MAVEN_ONNX_LIB); the assertions here are a regression ratchet plus a tight bound on the dangerous direction: ambiguous utterances must not start being routed confidently. Seeding is order-fixed on purpose — a few phrases appear under two intents and map iteration handed them to a different centroid each run, which made the score jitter between 9 and 10. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01X5JApcrCRVGmqrxnhynSik
This commit is contained in:
kami
@@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper
|
||||
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
|
||||
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
|
||||
|
||||
.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go
|
||||
.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go eval-router
|
||||
|
||||
all: build
|
||||
|
||||
@@ -73,6 +73,13 @@ test:
|
||||
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
||||
$(GO) test -race -coverprofile=coverage.out ./internal/... ./cmd/...
|
||||
|
||||
# eval-router — score the held-out RU routing fixture (internal/router/eval).
|
||||
# Verbose so the report table lands in the terminal; MAVEN_ONNX_LIB additionally
|
||||
# runs the prod-representative ONNX baseline (skipped without it). This is the
|
||||
# measurement Vikunja #319 compares before #320 flips the route decider.
|
||||
eval-router:
|
||||
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
|
||||
|
||||
run-stt: build-stt
|
||||
LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
||||
./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL)
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
// Package eval is the held-out routing contract — the fixture Vikunja #319
|
||||
// measures against before #320 flips the default route decider.
|
||||
//
|
||||
// Why it is a separate package from router: the fixture must be scorable by
|
||||
// BOTH paths (today's classifier cascade and the resident model's LLM router)
|
||||
// from outside the router package, including from cmd/mavend where the real
|
||||
// llama-server client lives. A _test.go file in router can't be imported, and
|
||||
// testdata isn't reachable from another package's working directory — so the
|
||||
// fixture is embedded here and the scorer takes a Router interface.
|
||||
//
|
||||
// The fixture is HELD OUT from models/seeds/*.txt on purpose: a classifier
|
||||
// scored on its own seed phrases measures memorisation of frozen centroids,
|
||||
// which is exactly the illusion that hid the weak RU query handling. See
|
||||
// TestFixtureIsHeldOut, which enforces it.
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
//go:embed ru_routing_v1.json
|
||||
var fixtureJSON []byte
|
||||
|
||||
// Case — one utterance and the route it must produce. Slot expectations are
|
||||
// deliberately coarse (see the fixture's notes): want_fn is a boolean because
|
||||
// the fn allowlist lives in deploy config, and want_fact_key names the loop's
|
||||
// rule keys because a fact under the wrong key starves its predicate.
|
||||
//
|
||||
// Intent is empty exactly when WantClarify is set: the contract there is that
|
||||
// the router refuses instead of guessing.
|
||||
type Case struct {
|
||||
ID string `json:"id"`
|
||||
Utterance string `json:"utterance"`
|
||||
Lang string `json:"lang"`
|
||||
Intent router.Intent `json:"intent"`
|
||||
WantTime bool `json:"want_time"`
|
||||
WantFn bool `json:"want_fn"`
|
||||
WantFactKey string `json:"want_fact_key"`
|
||||
WantClarify bool `json:"want_clarify"`
|
||||
Tags []string `json:"tags"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// Fixture — the versioned envelope, same shape as
|
||||
// cmd/mavend/testdata/system_safety_scenarios.json. SchemaVersion gates the
|
||||
// loader so an older binary refuses a fixture it would misread rather than
|
||||
// scoring it wrong and reporting a number.
|
||||
type Fixture struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Name string `json:"name"`
|
||||
ReferenceNow string `json:"reference_now"`
|
||||
Notes []string `json:"notes"`
|
||||
Cases []Case `json:"cases"`
|
||||
}
|
||||
|
||||
// SchemaVersion — the version this package understands.
|
||||
const SchemaVersion = 1
|
||||
|
||||
// Load returns the embedded fixture.
|
||||
func Load() (Fixture, error) {
|
||||
var f Fixture
|
||||
if err := json.Unmarshal(fixtureJSON, &f); err != nil {
|
||||
return Fixture{}, fmt.Errorf("parse fixture: %w", err)
|
||||
}
|
||||
if f.SchemaVersion != SchemaVersion {
|
||||
return Fixture{}, fmt.Errorf("fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion)
|
||||
}
|
||||
if len(f.Cases) == 0 {
|
||||
return Fixture{}, fmt.Errorf("fixture has no cases")
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Now — the fixture's reference clock. Relative reminder slots ("через
|
||||
// полчаса") resolve against it, so a scoring run is reproducible regardless of
|
||||
// when it runs.
|
||||
func (f Fixture) Now() (time.Time, error) {
|
||||
t, err := time.Parse(time.RFC3339, f.ReferenceNow)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("parse reference_now %q: %w", f.ReferenceNow, err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Router — the one thing a route decider must do to be scorable. *router.Router
|
||||
// satisfies it directly; an LLM-only path wraps its Route in RouterFunc.
|
||||
type Router interface {
|
||||
Route(ctx context.Context, utterance string, now time.Time) (router.Decision, error)
|
||||
}
|
||||
|
||||
// RouterFunc adapts a bare function to Router — for scoring a single stage
|
||||
// (e.g. *router.LLMRouter, whose Route returns an extra ok bool) without
|
||||
// standing up the whole cascade.
|
||||
type RouterFunc func(ctx context.Context, utterance string, now time.Time) (router.Decision, error)
|
||||
|
||||
// Route implements Router.
|
||||
func (f RouterFunc) Route(ctx context.Context, utterance string, now time.Time) (router.Decision, error) {
|
||||
return f(ctx, utterance, now)
|
||||
}
|
||||
|
||||
// Outcome — one scored case. Reasons is empty exactly when Pass is true.
|
||||
type Outcome struct {
|
||||
Case Case
|
||||
Decision router.Decision
|
||||
Err error
|
||||
Latency time.Duration
|
||||
Pass bool
|
||||
// IntentOK is tracked separately from Pass: a case can land the right
|
||||
// intent and still fail on a slot, and #319 needs those two numbers apart
|
||||
// (a slot gap is a parser fix; a wrong intent is a router fix).
|
||||
IntentOK bool
|
||||
Reasons []string
|
||||
}
|
||||
|
||||
// Report — the aggregate. Accuracy is the headline; the rest exists so a
|
||||
// regression names itself instead of just moving a percentage.
|
||||
type Report struct {
|
||||
Name string
|
||||
Total int
|
||||
Passed int
|
||||
IntentHit int
|
||||
// FalseClarify — the router asked when the fixture expected a decision.
|
||||
// A gap, recoverable by asking again.
|
||||
FalseClarify int
|
||||
// MissedClarify — the router decided confidently where the fixture
|
||||
// expected a refusal. The dangerous direction: "сделай это" routed to an
|
||||
// act is a confident destructive guess.
|
||||
MissedClarify int
|
||||
Errors int
|
||||
Outcomes []Outcome
|
||||
// Confusion counts want→got intent pairs, decided cases only.
|
||||
Confusion map[string]int
|
||||
// ByTag accuracy for the fixture's tags ("hard", "homelab", …).
|
||||
ByTag map[string]TagStat
|
||||
// ByLang accuracy — the RU/EN split is the whole reason this fixture
|
||||
// exists.
|
||||
ByLang map[string]TagStat
|
||||
P50 time.Duration
|
||||
P95 time.Duration
|
||||
Max time.Duration
|
||||
}
|
||||
|
||||
// TagStat — passed/total for one slice of the fixture.
|
||||
type TagStat struct{ Passed, Total int }
|
||||
|
||||
// Accuracy — fraction of cases fully satisfied (intent AND slots AND the
|
||||
// clarify contract).
|
||||
func (r Report) Accuracy() float64 {
|
||||
if r.Total == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(r.Passed) / float64(r.Total)
|
||||
}
|
||||
|
||||
// IntentAccuracy — fraction with the right intent, ignoring slot fills.
|
||||
func (r Report) IntentAccuracy() float64 {
|
||||
if r.Total == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(r.IntentHit) / float64(r.Total)
|
||||
}
|
||||
|
||||
// Score runs every case through r and aggregates. It never fails the run on a
|
||||
// route error — an erroring case scores as a miss and is counted in Errors,
|
||||
// because "the model was down" and "the model was wrong" are different numbers
|
||||
// and #319 needs to tell them apart.
|
||||
//
|
||||
// Latency is wall-clock per Route call, including any llama-server round trip.
|
||||
// That is the point on the CPU-only target: a correctness win the resident
|
||||
// model pays for with seconds per turn is not a win.
|
||||
func Score(ctx context.Context, name string, r Router, f Fixture) (Report, error) {
|
||||
now, err := f.Now()
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
rep := Report{
|
||||
Name: name,
|
||||
Total: len(f.Cases),
|
||||
Confusion: map[string]int{},
|
||||
ByTag: map[string]TagStat{},
|
||||
ByLang: map[string]TagStat{},
|
||||
}
|
||||
lat := make([]time.Duration, 0, len(f.Cases))
|
||||
|
||||
for _, c := range f.Cases {
|
||||
start := time.Now()
|
||||
d, err := r.Route(ctx, c.Utterance, now)
|
||||
o := Outcome{Case: c, Decision: d, Err: err, Latency: time.Since(start)}
|
||||
lat = append(lat, o.Latency)
|
||||
|
||||
switch {
|
||||
case err != nil:
|
||||
rep.Errors++
|
||||
o.Reasons = append(o.Reasons, fmt.Sprintf("route error: %v", err))
|
||||
case c.WantClarify:
|
||||
// Only the refusal matters here; whatever intent the cascade
|
||||
// guessed underneath is irrelevant if it gated.
|
||||
o.IntentOK = d.Clarify
|
||||
if !d.Clarify {
|
||||
rep.MissedClarify++
|
||||
o.Reasons = append(o.Reasons, fmt.Sprintf("decided %q confidently (%.3f), want clarify", d.Intent, d.Confidence))
|
||||
}
|
||||
default:
|
||||
o.IntentOK = d.Intent == c.Intent && !d.Clarify
|
||||
if d.Clarify {
|
||||
rep.FalseClarify++
|
||||
o.Reasons = append(o.Reasons, fmt.Sprintf("clarified (%.3f), want intent %q", d.Confidence, c.Intent))
|
||||
} else if d.Intent != c.Intent {
|
||||
rep.Confusion[string(c.Intent)+"→"+string(d.Intent)]++
|
||||
o.Reasons = append(o.Reasons, fmt.Sprintf("intent %q, want %q (%.3f)", d.Intent, c.Intent, d.Confidence))
|
||||
}
|
||||
if c.WantTime && !d.Slots.HasTime {
|
||||
o.Reasons = append(o.Reasons, "no time slot, want one")
|
||||
}
|
||||
if c.WantFn && !d.Slots.HasFn {
|
||||
o.Reasons = append(o.Reasons, "no fn slot, want an allowlist match")
|
||||
}
|
||||
if c.WantFactKey != "" && d.Slots.Key != c.WantFactKey {
|
||||
o.Reasons = append(o.Reasons, fmt.Sprintf("fact key %q, want %q", d.Slots.Key, c.WantFactKey))
|
||||
}
|
||||
}
|
||||
|
||||
o.Pass = len(o.Reasons) == 0
|
||||
if o.Pass {
|
||||
rep.Passed++
|
||||
}
|
||||
if o.IntentOK {
|
||||
rep.IntentHit++
|
||||
}
|
||||
bump(rep.ByLang, c.Lang, o.Pass)
|
||||
for _, tag := range c.Tags {
|
||||
bump(rep.ByTag, tag, o.Pass)
|
||||
}
|
||||
rep.Outcomes = append(rep.Outcomes, o)
|
||||
}
|
||||
|
||||
sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] })
|
||||
rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95)
|
||||
if len(lat) > 0 {
|
||||
rep.Max = lat[len(lat)-1]
|
||||
}
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
func bump(m map[string]TagStat, key string, pass bool) {
|
||||
if key == "" {
|
||||
return
|
||||
}
|
||||
s := m[key]
|
||||
s.Total++
|
||||
if pass {
|
||||
s.Passed++
|
||||
}
|
||||
m[key] = s
|
||||
}
|
||||
|
||||
// percentile — nearest-rank on a pre-sorted slice. No interpolation: with ~80
|
||||
// samples an interpolated p95 invents a latency no turn actually took.
|
||||
func percentile(sorted []time.Duration, p float64) time.Duration {
|
||||
if len(sorted) == 0 {
|
||||
return 0
|
||||
}
|
||||
i := int(p * float64(len(sorted)))
|
||||
if i >= len(sorted) {
|
||||
i = len(sorted) - 1
|
||||
}
|
||||
return sorted[i]
|
||||
}
|
||||
|
||||
// String renders the report as the comparison table #319 pastes into the task —
|
||||
// headline accuracy, the two clarify directions apart, latency, and the slices
|
||||
// that name where a path is weak.
|
||||
func (r Report) String() string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%s: %d/%d cases (%.1f%% full, %.1f%% intent-only)\n",
|
||||
r.Name, r.Passed, r.Total, 100*r.Accuracy(), 100*r.IntentAccuracy())
|
||||
fmt.Fprintf(&b, " clarify: %d false (asked, shouldn't) / %d missed (guessed, shouldn't) | errors: %d\n",
|
||||
r.FalseClarify, r.MissedClarify, r.Errors)
|
||||
fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max)
|
||||
fmt.Fprintf(&b, " by lang: %s\n", renderStats(r.ByLang))
|
||||
fmt.Fprintf(&b, " by tag: %s\n", renderStats(r.ByTag))
|
||||
if len(r.Confusion) > 0 {
|
||||
fmt.Fprintf(&b, " confusion: %s\n", renderCounts(r.Confusion))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Failures — the per-case detail, sorted by ID so two runs diff cleanly.
|
||||
func (r Report) Failures() string {
|
||||
var b strings.Builder
|
||||
out := append([]Outcome(nil), r.Outcomes...)
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID })
|
||||
for _, o := range out {
|
||||
if o.Pass {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, " %s %q: %s\n", o.Case.ID, o.Case.Utterance, strings.Join(o.Reasons, "; "))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func renderStats(m map[string]TagStat) string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
s := m[k]
|
||||
parts = append(parts, fmt.Sprintf("%s %d/%d", k, s.Passed, s.Total))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func renderCounts(m map[string]int) string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if m[keys[i]] != m[keys[j]] {
|
||||
return m[keys[i]] > m[keys[j]]
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s ×%d", k, m[k]))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
const seedDir = "../../../models/seeds"
|
||||
|
||||
// actFns — the verbs the baseline act matcher accepts. In the daemon the
|
||||
// allowlist is exactly the enabled tool names (see buildRouter in
|
||||
// cmd/mavend/voice.go); here it stands in for a deployment's tools so the
|
||||
// fixture's want_fn cases are satisfiable at all. DefaultActMatcher is
|
||||
// verb-prefix only, so these are verbs, not full commands.
|
||||
var actFns = []string{
|
||||
"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
|
||||
"закрой", "открой", "сделай", "поставь",
|
||||
"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
|
||||
}
|
||||
|
||||
func TestLoadFixture(t *testing.T) {
|
||||
f, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if _, err := f.Now(); err != nil {
|
||||
t.Fatalf("Now: %v", err)
|
||||
}
|
||||
valid := map[router.Intent]bool{
|
||||
router.IntentAct: true, router.IntentReminder: true, router.IntentFact: true,
|
||||
router.IntentNote: true, router.IntentQuery: true, router.IntentChat: true,
|
||||
router.IntentSystem: true,
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, c := range f.Cases {
|
||||
if c.ID == "" || seen[c.ID] {
|
||||
t.Errorf("case %q: empty or duplicate id", c.ID)
|
||||
}
|
||||
seen[c.ID] = true
|
||||
if strings.TrimSpace(c.Utterance) == "" {
|
||||
t.Errorf("%s: empty utterance", c.ID)
|
||||
}
|
||||
if c.Lang != "ru" && c.Lang != "en" {
|
||||
t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang)
|
||||
}
|
||||
// Intent is empty exactly when the case expects a refusal — otherwise
|
||||
// a typo'd intent would score as an unreachable target forever.
|
||||
if c.WantClarify {
|
||||
if c.Intent != "" {
|
||||
t.Errorf("%s: want_clarify with intent %q — pick one", c.ID, c.Intent)
|
||||
}
|
||||
if c.WantTime || c.WantFn || c.WantFactKey != "" {
|
||||
t.Errorf("%s: want_clarify with slot expectations", c.ID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !valid[c.Intent] {
|
||||
t.Errorf("%s: intent %q not in the seven", c.ID, c.Intent)
|
||||
}
|
||||
// Slot expectations must match the intent that owns that slot, or the
|
||||
// case asserts something Extract never fills.
|
||||
if c.WantTime && c.Intent != router.IntentReminder {
|
||||
t.Errorf("%s: want_time on intent %q", c.ID, c.Intent)
|
||||
}
|
||||
if c.WantFn && c.Intent != router.IntentAct {
|
||||
t.Errorf("%s: want_fn on intent %q", c.ID, c.Intent)
|
||||
}
|
||||
if c.WantFactKey != "" && c.Intent != router.IntentFact {
|
||||
t.Errorf("%s: want_fact_key on intent %q", c.ID, c.Intent)
|
||||
}
|
||||
}
|
||||
// Coverage floor: every intent plus the refusal lane must be represented,
|
||||
// or a path can regress to zero without the number moving.
|
||||
byIntent := map[router.Intent]int{}
|
||||
clarify := 0
|
||||
for _, c := range f.Cases {
|
||||
if c.WantClarify {
|
||||
clarify++
|
||||
continue
|
||||
}
|
||||
byIntent[c.Intent]++
|
||||
}
|
||||
for in := range valid {
|
||||
if byIntent[in] < 5 {
|
||||
t.Errorf("intent %q has %d cases, want >= 5", in, byIntent[in])
|
||||
}
|
||||
}
|
||||
if clarify < 5 {
|
||||
t.Errorf("%d clarify cases, want >= 5", clarify)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFixtureIsHeldOut — the fixture's whole claim to measuring anything. The
|
||||
// classifier routes by similarity to frozen seed phrases, so a fixture sharing
|
||||
// utterances with models/seeds/*.txt would score its own training set. Verbatim
|
||||
// match is the line drawn: paraphrases are the point of the fixture, copies are
|
||||
// the failure.
|
||||
func TestFixtureIsHeldOut(t *testing.T) {
|
||||
f, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
seeds := loadSeeds(t)
|
||||
for _, c := range f.Cases {
|
||||
if src, ok := seeds[normalize(c.Utterance)]; ok {
|
||||
t.Errorf("%s: %q is verbatim in %s — not held out", c.ID, c.Utterance, src)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassifierBaseline — the number Vikunja #319 compares against. This is
|
||||
// the committed stopgap path (stage 0 grammar → nearest-centroid classifier
|
||||
// over the real models/seeds corpus), scored on held-out utterances.
|
||||
//
|
||||
// It runs on HashEmbedder, not the ONNX multilingual embedder: the hash floor
|
||||
// is deterministic, so this baseline is reproducible in CI. The production
|
||||
// ONNX number will be higher; measure both against the same fixture before
|
||||
// flipping the default (#320), and never compare a hash-embedder run to an
|
||||
// ONNX one.
|
||||
//
|
||||
// The assertion is a ratchet, not a target — it only catches a regression
|
||||
// below what the cascade already does. The gap between it and 100% is the
|
||||
// backlog.
|
||||
func TestClassifierBaseline(t *testing.T) {
|
||||
f, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
// dim 1024: the hash embedder is bag-of-words, so a narrower space would
|
||||
// collide tokens across intents and measure the hash, not the centroids.
|
||||
rep, err := Score(context.Background(), "classifier+hash", newBaselineRouter(t, router.NewHashEmbedder(1024)), f)
|
||||
if err != nil {
|
||||
t.Fatalf("Score: %v", err)
|
||||
}
|
||||
t.Log("\n" + rep.String() + rep.Failures())
|
||||
|
||||
// 0.10 is under the observed 0.118, not a target. Almost every case here
|
||||
// falls to clarify because the hash embedder's cosine never clears the
|
||||
// 0.55 gate on paraphrases — which is the documented floor behaviour
|
||||
// (AGENTS.md: "Russian recall rarely clears the confidence gate"), not a
|
||||
// bug this fixture is asking anyone to fix. The number worth moving is
|
||||
// TestONNXBaseline's.
|
||||
const floor = 0.10
|
||||
if rep.Accuracy() < floor {
|
||||
t.Errorf("accuracy %.3f below ratchet %.2f — routing regressed", rep.Accuracy(), floor)
|
||||
}
|
||||
// The dangerous direction is asserted separately and tightly: a confident
|
||||
// route for "сделай это" is a destructive guess, and unlike a miss the user
|
||||
// never gets asked. Clarify cases must not silently start deciding.
|
||||
if rep.MissedClarify > 2 {
|
||||
t.Errorf("%d ambiguous utterances routed confidently, want <= 2:\n%s", rep.MissedClarify, rep.Failures())
|
||||
}
|
||||
}
|
||||
|
||||
// TestONNXBaseline — the number that actually belongs in Vikunja #319: the
|
||||
// deployed cascade with the multilingual ONNX embedder, the configuration
|
||||
// homesrv runs. Skipped unless the runtime is present, because the .so is a
|
||||
// gitignored ~200MB download and CI has the hash ratchet above instead.
|
||||
//
|
||||
// MAVEN_ONNX_LIB=/usr/local/lib/libonnxruntime.so go test -run ONNXBaseline ./internal/router/eval/
|
||||
//
|
||||
// Reports rather than asserts: the score is an input to the #320 flip decision,
|
||||
// and a threshold invented here would just be a second opinion about the same
|
||||
// unmeasured thing. Compare it to an LLM-router run over the SAME fixture —
|
||||
// accuracy and p50/p95 latency both, since the resident model pays CPU seconds
|
||||
// per turn that the classifier's ~10µs does not.
|
||||
func TestONNXBaseline(t *testing.T) {
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||||
}
|
||||
model := filepath.Join("../../..", "models/embedder/model.onnx")
|
||||
tok := filepath.Join("../../..", "models/embedder/tokenizer.json")
|
||||
for _, p := range []string{lib, model, tok} {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
t.Skipf("missing %s: %v", p, err)
|
||||
}
|
||||
}
|
||||
emb, err := router.NewONNXEmbedder(model, tok, lib)
|
||||
if err != nil {
|
||||
t.Skipf("onnx embedder unavailable: %v", err)
|
||||
}
|
||||
defer emb.Close()
|
||||
|
||||
f, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
rep, err := Score(context.Background(), "classifier+onnx", newBaselineRouter(t, emb), f)
|
||||
if err != nil {
|
||||
t.Fatalf("Score: %v", err)
|
||||
}
|
||||
t.Log("\n" + rep.String() + rep.Failures())
|
||||
}
|
||||
|
||||
// newBaselineRouter mirrors buildRouter in cmd/mavend/voice.go — same grammar
|
||||
// set, same extractor floors, same seed corpus, same confidence gate — so the
|
||||
// score reflects the deployed cascade and not a test-local approximation. Only
|
||||
// the embedder varies, and that is the axis being measured.
|
||||
func newBaselineRouter(t *testing.T, emb router.Embedder) *router.Router {
|
||||
t.Helper()
|
||||
acts := router.DefaultActMatcher{Fns: actFns}
|
||||
cls := router.NewClassifier(emb)
|
||||
ctx := context.Background()
|
||||
seeds := seedsWithIntent(t)
|
||||
texts := make([]string, 0, len(seeds))
|
||||
for text := range seeds {
|
||||
texts = append(texts, text)
|
||||
}
|
||||
// Sorted: centroids are order-independent, but tie-breaking in Classify's
|
||||
// result sort is not, and a jittering baseline can't be a ratchet.
|
||||
sort.Strings(texts)
|
||||
for _, text := range texts {
|
||||
if err := cls.AddExample(ctx, seeds[text], text); err != nil {
|
||||
t.Fatalf("seed %q: %v", text, err)
|
||||
}
|
||||
}
|
||||
grammars := router.DefaultGrammars(acts)
|
||||
grammars = append(grammars, router.SystemTimeDateGrammars()...)
|
||||
grammars = append(grammars, router.ReminderGrammar())
|
||||
return router.New(router.Config{
|
||||
Grammars: grammars,
|
||||
Classifier: cls,
|
||||
Extractor: router.Extractor{
|
||||
Time: router.StubDateTimeParser{},
|
||||
Acts: acts,
|
||||
Facts: router.DefaultFactParser{},
|
||||
},
|
||||
// The deployed gate, not a test-local one: a fixture scored at a looser
|
||||
// threshold reports an accuracy no real turn would see.
|
||||
Threshold: config.DefaultRouterThreshold,
|
||||
})
|
||||
}
|
||||
|
||||
// seedOrder — fixed iteration order over the corpus. Not cosmetic: a few
|
||||
// phrases appear under two intents ("как дела у сервера" is in both query.txt
|
||||
// and system.txt), and ranging over a map would hand the duplicate to a
|
||||
// different centroid on every run, which makes the baseline score jitter and
|
||||
// the ratchet meaningless.
|
||||
var seedOrder = []router.Intent{
|
||||
router.IntentAct, router.IntentReminder, router.IntentFact,
|
||||
router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
|
||||
}
|
||||
|
||||
// loadSeeds maps normalized seed text → the file it came from.
|
||||
func loadSeeds(t *testing.T) map[string]string {
|
||||
t.Helper()
|
||||
files := readSeedFiles(t)
|
||||
out := map[string]string{}
|
||||
for _, intent := range seedOrder {
|
||||
for _, l := range files[intent] {
|
||||
out[normalize(l)] = string(intent) + ".txt"
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seedsWithIntent flattens the corpus to text→intent. A phrase appearing under
|
||||
// two intents collapses to one example — the classifier would otherwise train
|
||||
// two centroids to fight over the same vector. First intent in seedOrder wins,
|
||||
// deterministically.
|
||||
func seedsWithIntent(t *testing.T) map[string]router.Intent {
|
||||
t.Helper()
|
||||
files := readSeedFiles(t)
|
||||
out := map[string]router.Intent{}
|
||||
for _, intent := range seedOrder {
|
||||
for _, l := range files[intent] {
|
||||
if _, dup := out[l]; dup {
|
||||
continue
|
||||
}
|
||||
out[l] = intent
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func readSeedFiles(t *testing.T) map[router.Intent][]string {
|
||||
t.Helper()
|
||||
out := map[router.Intent][]string{}
|
||||
for _, in := range seedOrder {
|
||||
path := filepath.Join(seedDir, string(in)+".txt")
|
||||
fh, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", path, err)
|
||||
}
|
||||
sc := bufio.NewScanner(fh)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[in] = append(out[in], line)
|
||||
}
|
||||
err = sc.Err()
|
||||
fh.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// normalize — held-out comparison is on lowercased, whitespace-collapsed text
|
||||
// so a copied seed can't sneak in behind capitalisation.
|
||||
func normalize(s string) string {
|
||||
return strings.Join(strings.Fields(strings.ToLower(s)), " ")
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"name": "ru_routing_v1",
|
||||
"reference_now": "2026-07-30T12:00:00Z",
|
||||
"notes": [
|
||||
"Held-out routing contract. Every utterance here is absent from models/seeds/*.txt (TestFixtureIsHeldOut enforces it verbatim) — scoring a classifier on its own seed phrases measures memorisation, not routing.",
|
||||
"This is a CONTRACT, not a snapshot of current behaviour. Cases the classifier cascade fails today are expected to stay in the file and fail loudly; that failure count is the number Vikunja #319 compares against the LLM router before #320 flips the default.",
|
||||
"Slot expectations are deployment-independent on purpose. want_fn is a boolean (the act must resolve to SOME allowlisted fn) because the allowlist lives in deploy config, not here. want_fact_key names the loop's rule keys (water/meal/sleep/break/shower) — a fact that lands under the wrong key silently starves the predicate that reads it.",
|
||||
"want_clarify cases carry intent \"\": the contract is that the router refuses rather than guesses. A confident answer there is a worse failure than a miss."
|
||||
],
|
||||
"cases": [
|
||||
{ "id": "ru-query-001", "utterance": "сколько воды я выпил с утра", "lang": "ru", "intent": "query", "tags": ["aggregate"] },
|
||||
{ "id": "ru-query-002", "utterance": "я сегодня вообще пил воду", "lang": "ru", "intent": "query", "tags": ["hard", "fact-shaped"], "note": "past-tense fact lexicon in a question — the classifier's fact centroid pulls this hard" },
|
||||
{ "id": "ru-query-003", "utterance": "во сколько я лёг вчера", "lang": "ru", "intent": "query", "tags": ["temporal"] },
|
||||
{ "id": "ru-query-004", "utterance": "давно я не тренировался", "lang": "ru", "intent": "query", "tags": ["hard", "no-question-word"] },
|
||||
{ "id": "ru-query-005", "utterance": "напоминания на завтра есть", "lang": "ru", "intent": "query", "tags": ["hard", "reminder-shaped"], "note": "asks about reminders, does not create one" },
|
||||
{ "id": "ru-query-006", "utterance": "что я записывал про кота", "lang": "ru", "intent": "query", "tags": ["recall"] },
|
||||
{ "id": "ru-query-007", "utterance": "сколько раз я ел вчера", "lang": "ru", "intent": "query", "tags": ["aggregate", "hard"] },
|
||||
{ "id": "ru-query-008", "utterance": "мой вес за последний месяц", "lang": "ru", "intent": "query", "tags": ["no-verb"] },
|
||||
{ "id": "ru-query-009", "utterance": "когда я в последний раз принимал витамины", "lang": "ru", "intent": "query", "tags": ["temporal"] },
|
||||
{ "id": "ru-query-010", "utterance": "есть новости по бэкапу базы", "lang": "ru", "intent": "query", "tags": ["homelab"] },
|
||||
{ "id": "ru-query-011", "utterance": "почему сервер тормозит", "lang": "ru", "intent": "query", "tags": ["homelab", "hard"], "note": "diagnostic question, not a chat opener" },
|
||||
{ "id": "ru-query-012", "utterance": "какие заметки я оставил про полив", "lang": "ru", "intent": "query", "tags": ["recall"] },
|
||||
{ "id": "ru-query-013", "utterance": "во сколько у меня встреча", "lang": "ru", "intent": "query", "tags": ["calendar"] },
|
||||
{ "id": "ru-query-014", "utterance": "я успеваю до дедлайна", "lang": "ru", "intent": "query", "tags": ["hard", "no-question-word"] },
|
||||
{ "id": "ru-query-015", "utterance": "сколько я прошёл шагов", "lang": "ru", "intent": "query", "tags": ["aggregate"] },
|
||||
{ "id": "ru-query-016", "utterance": "покажи давление за неделю", "lang": "ru", "intent": "query", "tags": ["hard", "imperative"], "note": "imperative form but a read — must not route to act" },
|
||||
{ "id": "ru-query-017", "utterance": "чем я занимался в среду", "lang": "ru", "intent": "query", "tags": ["hard", "chat-shaped"] },
|
||||
{ "id": "ru-query-018", "utterance": "хватает ли места под новые бэкапы", "lang": "ru", "intent": "query", "tags": ["homelab"] },
|
||||
{ "id": "en-query-001", "utterance": "did I take my vitamins today", "lang": "en", "intent": "query", "tags": ["fact-shaped"] },
|
||||
{ "id": "en-query-002", "utterance": "how long since the last backup finished", "lang": "en", "intent": "query", "tags": ["temporal"] },
|
||||
{ "id": "en-query-003", "utterance": "show me this week's weight", "lang": "en", "intent": "query", "tags": ["imperative"] },
|
||||
|
||||
{ "id": "ru-fact-001", "utterance": "только что выпил кружку воды", "lang": "ru", "intent": "fact", "want_fact_key": "water" },
|
||||
{ "id": "ru-fact-002", "utterance": "воды попил наконец", "lang": "ru", "intent": "fact", "want_fact_key": "water", "tags": ["inverted"] },
|
||||
{ "id": "ru-fact-003", "utterance": "поужинал", "lang": "ru", "intent": "fact", "want_fact_key": "meal", "tags": ["single-word"] },
|
||||
{ "id": "ru-fact-004", "utterance": "отметь что я позавтракал овсянкой", "lang": "ru", "intent": "fact", "want_fact_key": "meal" },
|
||||
{ "id": "ru-fact-005", "utterance": "поспал часов пять", "lang": "ru", "intent": "fact", "want_fact_key": "sleep" },
|
||||
{ "id": "ru-fact-006", "utterance": "сходил в душ", "lang": "ru", "intent": "fact", "want_fact_key": "shower" },
|
||||
{ "id": "ru-fact-007", "utterance": "отдохнул минут двадцать", "lang": "ru", "intent": "fact", "want_fact_key": "break" },
|
||||
{ "id": "ru-fact-008", "utterance": "запиши вес 74 килограмма", "lang": "ru", "intent": "fact", "tags": ["unparsed-key"], "note": "no recognizer for weight yet — intent must still be fact; HasKey false is allowed" },
|
||||
{ "id": "ru-fact-009", "utterance": "отметь что я выпил таблетки утром", "lang": "ru", "intent": "fact", "tags": ["unparsed-key"] },
|
||||
{ "id": "ru-fact-010", "utterance": "сделал зарядку двадцать минут", "lang": "ru", "intent": "fact", "tags": ["unparsed-key"] },
|
||||
{ "id": "en-fact-001", "utterance": "just drank a glass of water", "lang": "en", "intent": "fact", "want_fact_key": "water" },
|
||||
{ "id": "en-fact-002", "utterance": "slept about seven hours", "lang": "en", "intent": "fact", "want_fact_key": "sleep" },
|
||||
|
||||
{ "id": "ru-rem-001", "utterance": "напомни через сорок минут выключить свет", "lang": "ru", "intent": "reminder", "want_time": true },
|
||||
{ "id": "ru-rem-002", "utterance": "напомни завтра в 7 позвонить в клинику", "lang": "ru", "intent": "reminder", "want_time": true },
|
||||
{ "id": "ru-rem-003", "utterance": "не забудь напомнить мне про счёт за свет", "lang": "ru", "intent": "reminder", "tags": ["no-time"], "note": "no datetime — HasTime false is correct; the daemon asks for a time" },
|
||||
{ "id": "ru-rem-004", "utterance": "поставь напоминание через полчаса", "lang": "ru", "intent": "reminder", "want_time": true },
|
||||
{ "id": "ru-rem-005", "utterance": "разбуди меня в 6:30", "lang": "ru", "intent": "reminder", "want_time": true, "tags": ["hard"], "note": "wake-me phrasing, no напомни stem" },
|
||||
{ "id": "ru-rem-006", "utterance": "напомни послезавтра в 12 забрать заказ", "lang": "ru", "intent": "reminder", "want_time": true },
|
||||
{ "id": "ru-rem-007", "utterance": "через два часа напомни проверить бэкап", "lang": "ru", "intent": "reminder", "want_time": true, "tags": ["inverted"] },
|
||||
{ "id": "en-rem-001", "utterance": "remind me in 45 minutes to stretch", "lang": "en", "intent": "reminder", "want_time": true },
|
||||
{ "id": "en-rem-002", "utterance": "wake me at 6:15", "lang": "en", "intent": "reminder", "want_time": true, "tags": ["hard"] },
|
||||
|
||||
{ "id": "ru-act-001", "utterance": "перезапусти докер", "lang": "ru", "intent": "act", "want_fn": true, "tags": ["homelab", "destructive"] },
|
||||
{ "id": "ru-act-002", "utterance": "выключи свет в спальне", "lang": "ru", "intent": "act", "want_fn": true },
|
||||
{ "id": "ru-act-003", "utterance": "maven, останови бэкап", "lang": "ru", "intent": "act", "want_fn": true, "tags": ["wake-token"] },
|
||||
{ "id": "ru-act-004", "utterance": "включи вытяжку", "lang": "ru", "intent": "act", "want_fn": true },
|
||||
{ "id": "ru-act-005", "utterance": "запусти бэкап сейчас", "lang": "ru", "intent": "act", "want_fn": true },
|
||||
{ "id": "ru-act-006", "utterance": "закрой жалюзи", "lang": "ru", "intent": "act", "want_fn": true },
|
||||
{ "id": "en-act-001", "utterance": "maven, restart the media server", "lang": "en", "intent": "act", "want_fn": true, "tags": ["wake-token"] },
|
||||
{ "id": "en-act-002", "utterance": "turn off the kitchen light", "lang": "en", "intent": "act", "want_fn": true },
|
||||
|
||||
{ "id": "ru-note-001", "utterance": "заметка: продлить домен в августе", "lang": "ru", "intent": "note" },
|
||||
{ "id": "ru-note-002", "utterance": "запиши что кран на кухне снова капает", "lang": "ru", "intent": "note" },
|
||||
{ "id": "ru-note-003", "utterance": "заметка про настройку vlan на свитче", "lang": "ru", "intent": "note", "tags": ["homelab"] },
|
||||
{ "id": "ru-note-004", "utterance": "запиши идею: гидропоника на балконе", "lang": "ru", "intent": "note" },
|
||||
{ "id": "ru-note-005", "utterance": "запиши что сосед просил номер электрика", "lang": "ru", "intent": "note" },
|
||||
{ "id": "en-note-001", "utterance": "note: rotate the kuma api key", "lang": "en", "intent": "note", "tags": ["homelab"] },
|
||||
|
||||
{ "id": "ru-sys-001", "utterance": "сколько сейчас времени в киеве", "lang": "ru", "intent": "system", "tags": ["time"] },
|
||||
{ "id": "ru-sys-002", "utterance": "какое число завтра", "lang": "ru", "intent": "system", "tags": ["date"] },
|
||||
{ "id": "ru-sys-003", "utterance": "переходи в тихий режим", "lang": "ru", "intent": "system", "tags": ["quiet"] },
|
||||
{ "id": "ru-sys-004", "utterance": "хватит тихого режима", "lang": "ru", "intent": "system", "tags": ["quiet", "hard"] },
|
||||
{ "id": "ru-sys-005", "utterance": "какой день недели послезавтра", "lang": "ru", "intent": "system", "tags": ["date"] },
|
||||
{ "id": "en-sys-001", "utterance": "what time is it now", "lang": "en", "intent": "system", "tags": ["time"] },
|
||||
{ "id": "en-sys-002", "utterance": "turn quiet mode back on", "lang": "en", "intent": "system", "tags": ["quiet"] },
|
||||
|
||||
{ "id": "ru-chat-001", "utterance": "мне немного грустно сегодня", "lang": "ru", "intent": "chat" },
|
||||
{ "id": "ru-chat-002", "utterance": "что думаешь про переезд на другую квартиру", "lang": "ru", "intent": "chat" },
|
||||
{ "id": "ru-chat-003", "utterance": "расскажи анекдот про программистов", "lang": "ru", "intent": "chat" },
|
||||
{ "id": "ru-chat-004", "utterance": "доброе утро", "lang": "ru", "intent": "chat", "tags": ["greeting"] },
|
||||
{ "id": "ru-chat-005", "utterance": "спасибо тебе", "lang": "ru", "intent": "chat" },
|
||||
{ "id": "en-chat-001", "utterance": "good morning", "lang": "en", "intent": "chat", "tags": ["greeting"] },
|
||||
{ "id": "en-chat-002", "utterance": "i had a rough day", "lang": "en", "intent": "chat" },
|
||||
|
||||
{ "id": "amb-001", "utterance": "вода", "lang": "ru", "want_clarify": true, "tags": ["ambiguous"], "note": "fact write or a query? one noun decides nothing — ask" },
|
||||
{ "id": "amb-002", "utterance": "бэкап", "lang": "ru", "want_clarify": true, "tags": ["ambiguous"], "note": "run it, or report on it? a confident act here is destructive" },
|
||||
{ "id": "amb-003", "utterance": "ну это", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "filler"] },
|
||||
{ "id": "amb-004", "utterance": "сделай это", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "anaphora"], "note": "unresolved anaphora with an imperative — must not guess an fn" },
|
||||
{ "id": "amb-005", "utterance": "потом", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "filler"] },
|
||||
{ "id": "amb-006", "utterance": "the thing from earlier", "lang": "en", "want_clarify": true, "tags": ["ambiguous", "anaphora"] }
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user