diff --git a/internal/router/eval/reach_test.go b/internal/router/eval/reach_test.go new file mode 100644 index 0000000..2513da1 --- /dev/null +++ b/internal/router/eval/reach_test.go @@ -0,0 +1,228 @@ +package eval + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kami/maven/internal/router" +) + +func TestLoadReachFixture(t *testing.T) { + f, err := LoadReach() + if err != nil { + t.Fatalf("LoadReach: %v", err) + } + if _, err := f.Now(); err != nil { + t.Fatalf("Now: %v", err) + } + valid := map[Service]bool{ServiceNone: true, ServicePraxis: true, ServiceHexis: true} + seen := map[string]bool{} + byService := map[Service]int{} + for _, c := range f.Cases { + if c.ID == "" || seen[c.ID] { + t.Errorf("case %q: empty or duplicate id", c.ID) + } + seen[c.ID] = true + if !valid[c.WantService] { + t.Errorf("%s: want_service %q, want none|praxis|hexis", c.ID, c.WantService) + } + if c.Lang != "ru" && c.Lang != "en" { + t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang) + } + // A capability only means something on the Praxis side, and it must + // name an arm that exists — otherwise the case asserts a target the + // dispatch can never hit. + if c.WantCapability != "" { + if c.WantService != ServicePraxis { + t.Errorf("%s: want_capability on %q", c.ID, c.WantService) + } + if PraxisAliases[c.WantCapability] != c.WantCapability { + t.Errorf("%s: capability %q is not a praxis arm", c.ID, c.WantCapability) + } + } + byService[c.WantService]++ + } + // Coverage floor. The negative cases matter most: without them a router + // that sent everything to Hexis would score perfectly. + for s := range valid { + if byService[s] < 5 { + t.Errorf("want_service %q has %d cases, want >= 5", s, byService[s]) + } + } +} + +// TestReachFixtureIsHeldOut — same rule as TestFixtureIsHeldOut. The classifier +// routes by similarity to frozen seeds, so a case copied from models/seeds +// would measure memorisation rather than reach. +func TestReachFixtureIsHeldOut(t *testing.T) { + f, err := LoadReach() + if err != nil { + t.Fatalf("LoadReach: %v", err) + } + seeds := loadSeeds(t) + for _, c := range f.Cases { + if src, ok := seeds[normalize(c.Utterance)]; ok { + t.Errorf("%s: %q is verbatim in %s — not held out", c.ID, c.Utterance, src) + } + } +} + +// TestReachDerivation pins the gate order that ScoreReach depends on. These are +// the four branches of actionAct plus the clarify pre-empt, asserted directly +// so a change to cmd/mavend/actions_act.go that this package no longer mirrors +// fails here rather than silently moving the score. +func TestReachDerivation(t *testing.T) { + m := router.DefaultActMatcher{Fns: actFns} + cases := []struct { + name string + dec router.Decision + want Service + capability string + }{ + { + name: "act with a praxis alias in the fn slot", + dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "list_attention", HasFn: true}}, + want: ServicePraxis, capability: "list_attention", + }, + { + name: "act with a russian praxis alias", + dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "готово", HasFn: true}}, + want: ServicePraxis, capability: "resolve_item", + }, + { + name: "act with an entity but no fn earns one from the matcher", + dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Text: "выключи свет в спальне"}}, + want: ServiceHexis, + }, + { + name: "act with a non-praxis fn and text goes to hexis", + dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "restart", HasFn: true, Text: "перезапусти гитею"}}, + want: ServiceHexis, + }, + { + name: "act with a fn and no text stays local", + dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "restart", HasFn: true}}, + want: ServiceNone, + }, + { + name: "a clarified act with text still reaches hexis", + dec: router.Decision{Intent: router.IntentAct, Clarify: true, Slots: router.Slots{Text: "выключи это"}}, + want: ServiceHexis, + }, + { + name: "a clarified act that already has a fn does not", + dec: router.Decision{Intent: router.IntentAct, Clarify: true, Slots: router.Slots{Fn: "restart", HasFn: true, Text: "перезапусти"}}, + want: ServiceNone, + }, + { + name: "no intent but act reaches nothing", + dec: router.Decision{Intent: router.IntentQuery, Slots: router.Slots{Fn: "list_attention", HasFn: true, Text: "что требует внимания"}}, + want: ServiceNone, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, capability := Reach(tc.dec, m) + if got != tc.want { + t.Errorf("Reach = %q, want %q", got, tc.want) + } + if capability != tc.capability { + t.Errorf("capability = %q, want %q", capability, tc.capability) + } + }) + } +} + +// TestPraxisAliasesShape — the copy of praxisCapabilities in reach.go is a +// drift risk (see its comment). This does not close it, but it does catch the +// cheap half: an arm losing all its aliases, or an alias pointing at an arm +// that no longer has a canonical name. +func TestPraxisAliasesShape(t *testing.T) { + arms := map[string]int{} + for alias, capability := range PraxisAliases { + if alias == "" || capability == "" { + t.Errorf("empty alias or capability: %q → %q", alias, capability) + } + arms[capability]++ + } + for _, want := range []string{ + "list_attention", "acknowledge_item", "resolve_item", + "ignore_item", "pin_item", "list_changes", "entity_attention", + } { + if arms[want] == 0 { + t.Errorf("praxis arm %q has no aliases", want) + } + // Every arm must be reachable by its own name, which is the alias the + // LLM router's fn slot actually emits. + if PraxisAliases[want] != want { + t.Errorf("arm %q does not alias to itself", want) + } + } +} + +// TestReachBaselineHash — the deterministic ratchet. Same deal as +// TestClassifierBaseline: the hash embedder never clears the confidence gate on +// paraphrases, so almost everything lands on clarify, and the number this +// asserts is the overreach count rather than the accuracy. +// +// Overreach is the direction worth a hard assertion. An utterance that should +// stay inside Maven and instead resolves an entity through Nexus is one Hexis +// capability away from executing something nobody asked for, and unlike a miss +// he never gets asked about it. +func TestReachBaselineHash(t *testing.T) { + f, err := LoadReach() + if err != nil { + t.Fatalf("LoadReach: %v", err) + } + m := router.DefaultActMatcher{Fns: actFns} + rep, err := ScoreReach(context.Background(), "reach: classifier+hash", + newBaselineRouter(t, router.NewHashEmbedder(1024), nil), m, f) + if err != nil { + t.Fatalf("ScoreReach: %v", err) + } + t.Log("\n" + rep.String() + rep.Failures()) + + if rep.Overreach > 4 { + t.Errorf("%d utterances reached a service they should not have, want <= 4:\n%s", + rep.Overreach, rep.Failures()) + } +} + +// TestReachBaselineONNX — the deployed configuration: the cascade with the +// multilingual e5 embedder. Opt-in via MAVEN_ONNX_LIB, same as TestONNXBaseline, +// because deps/ is gitignored. `make eval-reach` points it at the vendored copy. +// +// Reports rather than asserts. This is the number Vikunja #405 asked for, and a +// threshold invented alongside the first measurement is not a ratchet, it is a +// guess written down twice. +func TestReachBaselineONNX(t *testing.T) { + lib := os.Getenv("MAVEN_ONNX_LIB") + if lib == "" { + t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") + } + model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx") + tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json") + for _, p := range []string{lib, model, tok} { + if _, err := os.Stat(p); err != nil { + t.Skipf("missing %s: %v", p, err) + } + } + emb, err := router.NewONNXEmbedder(model, tok, lib) + if err != nil { + t.Skipf("onnx embedder unavailable: %v", err) + } + defer emb.Close() + + f, err := LoadReach() + if err != nil { + t.Fatalf("LoadReach: %v", err) + } + rep, err := ScoreReach(context.Background(), "reach: classifier+onnx", + newBaselineRouter(t, emb, nil), router.DefaultActMatcher{Fns: actFns}, f) + if err != nil { + t.Fatalf("ScoreReach: %v", err) + } + t.Log("\n" + rep.String() + rep.Failures()) +}