Maven/internal/router/eval/eval_test.go

package eval

import (
	"bufio"
	"context"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/kami/maven/internal/config"
	"github.com/kami/maven/internal/router"
)

const seedDir = "../../../models/seeds"

// actFns — the verbs the baseline act matcher accepts. In the daemon the
// allowlist is exactly the enabled tool names (see buildRouter in
// cmd/mavend/voice.go); here it stands in for a deployment's tools so the
// fixture's want_fn cases are satisfiable at all. DefaultActMatcher is
// verb-prefix only, so these are verbs, not full commands.
var actFns = []string{
	"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
	"закрой", "открой", "сделай", "поставь",
	"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
}

func TestLoadFixture(t *testing.T) {
	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	if _, err := f.Now(); err != nil {
		t.Fatalf("Now: %v", err)
	}
	valid := map[router.Intent]bool{
		router.IntentAct: true, router.IntentReminder: true, router.IntentFact: true,
		router.IntentNote: true, router.IntentQuery: true, router.IntentChat: true,
		router.IntentSystem: true,
	}
	seen := map[string]bool{}
	for _, c := range f.Cases {
		if c.ID == "" || seen[c.ID] {
			t.Errorf("case %q: empty or duplicate id", c.ID)
		}
		seen[c.ID] = true
		if strings.TrimSpace(c.Utterance) == "" {
			t.Errorf("%s: empty utterance", c.ID)
		}
		if c.Lang != "ru" && c.Lang != "en" {
			t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang)
		}
		// Intent is empty exactly when the case expects a refusal — otherwise
		// a typo'd intent would score as an unreachable target forever.
		if c.WantClarify {
			if c.Intent != "" {
				t.Errorf("%s: want_clarify with intent %q — pick one", c.ID, c.Intent)
			}
			if c.WantTime || c.WantFn || c.WantFactKey != "" {
				t.Errorf("%s: want_clarify with slot expectations", c.ID)
			}
			continue
		}
		if !valid[c.Intent] {
			t.Errorf("%s: intent %q not in the seven", c.ID, c.Intent)
		}
		// Slot expectations must match the intent that owns that slot, or the
		// case asserts something Extract never fills.
		if c.WantTime && c.Intent != router.IntentReminder {
			t.Errorf("%s: want_time on intent %q", c.ID, c.Intent)
		}
		if c.WantFn && c.Intent != router.IntentAct {
			t.Errorf("%s: want_fn on intent %q", c.ID, c.Intent)
		}
		if c.WantFactKey != "" && c.Intent != router.IntentFact {
			t.Errorf("%s: want_fact_key on intent %q", c.ID, c.Intent)
		}
	}
	// Coverage floor: every intent plus the refusal lane must be represented,
	// or a path can regress to zero without the number moving.
	byIntent := map[router.Intent]int{}
	clarify := 0
	for _, c := range f.Cases {
		if c.WantClarify {
			clarify++
			continue
		}
		byIntent[c.Intent]++
	}
	for in := range valid {
		if byIntent[in] < 5 {
			t.Errorf("intent %q has %d cases, want >= 5", in, byIntent[in])
		}
	}
	if clarify < 5 {
		t.Errorf("%d clarify cases, want >= 5", clarify)
	}
}

// TestFixtureIsHeldOut — the fixture's whole claim to measuring anything. The
// classifier routes by similarity to frozen seed phrases, so a fixture sharing
// utterances with models/seeds/*.txt would score its own training set. Verbatim
// match is the line drawn: paraphrases are the point of the fixture, copies are
// the failure.
func TestFixtureIsHeldOut(t *testing.T) {
	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	seeds := loadSeeds(t)
	for _, c := range f.Cases {
		if src, ok := seeds[normalize(c.Utterance)]; ok {
			t.Errorf("%s: %q is verbatim in %s — not held out", c.ID, c.Utterance, src)
		}
	}
}

// TestClassifierBaseline — the number Vikunja #319 compares against. This is
// the committed stopgap path (stage 0 grammar → nearest-centroid classifier
// over the real models/seeds corpus), scored on held-out utterances.
//
// It runs on HashEmbedder, not the ONNX multilingual embedder: the hash floor
// is deterministic, so this baseline is reproducible in CI. The production
// ONNX number will be higher; measure both against the same fixture before
// flipping the default (#320), and never compare a hash-embedder run to an
// ONNX one.
//
// The assertion is a ratchet, not a target — it only catches a regression
// below what the cascade already does. The gap between it and 100% is the
// backlog.
func TestClassifierBaseline(t *testing.T) {
	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	// dim 1024: the hash embedder is bag-of-words, so a narrower space would
	// collide tokens across intents and measure the hash, not the centroids.
	rep, err := Score(context.Background(), "classifier+hash", newBaselineRouter(t, router.NewHashEmbedder(1024), nil), f)
	if err != nil {
		t.Fatalf("Score: %v", err)
	}
	t.Log("\n" + rep.String() + rep.Failures())

	// 0.15 is under the observed 0.171, not a target. Almost every case here
	// falls to clarify because the hash embedder's cosine never clears the
	// 0.55 gate on paraphrases — which is the documented floor behaviour
	// (AGENTS.md: "Russian recall rarely clears the confidence gate"), not a
	// bug this fixture is asking anyone to fix. The number worth moving is
	// TestONNXBaseline's.
	const floor = 0.15
	if rep.Accuracy() < floor {
		t.Errorf("accuracy %.3f below ratchet %.2f — routing regressed", rep.Accuracy(), floor)
	}
	// The dangerous direction is asserted separately and tightly: a confident
	// route for "сделай это" is a destructive guess, and unlike a miss the user
	// never gets asked. Clarify cases must not silently start deciding.
	if rep.MissedClarify > 2 {
		t.Errorf("%d ambiguous utterances routed confidently, want <= 2:\n%s", rep.MissedClarify, rep.Failures())
	}
}

// TestONNXBaseline — the number that actually belongs in Vikunja #319: the
// deployed cascade with the multilingual ONNX embedder, the configuration
// homesrv runs. Opt-in via MAVEN_ONNX_LIB because deps/ is gitignored, so the
// runtime is not guaranteed to exist on a fresh clone; CI has the hash ratchet
// above instead. `make eval-router` defaults the variable to the vendored
// deps/onnxruntime-linux-x64-1.26.0 copy.
//
// Measured 2026-07-31 on deps/onnxruntime-linux-x64-1.26.0: 28/76 (36.8%),
// 21 false clarifies, 5 MISSED clarifies, p50 ~30-70ms. The missed clarifies
// are the finding — every one of the six ambiguous utterances scores higher
// cosine under ONNX than under the hash floor, so the 0.55 gate that held them
// back stops holding: "сделай это" routes to act at 0.847. A better embedder
// made the refusal lane worse, which is an argument about the gate, not the
// embedder.
//
// Reports rather than asserts: the score is an input to the #320 flip decision,
// and a threshold invented here would just be a second opinion about the same
// unmeasured thing. Compare it to an LLM-router run over the SAME fixture —
// accuracy and p50/p95 latency both, since the resident model pays CPU seconds
// per turn that the classifier's ~10µs does not.
func TestONNXBaseline(t *testing.T) {
	lib := os.Getenv("MAVEN_ONNX_LIB")
	if lib == "" {
		t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
	}
	model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx")
	tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json")
	for _, p := range []string{lib, model, tok} {
		if _, err := os.Stat(p); err != nil {
			t.Skipf("missing %s: %v", p, err)
		}
	}
	emb, err := router.NewONNXEmbedder(model, tok, lib)
	if err != nil {
		t.Skipf("onnx embedder unavailable: %v", err)
	}
	defer emb.Close()

	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	rep, err := Score(context.Background(), "classifier+onnx", newBaselineRouter(t, emb, nil), f)
	if err != nil {
		t.Fatalf("Score: %v", err)
	}
	t.Log("\n" + rep.String() + rep.Failures())
}

// newBaselineRouter mirrors buildRouter in cmd/mavend/voice.go — same grammar
// set, same extractor floors, same seed corpus, same confidence gate — so the
// score reflects the deployed cascade and not a test-local approximation. Only
// the embedder varies, and that is the axis being measured.
func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter) *router.Router {
	t.Helper()
	acts := router.DefaultActMatcher{Fns: actFns}
	cls := router.NewClassifier(emb)
	ctx := context.Background()
	seeds := seedsWithIntent(t)
	texts := make([]string, 0, len(seeds))
	for text := range seeds {
		texts = append(texts, text)
	}
	// Sorted: centroids are order-independent, but tie-breaking in Classify's
	// result sort is not, and a jittering baseline can't be a ratchet.
	sort.Strings(texts)
	for _, text := range texts {
		if err := cls.AddExample(ctx, seeds[text], text); err != nil {
			t.Fatalf("seed %q: %v", text, err)
		}
	}
	grammars := router.DefaultGrammars(acts)
	grammars = append(grammars, router.SystemTimeDateGrammars()...)
	// Same order as buildRouter (voicewire.go). The fixture is only worth
	// anything while its grammar set is the daemon's grammar set.
	grammars = append(grammars, router.AgendaQueryGrammars()...)
	grammars = append(grammars, router.FeedQueryGrammar())
	// The list side of the same exposure: a phrasing with no possessive in it
	// ("список дел") routed system and never reached queryTasks (Vikunja #467).
	grammars = append(grammars, router.TaskListGrammar())
	grammars = append(grammars, router.ReminderGrammar())
	grammars = append(grammars, router.TaskCaptureGrammar())
	// "расскажи про X" is a world question the model called a fact, and the
	// rule goes last because it matches on the first word alone (Vikunja #498).
	grammars = append(grammars, router.NarrativeQueryGrammars()...)
	return router.New(router.Config{
		Grammars:   grammars,
		Classifier: cls,
		Extractor: router.Extractor{
			Time:  router.StubDateTimeParser{},
			Acts:  acts,
			Facts: router.DefaultFactParser{},
		},
		// The deployed gate, not a test-local one: a fixture scored at a looser
		// threshold reports an accuracy no real turn would see.
		Threshold: config.DefaultRouterThreshold,
		LLM:       llmR,
	})
}

// seedOrder — fixed iteration order over the corpus. Not cosmetic: a few
// phrases appear under two intents ("как дела у сервера" is in both query.txt
// and system.txt), and ranging over a map would hand the duplicate to a
// different centroid on every run, which makes the baseline score jitter and
// the ratchet meaningless.
var seedOrder = []router.Intent{
	router.IntentAct, router.IntentReminder, router.IntentFact,
	router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
}

// loadSeeds maps normalized seed text → the file it came from.
func loadSeeds(t *testing.T) map[string]string {
	t.Helper()
	files := readSeedFiles(t)
	out := map[string]string{}
	for _, intent := range seedOrder {
		for _, l := range files[intent] {
			out[normalize(l)] = string(intent) + ".txt"
		}
	}
	return out
}

// seedsWithIntent flattens the corpus to text→intent. A phrase appearing under
// two intents collapses to one example — the classifier would otherwise train
// two centroids to fight over the same vector. First intent in seedOrder wins,
// deterministically.
func seedsWithIntent(t *testing.T) map[string]router.Intent {
	t.Helper()
	files := readSeedFiles(t)
	out := map[string]router.Intent{}
	for _, intent := range seedOrder {
		for _, l := range files[intent] {
			if _, dup := out[l]; dup {
				continue
			}
			out[l] = intent
		}
	}
	return out
}

func readSeedFiles(t *testing.T) map[router.Intent][]string {
	t.Helper()
	out := map[router.Intent][]string{}
	for _, in := range seedOrder {
		path := filepath.Join(seedDir, string(in)+".txt")
		fh, err := os.Open(path)
		if err != nil {
			t.Fatalf("open %s: %v", path, err)
		}
		sc := bufio.NewScanner(fh)
		for sc.Scan() {
			line := strings.TrimSpace(sc.Text())
			if line == "" || strings.HasPrefix(line, "#") {
				continue
			}
			out[in] = append(out[in], line)
		}
		err = sc.Err()
		fh.Close()
		if err != nil {
			t.Fatalf("read %s: %v", path, err)
		}
	}
	return out
}

// normalize — held-out comparison is on lowercased, whitespace-collapsed text
// so a copied seed can't sneak in behind capitalisation.
func normalize(s string) string {
	return strings.Join(strings.Fields(strings.ToLower(s)), " ")
}