Maven/internal/memory/recalleval/recalleval_test.go

package recalleval

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"unicode"

	"github.com/kami/maven/internal/config"
	"github.com/kami/maven/internal/memory"
	"github.com/kami/maven/internal/router"
	"github.com/kami/maven/internal/store"
)

// dim 1024 for the hash embedder: it is bag-of-words, so a narrower space
// collides tokens between unrelated notes and would measure the hash.
const hashDim = 1024

func TestLoadFixture(t *testing.T) {
	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	if len(f.Cases) < 25 {
		t.Errorf("%d cases, want >= 25", len(f.Cases))
	}
	// Filler is what stops recall@3 being free: three case notes and a top-3
	// search would put the wanted note in the top 3 every time.
	if len(f.Filler) < 10 {
		t.Errorf("%d filler notes, want >= 10", len(f.Filler))
	}
	seen := map[string]bool{}
	silent, en := 0, 0
	for _, c := range f.Cases {
		if c.ID == "" || seen[c.ID] {
			t.Errorf("case %q: empty or duplicate id", c.ID)
		}
		seen[c.ID] = true
		if c.Lang != "ru" && c.Lang != "en" {
			t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang)
		}
		if c.Lang == "en" {
			en++
		}
		if strings.TrimSpace(c.Query) == "" {
			t.Errorf("%s: empty query", c.ID)
		}
		// Fewer than three notes and a wrong answer has nowhere to come from,
		// so recall@1 would be near-free.
		if len(c.Notes) < 3 {
			t.Errorf("%s: %d notes, want >= 3", c.ID, len(c.Notes))
		}
		ids := map[string]bool{}
		for _, n := range c.Notes {
			if n.ID == "" || ids[n.ID] {
				t.Errorf("%s: note %q empty or duplicate id", c.ID, n.ID)
			}
			ids[n.ID] = true
			if strings.TrimSpace(n.Text) == "" {
				t.Errorf("%s: note %q empty text", c.ID, n.ID)
			}
			if n.Kind != "note" && n.Kind != "fact" {
				t.Errorf("%s: note %q kind %q, want note|fact", c.ID, n.ID, n.Kind)
			}
		}
		if !c.Answerable() {
			silent++
			continue
		}
		if !ids[c.Want] {
			t.Errorf("%s: want %q is not one of the case's notes", c.ID, c.Want)
		}
	}
	// Both lanes need enough cases that a rate means something.
	if silent < 5 {
		t.Errorf("%d must-be-silent cases, want >= 5", silent)
	}
	if en < 5 {
		t.Errorf("%d English cases, want >= 5", en)
	}
}

// TestFixtureIsParaphrased — the fixture's claim to measuring recall at all. If
// a query repeats its note's words, cosine over a bag-of-words embedder gets it
// for free and the score says nothing about semantic recall. Half the query's
// words is the line: some shared vocabulary is natural ("nginx", "чай"), a copy
// is the failure.
func TestFixtureIsParaphrased(t *testing.T) {
	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	for _, c := range f.Cases {
		if !c.Answerable() {
			continue
		}
		var want string
		for _, n := range c.Notes {
			if n.ID == c.Want {
				want = n.Text
			}
		}
		q := words(c.Query)
		if len(q) == 0 {
			continue
		}
		inNote := map[string]bool{}
		for _, w := range words(want) {
			inNote[w] = true
		}
		shared := 0
		for _, w := range q {
			if inNote[w] {
				shared++
			}
		}
		if frac := float64(shared) / float64(len(q)); frac > 0.5 {
			t.Errorf("%s: query shares %.0f%% of its words with the note — not a paraphrase\n  query: %q\n  note:  %q",
				c.ID, 100*frac, c.Query, want)
		}
	}
}

// words — lowercased words of two runes or more, matching how the hash
// embedder tokenizes.
func words(s string) []string {
	var out []string
	for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
	}) {
		if len([]rune(w)) > 1 {
			out = append(out, w)
		}
	}
	return out
}

// TestBestRecallMatchesDaemon — the harness duplicates bestRecall from
// cmd/mavend/recall.go (package main is not importable). This pins the copy to
// the original's three rules: no hits, below the gate, or no text ⇒ silence.
func TestBestRecallMatchesDaemon(t *testing.T) {
	if got := bestRecall(nil, 0.55, 0); got != "" {
		t.Errorf("no hits: got %q, want silence", got)
	}
	low := []memory.Result{{ID: "a", Score: 0.4, Meta: map[string]string{"text": "чай"}}}
	if got := bestRecall(low, 0.55, 0); got != "" {
		t.Errorf("below gate: got %q, want silence", got)
	}
	noText := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{}}}
	if got := bestRecall(noText, 0.55, 0); got != "" {
		t.Errorf("no text: got %q, want silence", got)
	}
	ok := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{"text": "чай"}}}
	if got := bestRecall(ok, 0.55, 0); got != "чай" {
		t.Errorf("above gate: got %q, want %q", got, "чай")
	}
	// Margin: a close runner-up means the embedder cannot tell the two apart,
	// so Maven stays silent even though both clear the absolute floor.
	close := []memory.Result{
		{ID: "a", Score: 0.86, Meta: map[string]string{"text": "чай"}},
		{ID: "b", Score: 0.85, Meta: map[string]string{"text": "кофе"}},
	}
	if got := bestRecall(close, 0.55, 0.03); got != "" {
		t.Errorf("thin margin: got %q, want silence", got)
	}
	if got := bestRecall(close, 0.55, 0); got != "чай" {
		t.Errorf("margin off: got %q, want %q", got, "чай")
	}
	clear := []memory.Result{
		{ID: "a", Score: 0.86, Meta: map[string]string{"text": "чай"}},
		{ID: "b", Score: 0.70, Meta: map[string]string{"text": "кофе"}},
	}
	if got := bestRecall(clear, 0.55, 0.03); got != "чай" {
		t.Errorf("wide margin: got %q, want %q", got, "чай")
	}
}

// TestHashRecallBaseline — the CI ratchet. HashEmbedder, so it needs no model
// files and is byte-for-byte reproducible.
//
// It is a floor, not a target. The hash embedder is lexical, so most of this
// fixture is unwinnable for it by construction; the number worth moving is
// TestONNXRecall's. Never compare a hash-embedder number to an ONNX one.
func TestHashRecallBaseline(t *testing.T) {
	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	rep, err := Score(context.Background(), "recall+hash", router.NewHashEmbedder(hashDim), InMemory,
		config.DefaultQueryMinScore, config.DefaultQueryMinMargin, f)
	if err != nil {
		t.Fatalf("Score: %v", err)
	}
	t.Log("\n" + rep.String() + rep.Failures())
	t.Log("\ngate sweep:\n" + sweep(t, router.NewHashEmbedder(hashDim), f))

	// 0.32 sits under the observed 0.370 recall@1 (was 0.360 over 25 answerable
	// cases; the two mixed note+fact cases added with #373 make it 27).
	const floorRecall1 = 0.32
	if rep.Recall1() < floorRecall1 {
		t.Errorf("recall@1 %.3f below ratchet %.2f — note recall regressed", rep.Recall1(), floorRecall1)
	}
	// The dangerous direction, asserted tightly and separately: answering from
	// the wrong note is worse than a gap. Observed 0 under the hash floor.
	if rep.FalseRecall > 1 {
		t.Errorf("%d false recalls, want <= 1:\n%s", rep.FalseRecall, rep.Failures())
	}
}

// TestPersistentStoreScoresTheSame — the deployed store is sqlite-backed
// (store.MemoryStore via st.VectorMemory()), not the in-memory fallback. Its
// Search is a separate implementation of the same cosine scan, so it gets its
// own run: a divergence here would mean recall quality depends on whether a
// database was configured.
func TestPersistentStoreScoresTheSame(t *testing.T) {
	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	emb := router.NewHashEmbedder(hashDim)
	inMem, err := Score(context.Background(), "recall+hash+memory", emb, InMemory, config.DefaultQueryMinScore, config.DefaultQueryMinMargin, f)
	if err != nil {
		t.Fatalf("Score in-memory: %v", err)
	}
	persistent, err := Score(context.Background(), "recall+hash+sqlite", emb, sqliteStores(t), config.DefaultQueryMinScore, config.DefaultQueryMinMargin, f)
	if err != nil {
		t.Fatalf("Score sqlite: %v", err)
	}
	t.Log("\n" + persistent.String())
	if persistent.Rank1 != inMem.Rank1 || persistent.FalseRecall != inMem.FalseRecall {
		t.Errorf("sqlite recall@1 %d/%d fr %d, in-memory %d/%d fr %d — the two backends disagree",
			persistent.Rank1, persistent.Answerable, persistent.FalseRecall,
			inMem.Rank1, inMem.Answerable, inMem.FalseRecall)
	}
}

// sqliteStores returns a NewStore that hands each case its own plaintext
// database file, so cases stay isolated the way they are with InMemory.
func sqliteStores(t *testing.T) NewStore {
	t.Helper()
	dir := t.TempDir()
	n := 0
	return func() (memory.Store, func(), error) {
		n++
		st, err := store.Open(context.Background(), filepath.Join(dir, fmt.Sprintf("recall-%d.db", n)))
		if err != nil {
			return nil, nil, err
		}
		return st.VectorMemory(), func() { _ = st.Close() }, nil
	}
}

// TestONNXRecall — the number that matters: the multilingual embedder homesrv
// actually runs. Opt-in via MAVEN_ONNX_LIB because deps/ is gitignored, exactly
// like TestONNXBaseline in internal/router/eval. `make eval-recall` points it at
// the vendored runtime.
//
// Reports rather than asserts. The gate sweep is the point: it prints
// answered-vs-false-recall at a range of query_min_score values, so the right
// threshold is read off data instead of guessed.
func TestONNXRecall(t *testing.T) {
	lib := os.Getenv("MAVEN_ONNX_LIB")
	if lib == "" {
		t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
	}
	model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx")
	tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json")
	for _, p := range []string{lib, model, tok} {
		if _, err := os.Stat(p); err != nil {
			t.Skipf("missing %s: %v", p, err)
		}
	}
	emb, err := router.NewONNXEmbedder(model, tok, lib)
	if err != nil {
		t.Skipf("onnx embedder unavailable: %v", err)
	}
	defer emb.Close()

	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	rep, err := Score(context.Background(), "recall+onnx", emb, InMemory, config.DefaultQueryMinScore, config.DefaultQueryMinMargin, f)
	if err != nil {
		t.Fatalf("Score: %v", err)
	}
	t.Log("\n" + rep.String() + rep.Failures())
	// Cached for the sweeps only: the headline run above must pay the real
	// embedder cost so its latency numbers mean something.
	cached := Cache(emb)
	t.Log("\ngate sweep (margin off):\n" + sweep(t, cached, f))
	t.Log("\nmargin sweep (gate 0.55):\n" + marginSweep(t, cached, f))
}

// sweep scores the fixture at a range of gates and renders one line each. Two
// columns matter: how many real questions get answered, and how many made-up
// ones get answered anyway. A gate is only defensible if some value keeps the
// first high and the second at zero.
func sweep(t *testing.T, emb router.Embedder, f Fixture) string {
	t.Helper()
	var b strings.Builder
	for _, gate := range []float64{0.0, 0.30, 0.40, 0.50, 0.55, 0.60, 0.70, 0.80, 0.90} {
		rep, err := Score(context.Background(), "sweep", emb, InMemory, gate, 0, f)
		if err != nil {
			t.Fatalf("sweep at %.2f: %v", gate, err)
		}
		fmt.Fprintf(&b, "  gate %.2f: answered %d/%d (%.0f%%)  false recall %d/%d\n",
			gate, rep.Rank1-rep.Gated, rep.Answerable, 100*rep.Answered(), rep.FalseRecall, rep.NoAnswer)
	}
	return b.String()
}

// marginSweep is the same idea for the margin gate (top1 − top2 > delta), with
// the absolute gate held at its default. The absolute score cannot separate a
// real hit from a made-up question under e5 — every score lands in one narrow
// band — so this sweep is the one that picks a number.
func marginSweep(t *testing.T, emb router.Embedder, f Fixture) string {
	t.Helper()
	var b strings.Builder
	for _, d := range []float64{0, 0.002, 0.005, 0.008, 0.01, 0.012, 0.015, 0.02, 0.025, 0.03, 0.04, 0.05, 0.06} {
		rep, err := Score(context.Background(), "margin sweep", emb, InMemory, config.DefaultQueryMinScore, d, f)
		if err != nil {
			t.Fatalf("margin sweep at %.3f: %v", d, err)
		}
		fmt.Fprintf(&b, "  delta %.3f: answered %d/%d (%.0f%%)  false recall %d/%d\n",
			d, rep.Rank1-rep.Gated, rep.Answerable, 100*rep.Answered(), rep.FalseRecall, rep.NoAnswer)
	}
	return b.String()
}