claude
9aabb01e2a
Every case is scored over its own notes plus the whole filler set, and the two stores disagree about a repeated id: sqlite upserts on it, the in-memory store appends. So one collision makes a case score differently on the two backends, and it reads as an embedder or gate difference — the one thing this harness exists to measure. It was dodged by hand during #373 by renaming two ids. The check sits in Load rather than in TestLoadFixture, so it covers every caller of the fixture and not only the one that remembers to look.
365 lines
13 KiB
Go
365 lines
13 KiB
Go
package recalleval
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"unicode"
|
||
|
||
"github.com/kami/maven/internal/config"
|
||
"github.com/kami/maven/internal/memory"
|
||
"github.com/kami/maven/internal/router"
|
||
"github.com/kami/maven/internal/store"
|
||
)
|
||
|
||
// dim 1024 for the hash embedder: it is bag-of-words, so a narrower space
|
||
// collides tokens between unrelated notes and would measure the hash.
|
||
const hashDim = 1024
|
||
|
||
func TestLoadFixture(t *testing.T) {
|
||
f, err := Load()
|
||
if err != nil {
|
||
t.Fatalf("Load: %v", err)
|
||
}
|
||
if len(f.Cases) < 25 {
|
||
t.Errorf("%d cases, want >= 25", len(f.Cases))
|
||
}
|
||
// Filler is what stops recall@3 being free: three case notes and a top-3
|
||
// search would put the wanted note in the top 3 every time.
|
||
if len(f.Filler) < 10 {
|
||
t.Errorf("%d filler notes, want >= 10", len(f.Filler))
|
||
}
|
||
seen := map[string]bool{}
|
||
silent, en := 0, 0
|
||
for _, c := range f.Cases {
|
||
if c.ID == "" || seen[c.ID] {
|
||
t.Errorf("case %q: empty or duplicate id", c.ID)
|
||
}
|
||
seen[c.ID] = true
|
||
if c.Lang != "ru" && c.Lang != "en" {
|
||
t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang)
|
||
}
|
||
if c.Lang == "en" {
|
||
en++
|
||
}
|
||
if strings.TrimSpace(c.Query) == "" {
|
||
t.Errorf("%s: empty query", c.ID)
|
||
}
|
||
// Fewer than three notes and a wrong answer has nowhere to come from,
|
||
// so recall@1 would be near-free.
|
||
if len(c.Notes) < 3 {
|
||
t.Errorf("%s: %d notes, want >= 3", c.ID, len(c.Notes))
|
||
}
|
||
ids := map[string]bool{}
|
||
for _, n := range c.Notes {
|
||
if n.ID == "" || ids[n.ID] {
|
||
t.Errorf("%s: note %q empty or duplicate id", c.ID, n.ID)
|
||
}
|
||
ids[n.ID] = true
|
||
if strings.TrimSpace(n.Text) == "" {
|
||
t.Errorf("%s: note %q empty text", c.ID, n.ID)
|
||
}
|
||
if n.Kind != "note" && n.Kind != "fact" {
|
||
t.Errorf("%s: note %q kind %q, want note|fact", c.ID, n.ID, n.Kind)
|
||
}
|
||
}
|
||
if !c.Answerable() {
|
||
silent++
|
||
continue
|
||
}
|
||
if !ids[c.Want] {
|
||
t.Errorf("%s: want %q is not one of the case's notes", c.ID, c.Want)
|
||
}
|
||
}
|
||
// Both lanes need enough cases that a rate means something.
|
||
if silent < 5 {
|
||
t.Errorf("%d must-be-silent cases, want >= 5", silent)
|
||
}
|
||
if en < 5 {
|
||
t.Errorf("%d English cases, want >= 5", en)
|
||
}
|
||
}
|
||
|
||
// TestFixtureIsParaphrased — the fixture's claim to measuring recall at all. If
|
||
// a query repeats its note's words, cosine over a bag-of-words embedder gets it
|
||
// for free and the score says nothing about semantic recall. Half the query's
|
||
// words is the line: some shared vocabulary is natural ("nginx", "чай"), a copy
|
||
// is the failure.
|
||
func TestFixtureIsParaphrased(t *testing.T) {
|
||
f, err := Load()
|
||
if err != nil {
|
||
t.Fatalf("Load: %v", err)
|
||
}
|
||
for _, c := range f.Cases {
|
||
if !c.Answerable() {
|
||
continue
|
||
}
|
||
var want string
|
||
for _, n := range c.Notes {
|
||
if n.ID == c.Want {
|
||
want = n.Text
|
||
}
|
||
}
|
||
q := words(c.Query)
|
||
if len(q) == 0 {
|
||
continue
|
||
}
|
||
inNote := map[string]bool{}
|
||
for _, w := range words(want) {
|
||
inNote[w] = true
|
||
}
|
||
shared := 0
|
||
for _, w := range q {
|
||
if inNote[w] {
|
||
shared++
|
||
}
|
||
}
|
||
if frac := float64(shared) / float64(len(q)); frac > 0.5 {
|
||
t.Errorf("%s: query shares %.0f%% of its words with the note — not a paraphrase\n query: %q\n note: %q",
|
||
c.ID, 100*frac, c.Query, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// words — lowercased words of two runes or more, matching how the hash
|
||
// embedder tokenizes.
|
||
func words(s string) []string {
|
||
var out []string
|
||
for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||
}) {
|
||
if len([]rune(w)) > 1 {
|
||
out = append(out, w)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// TestBestRecallMatchesDaemon — the harness duplicates bestRecall from
|
||
// cmd/mavend/recall.go (package main is not importable). This pins the copy to
|
||
// the original's rules: no hits, below the gate, no text, or no shared topic
|
||
// word ⇒ silence.
|
||
func TestBestRecallMatchesDaemon(t *testing.T) {
|
||
if got := bestRecall("чай", nil, 0.55, 0); got != "" {
|
||
t.Errorf("no hits: got %q, want silence", got)
|
||
}
|
||
low := []memory.Result{{ID: "a", Score: 0.4, Meta: map[string]string{"text": "чай"}}}
|
||
if got := bestRecall("чай", low, 0.55, 0); got != "" {
|
||
t.Errorf("below gate: got %q, want silence", got)
|
||
}
|
||
noText := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{}}}
|
||
if got := bestRecall("чай", noText, 0.55, 0); got != "" {
|
||
t.Errorf("no text: got %q, want silence", got)
|
||
}
|
||
ok := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{"text": "чай"}}}
|
||
if got := bestRecall("чай", ok, 0.55, 0); got != "чай" {
|
||
t.Errorf("above gate: got %q, want %q", got, "чай")
|
||
}
|
||
// Margin: a close runner-up means the embedder cannot tell the two apart,
|
||
// so Maven stays silent even though both clear the absolute floor.
|
||
close := []memory.Result{
|
||
{ID: "a", Score: 0.86, Meta: map[string]string{"text": "чай"}},
|
||
{ID: "b", Score: 0.85, Meta: map[string]string{"text": "кофе"}},
|
||
}
|
||
if got := bestRecall("чай", close, 0.55, 0.03); got != "" {
|
||
t.Errorf("thin margin: got %q, want silence", got)
|
||
}
|
||
if got := bestRecall("чай", close, 0.55, 0); got != "чай" {
|
||
t.Errorf("margin off: got %q, want %q", got, "чай")
|
||
}
|
||
// The topic veto (#470): the score is fine and the note is about
|
||
// something else.
|
||
offTopic := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{"text": "сеть какая-то медленная"}}}
|
||
if got := bestRecall("почему небо синее", offTopic, 0.55, 0); got != "" {
|
||
t.Errorf("off topic: got %q, want silence", got)
|
||
}
|
||
clear := []memory.Result{
|
||
{ID: "a", Score: 0.86, Meta: map[string]string{"text": "чай"}},
|
||
{ID: "b", Score: 0.70, Meta: map[string]string{"text": "кофе"}},
|
||
}
|
||
if got := bestRecall("чай", clear, 0.55, 0.03); got != "чай" {
|
||
t.Errorf("wide margin: got %q, want %q", got, "чай")
|
||
}
|
||
}
|
||
|
||
// TestHashRecallBaseline — the CI ratchet. HashEmbedder, so it needs no model
|
||
// files and is byte-for-byte reproducible.
|
||
//
|
||
// It is a floor, not a target. The hash embedder is lexical, so most of this
|
||
// fixture is unwinnable for it by construction; the number worth moving is
|
||
// TestONNXRecall's. Never compare a hash-embedder number to an ONNX one.
|
||
func TestHashRecallBaseline(t *testing.T) {
|
||
f, err := Load()
|
||
if err != nil {
|
||
t.Fatalf("Load: %v", err)
|
||
}
|
||
rep, err := Score(context.Background(), "recall+hash", router.NewHashEmbedder(hashDim), InMemory,
|
||
config.DefaultQueryMinScore, config.DefaultQueryMinMargin, f)
|
||
if err != nil {
|
||
t.Fatalf("Score: %v", err)
|
||
}
|
||
t.Log("\n" + rep.String() + rep.Failures())
|
||
t.Log("\ngate sweep:\n" + sweep(t, router.NewHashEmbedder(hashDim), f))
|
||
|
||
// 0.32 sits under the observed 0.370 recall@1 (was 0.360 over 25 answerable
|
||
// cases; the two mixed note+fact cases added with #373 make it 27).
|
||
const floorRecall1 = 0.32
|
||
if rep.Recall1() < floorRecall1 {
|
||
t.Errorf("recall@1 %.3f below ratchet %.2f — note recall regressed", rep.Recall1(), floorRecall1)
|
||
}
|
||
// The dangerous direction, asserted tightly and separately: answering from
|
||
// the wrong note is worse than a gap. Observed 0 under the hash floor.
|
||
if rep.FalseRecall > 1 {
|
||
t.Errorf("%d false recalls, want <= 1:\n%s", rep.FalseRecall, rep.Failures())
|
||
}
|
||
}
|
||
|
||
// TestPersistentStoreScoresTheSame — the deployed store is sqlite-backed
|
||
// (store.MemoryStore via st.VectorMemory()), not the in-memory fallback. Its
|
||
// Search is a separate implementation of the same cosine scan, so it gets its
|
||
// own run: a divergence here would mean recall quality depends on whether a
|
||
// database was configured.
|
||
func TestPersistentStoreScoresTheSame(t *testing.T) {
|
||
f, err := Load()
|
||
if err != nil {
|
||
t.Fatalf("Load: %v", err)
|
||
}
|
||
emb := router.NewHashEmbedder(hashDim)
|
||
inMem, err := Score(context.Background(), "recall+hash+memory", emb, InMemory, config.DefaultQueryMinScore, config.DefaultQueryMinMargin, f)
|
||
if err != nil {
|
||
t.Fatalf("Score in-memory: %v", err)
|
||
}
|
||
persistent, err := Score(context.Background(), "recall+hash+sqlite", emb, sqliteStores(t), config.DefaultQueryMinScore, config.DefaultQueryMinMargin, f)
|
||
if err != nil {
|
||
t.Fatalf("Score sqlite: %v", err)
|
||
}
|
||
t.Log("\n" + persistent.String())
|
||
if persistent.Rank1 != inMem.Rank1 || persistent.FalseRecall != inMem.FalseRecall {
|
||
t.Errorf("sqlite recall@1 %d/%d fr %d, in-memory %d/%d fr %d — the two backends disagree",
|
||
persistent.Rank1, persistent.Answerable, persistent.FalseRecall,
|
||
inMem.Rank1, inMem.Answerable, inMem.FalseRecall)
|
||
}
|
||
}
|
||
|
||
// sqliteStores returns a NewStore that hands each case its own plaintext
|
||
// database file, so cases stay isolated the way they are with InMemory.
|
||
func sqliteStores(t *testing.T) NewStore {
|
||
t.Helper()
|
||
dir := t.TempDir()
|
||
n := 0
|
||
return func() (memory.Store, func(), error) {
|
||
n++
|
||
st, err := store.Open(context.Background(), filepath.Join(dir, fmt.Sprintf("recall-%d.db", n)))
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
return st.VectorMemory(), func() { _ = st.Close() }, nil
|
||
}
|
||
}
|
||
|
||
// TestONNXRecall — the number that matters: the multilingual embedder homesrv
|
||
// actually runs. Opt-in via MAVEN_ONNX_LIB because deps/ is gitignored, exactly
|
||
// like TestONNXBaseline in internal/router/eval. `make eval-recall` points it at
|
||
// the vendored runtime.
|
||
//
|
||
// Reports rather than asserts. The gate sweep is the point: it prints
|
||
// answered-vs-false-recall at a range of query_min_score values, so the right
|
||
// threshold is read off data instead of guessed.
|
||
func TestONNXRecall(t *testing.T) {
|
||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||
if lib == "" {
|
||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||
}
|
||
model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx")
|
||
tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json")
|
||
for _, p := range []string{lib, model, tok} {
|
||
if _, err := os.Stat(p); err != nil {
|
||
t.Skipf("missing %s: %v", p, err)
|
||
}
|
||
}
|
||
emb, err := router.NewONNXEmbedder(model, tok, lib)
|
||
if err != nil {
|
||
t.Skipf("onnx embedder unavailable: %v", err)
|
||
}
|
||
defer emb.Close()
|
||
|
||
f, err := Load()
|
||
if err != nil {
|
||
t.Fatalf("Load: %v", err)
|
||
}
|
||
rep, err := Score(context.Background(), "recall+onnx", emb, InMemory, config.DefaultQueryMinScore, config.DefaultQueryMinMargin, f)
|
||
if err != nil {
|
||
t.Fatalf("Score: %v", err)
|
||
}
|
||
t.Log("\n" + rep.String() + rep.Failures())
|
||
// Cached for the sweeps only: the headline run above must pay the real
|
||
// embedder cost so its latency numbers mean something.
|
||
cached := Cache(emb)
|
||
t.Log("\ngate sweep (margin off):\n" + sweep(t, cached, f))
|
||
t.Log("\nmargin sweep (gate 0.55):\n" + marginSweep(t, cached, f))
|
||
}
|
||
|
||
// sweep scores the fixture at a range of gates and renders one line each. Two
|
||
// columns matter: how many real questions get answered, and how many made-up
|
||
// ones get answered anyway. A gate is only defensible if some value keeps the
|
||
// first high and the second at zero.
|
||
func sweep(t *testing.T, emb router.Embedder, f Fixture) string {
|
||
t.Helper()
|
||
var b strings.Builder
|
||
for _, gate := range []float64{0.0, 0.30, 0.40, 0.50, 0.55, 0.60, 0.70, 0.80, 0.90} {
|
||
rep, err := Score(context.Background(), "sweep", emb, InMemory, gate, 0, f)
|
||
if err != nil {
|
||
t.Fatalf("sweep at %.2f: %v", gate, err)
|
||
}
|
||
fmt.Fprintf(&b, " gate %.2f: answered %d/%d (%.0f%%) false recall %d/%d\n",
|
||
gate, rep.Rank1-rep.Gated, rep.Answerable, 100*rep.Answered(), rep.FalseRecall, rep.NoAnswer)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// marginSweep is the same idea for the margin gate (top1 − top2 > delta), with
|
||
// the absolute gate held at its default. The absolute score cannot separate a
|
||
// real hit from a made-up question under e5 — every score lands in one narrow
|
||
// band — so this sweep is the one that picks a number.
|
||
func marginSweep(t *testing.T, emb router.Embedder, f Fixture) string {
|
||
t.Helper()
|
||
var b strings.Builder
|
||
for _, d := range []float64{0, 0.002, 0.005, 0.008, 0.01, 0.012, 0.015, 0.02, 0.025, 0.03, 0.04, 0.05, 0.06} {
|
||
rep, err := Score(context.Background(), "margin sweep", emb, InMemory, config.DefaultQueryMinScore, d, f)
|
||
if err != nil {
|
||
t.Fatalf("margin sweep at %.3f: %v", d, err)
|
||
}
|
||
fmt.Fprintf(&b, " delta %.3f: answered %d/%d (%.0f%%) false recall %d/%d\n",
|
||
d, rep.Rank1-rep.Gated, rep.Answerable, 100*rep.Answered(), rep.FalseRecall, rep.NoAnswer)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// TestFillerIDCollisionIsRefused — the guard that keeps a fixture edit from
|
||
// looking like a backend difference (Vikunja #386).
|
||
func TestFillerIDCollisionIsRefused(t *testing.T) {
|
||
f := Fixture{
|
||
SchemaVersion: SchemaVersion,
|
||
Cases: []Case{{ID: "ru-001", Notes: []StoredNote{{ID: "f1", Text: "..."}}}},
|
||
Filler: []StoredNote{{ID: "f1", Text: "..."}},
|
||
}
|
||
if err := checkIDs(f); err == nil {
|
||
t.Fatal("a case note reusing a filler id must be refused")
|
||
}
|
||
f.Filler = append(f.Filler, StoredNote{ID: "f1", Text: "..."})
|
||
if err := checkIDs(Fixture{SchemaVersion: SchemaVersion, Filler: f.Filler}); err == nil {
|
||
t.Fatal("a duplicate filler id must be refused")
|
||
}
|
||
ok := Fixture{
|
||
SchemaVersion: SchemaVersion,
|
||
Cases: []Case{{ID: "ru-001", Notes: []StoredNote{{ID: "n1", Text: "..."}}}},
|
||
Filler: []StoredNote{{ID: "f1", Text: "..."}},
|
||
}
|
||
if err := checkIDs(ok); err != nil {
|
||
t.Fatalf("a clean fixture must pass: %v", err)
|
||
}
|
||
}
|