diff --git a/Makefile b/Makefile index f837391..6c4f8e6 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data -.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go eval-router +.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test run-stt run-tts run-web download-embedder deps-go eval-router eval-recall all: build @@ -83,6 +83,13 @@ MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxrunt eval-router: MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/ +# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval). +# Answers "can she find the note again when it matters": recall@1, recall@3, +# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as +# eval-router; without it only the deterministic hash ratchet runs. +eval-recall: + MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/memory/recalleval/ + run-stt: build-stt LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ ./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL) diff --git a/internal/memory/recalleval/recalleval.go b/internal/memory/recalleval/recalleval.go new file mode 100644 index 0000000..4763d19 --- /dev/null +++ b/internal/memory/recalleval/recalleval.go @@ -0,0 +1,444 @@ +// Package recalleval is the held-out contract for note recall: can Maven find +// the right note again when the user asks for it weeks later? +// +// Why it sits beside internal/memory rather than inside it: the thing under +// test is a whole path, not one function — an embedder (internal/router), a +// vector store (internal/memory or internal/store) and the confidence gate the +// daemon applies on top (cmd/mavend/recall.go's bestRecall, config's +// query_min_score). A _test.go file inside internal/memory could not reach the +// persistent store without an import cycle, and testdata is not reachable from +// another package's working directory — so the fixture is embedded here and the +// scorer takes the store as a factory. Same layout and same reasons as +// internal/router/eval. +// +// The fixture is HELD OUT the same way the routing fixture is: a query never +// repeats its note's wording verbatim beyond ordinary shared vocabulary, and +// TestFixtureIsParaphrased enforces a floor on how little the two overlap. +// Scoring recall on a query that is a copy of the note measures string +// matching, not recall. +package recalleval + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/kami/maven/internal/memory" + "github.com/kami/maven/internal/router" +) + +//go:embed ru_recall_v1.json +var fixtureJSON []byte + +// SchemaVersion — the version this package understands. The loader refuses any +// other version rather than misreading a fixture and reporting a number. +const SchemaVersion = 1 + +// StoredNote — one thing the user said once, as it lands in the semantic store. +// Kind is "note" or "fact"; both share the vector index (see +// cmd/mavend/recall.go), so a fact can legitimately win a recall. +type StoredNote struct { + ID string `json:"id"` + Text string `json:"text"` + Kind string `json:"kind"` +} + +// Case — a small set of notes, one query, and the note that must come back +// first. Want is empty exactly when the query should recall NOTHING: that lane +// measures false recall, which is the direction the spec cares about ("a +// confident wrong fact is worse than a known gap"). +type Case struct { + ID string `json:"id"` + Lang string `json:"lang"` + Notes []StoredNote `json:"notes"` + Query string `json:"query"` + Want string `json:"want"` + Tags []string `json:"tags"` + Note string `json:"note"` +} + +// Answerable reports whether the case expects a recall at all. +func (c Case) Answerable() bool { return c.Want != "" } + +// Fixture — the versioned envelope, same shape as the routing fixture. +// +// Filler is inserted into EVERY case's store on top of that case's own notes. +// Without it a case with three notes scores recall@3 = 100% by construction, +// which measures nothing. A real store holds months of unrelated notes, and the +// wanted note has to beat all of them. +type Fixture struct { + SchemaVersion int `json:"schema_version"` + Name string `json:"name"` + Notes []string `json:"notes"` + Filler []StoredNote `json:"filler"` + Cases []Case `json:"cases"` +} + +// Load returns the embedded fixture. +func Load() (Fixture, error) { + var f Fixture + if err := json.Unmarshal(fixtureJSON, &f); err != nil { + return Fixture{}, fmt.Errorf("parse fixture: %w", err) + } + if f.SchemaVersion != SchemaVersion { + return Fixture{}, fmt.Errorf("fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion) + } + if len(f.Cases) == 0 { + return Fixture{}, fmt.Errorf("fixture has no cases") + } + return f, nil +} + +// NewStore builds an empty store for one case, plus a function to release it. +// A factory rather than a store because every case needs a clean index — notes +// from case A must not be visible to case B's query. +type NewStore func() (memory.Store, func(), error) + +// InMemory is the NewStore for memory.InMemoryStore — the fallback the daemon +// uses when there is no database (cmd/mavend/voice.go:224). +func InMemory() (memory.Store, func(), error) { + return memory.NewInMemoryStore(), func() {}, nil +} + +// Outcome — one scored case. +type Outcome struct { + Case Case + Hits []memory.Result + Err error + // Latency is the read path only: embed the query, then Search. Insert time + // is excluded because it happens once, weeks earlier. + Latency time.Duration + // Rank1/Rank3 — the wanted note came back first / in the top three, + // ignoring the confidence gate. Ranking is the store's job. + Rank1 bool + Rank3 bool + // Recalled — what the daemon would actually say back: the top hit's text + // when it clears the gate. Mirrors bestRecall in cmd/mavend/recall.go. + Recalled string + // Pass — the wanted note was recalled AND survived the gate; or, for a + // no-answer case, nothing was recalled. + Pass bool + // Tied — the wanted note is on top but shares its score with the next hit, + // so the sort decided it, not the embedder. Counted apart from a real hit. + Tied bool + TopID string + TopScor float64 + Reasons []string +} + +// Report — the aggregate. Rank and gate are kept apart on purpose: a note that +// ranks first but is silenced by query_min_score is a threshold problem, and a +// note that never ranks first is an embedder problem. Those are different fixes. +type Report struct { + Name string + MinScore float64 + Total int + Answerable int + Rank1 int + Rank3 int + // Gated — ranked first but the score was under MinScore, so the daemon + // stays silent and answers "не знаю". + Gated int + // WrongTop — a different note outranked the right one. + WrongTop int + // Tied — the right note was on top only because of sort order. Not credited + // as recall; tracked because it is a distinct failure (the embedder scored + // the query and the note the same as everything else). + Tied int + // NoAnswer / FalseRecall — the cases that must recall nothing, and how many + // of them the daemon would answer anyway. + NoAnswer int + FalseRecall int + Errors int + Passed int + Outcomes []Outcome + ByTag map[string]TagStat + ByLang map[string]TagStat + // CorrectTop / NoAnswerTop — sorted top-1 scores for the answerable cases + // where the right note ranked first, and for the no-answer cases. The gap + // between these two distributions is what a defensible query_min_score + // would have to sit inside; if they overlap, no threshold separates them. + CorrectTop []float64 + NoAnswerTop []float64 + P50, P95, Max time.Duration +} + +// TagStat — passed/total for one slice of the fixture. +type TagStat struct{ Passed, Total int } + +// Recall1 — fraction of answerable cases whose wanted note ranked first. +func (r Report) Recall1() float64 { return ratio(r.Rank1, r.Answerable) } + +// Recall3 — same, in the top three. The daemon asks for 3 (voice.go), so this +// is the ceiling a better gate or a reranker could reach. +func (r Report) Recall3() float64 { return ratio(r.Rank3, r.Answerable) } + +// Answered — fraction of answerable cases the daemon would actually answer +// correctly, gate included. This is the number the operator experiences. +func (r Report) Answered() float64 { return ratio(r.Rank1-r.Gated, r.Answerable) } + +// FalseRecallRate — fraction of the no-answer cases the daemon answers anyway. +func (r Report) FalseRecallRate() float64 { return ratio(r.FalseRecall, r.NoAnswer) } + +func ratio(n, d int) float64 { + if d == 0 { + return 0 + } + return float64(n) / float64(d) +} + +// Score runs every case against a fresh store and aggregates. It never fails +// the run on an embed or search error: an erroring case scores as a miss and is +// counted in Errors, because "the embedder was down" and "the embedder was +// wrong" are different numbers. +func Score(ctx context.Context, name string, emb router.Embedder, newStore NewStore, minScore float64, f Fixture) (Report, error) { + rep := Report{ + Name: name, + MinScore: minScore, + Total: len(f.Cases), + ByTag: map[string]TagStat{}, + ByLang: map[string]TagStat{}, + } + lat := make([]time.Duration, 0, len(f.Cases)) + + for _, c := range f.Cases { + if c.Answerable() { + rep.Answerable++ + } else { + rep.NoAnswer++ + } + o, err := scoreCase(ctx, emb, newStore, minScore, c, f.Filler) + if err != nil { + return Report{}, err + } + lat = append(lat, o.Latency) + + switch { + case o.Err != nil: + rep.Errors++ + case c.Answerable(): + if o.Rank1 { + rep.Rank1++ + } + if o.Rank3 { + rep.Rank3++ + } + if o.Rank1 && o.Recalled == "" { + rep.Gated++ + } + if o.Tied { + rep.Tied++ + } else if !o.Rank1 { + rep.WrongTop++ + } + if o.Rank1 && o.Recalled != "" { + rep.CorrectTop = append(rep.CorrectTop, o.TopScor) + } + default: + if o.Recalled != "" { + rep.FalseRecall++ + } + rep.NoAnswerTop = append(rep.NoAnswerTop, o.TopScor) + } + + if o.Pass { + rep.Passed++ + } + bump(rep.ByLang, c.Lang, o.Pass) + for _, tag := range c.Tags { + bump(rep.ByTag, tag, o.Pass) + } + rep.Outcomes = append(rep.Outcomes, o) + } + + sort.Float64s(rep.CorrectTop) + sort.Float64s(rep.NoAnswerTop) + sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] }) + rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95) + if len(lat) > 0 { + rep.Max = lat[len(lat)-1] + } + return rep, nil +} + +// scoreCase inserts the case's notes into a fresh store, then runs the read +// path the daemon runs. The returned error is fatal (the harness is broken); +// an embedder or store failure on the query lands in Outcome.Err instead. +func scoreCase(ctx context.Context, emb router.Embedder, newStore NewStore, minScore float64, c Case, filler []StoredNote) (Outcome, error) { + st, release, err := newStore() + if err != nil { + return Outcome{}, fmt.Errorf("%s: new store: %w", c.ID, err) + } + defer release() + + all := append(append([]StoredNote(nil), c.Notes...), filler...) + for _, n := range all { + vec, err := emb.Embed(ctx, n.Text) + if err != nil { + return Outcome{}, fmt.Errorf("%s: embed note %s: %w", c.ID, n.ID, err) + } + meta := map[string]string{"text": n.Text, "type": n.Kind} + if err := st.Insert(ctx, n.ID, vec, meta); err != nil { + return Outcome{}, fmt.Errorf("%s: insert %s: %w", c.ID, n.ID, err) + } + } + + o := Outcome{Case: c} + start := time.Now() + qvec, err := emb.Embed(ctx, c.Query) + if err != nil { + o.Latency = time.Since(start) + o.Err = err + o.Reasons = []string{fmt.Sprintf("embed query: %v", err)} + return o, nil + } + hits, err := st.Search(ctx, qvec, 3) + o.Latency = time.Since(start) + if err != nil { + o.Err = err + o.Reasons = []string{fmt.Sprintf("search: %v", err)} + return o, nil + } + o.Hits = hits + + if len(hits) > 0 { + o.TopID, o.TopScor = hits[0].ID, hits[0].Score + o.Recalled = bestRecall(hits, minScore) + } + for i, h := range hits { + if h.ID != c.Want { + continue + } + o.Rank3 = true + // A tie is not a hit. With a lexical embedder several notes score + // exactly 0 against a paraphrased query, and whichever one the sort + // happens to leave on top would otherwise be credited as recall. + if i == 0 && (len(hits) < 2 || hits[0].Score > hits[1].Score) { + o.Rank1 = true + } + if i == 0 && !o.Rank1 { + o.Tied = true + } + } + + switch { + case !c.Answerable(): + if o.Recalled != "" { + o.Reasons = append(o.Reasons, fmt.Sprintf("false recall: %q at %.3f, want silence", o.TopID, o.TopScor)) + } + case o.Tied: + o.Reasons = append(o.Reasons, fmt.Sprintf("tie at %.3f — the right note is on top only by sort order", o.TopScor)) + case !o.Rank1: + o.Reasons = append(o.Reasons, fmt.Sprintf("top hit %q (%.3f), want %q%s", o.TopID, o.TopScor, c.Want, rankNote(o.Rank3))) + case o.Recalled == "": + o.Reasons = append(o.Reasons, fmt.Sprintf("right note ranked first but scored %.3f < gate %.2f — daemon says \"не знаю\"", o.TopScor, minScore)) + } + o.Pass = len(o.Reasons) == 0 + return o, nil +} + +func rankNote(inTop3 bool) string { + if inTop3 { + return " (wanted note is in the top 3)" + } + return " (wanted note is not in the top 3)" +} + +// bestRecall mirrors cmd/mavend/recall.go — the gate the daemon actually +// applies to a memory hit. Duplicated rather than imported because package main +// is not importable; recalleval_test.go asserts the two agree in behaviour. +func bestRecall(results []memory.Result, min float64) string { + if len(results) == 0 || results[0].Score < min { + return "" + } + return results[0].Meta["text"] +} + +func bump(m map[string]TagStat, key string, pass bool) { + if key == "" { + return + } + s := m[key] + s.Total++ + if pass { + s.Passed++ + } + m[key] = s +} + +// percentile — nearest-rank on a pre-sorted slice. No interpolation: with ~30 +// samples an interpolated p95 invents a latency no query actually took. +func percentile(sorted []time.Duration, p float64) time.Duration { + if len(sorted) == 0 { + return 0 + } + i := int(p * float64(len(sorted))) + if i >= len(sorted) { + i = len(sorted) - 1 + } + return sorted[i] +} + +// String renders the report in the routing eval's style — headline first, then +// the slices that name where the path is weak. +func (r Report) String() string { + var b strings.Builder + fmt.Fprintf(&b, "%s: %d/%d cases pass (gate %.2f)\n", r.Name, r.Passed, r.Total, r.MinScore) + fmt.Fprintf(&b, " recall@1 %.1f%% (%d/%d) recall@3 %.1f%% (%d/%d) answered after gate %.1f%% (%d/%d)\n", + 100*r.Recall1(), r.Rank1, r.Answerable, + 100*r.Recall3(), r.Rank3, r.Answerable, + 100*r.Answered(), r.Rank1-r.Gated, r.Answerable) + fmt.Fprintf(&b, " wrong note on top: %d | tie on top (sort order, not recall): %d | silenced by gate: %d | errors: %d\n", + r.WrongTop, r.Tied, r.Gated, r.Errors) + fmt.Fprintf(&b, " false recall %.1f%% (%d/%d must-be-silent cases answered anyway)\n", + 100*r.FalseRecallRate(), r.FalseRecall, r.NoAnswer) + fmt.Fprintf(&b, " top-1 score, right note first: %s\n", spread(r.CorrectTop)) + fmt.Fprintf(&b, " top-1 score, must be silent: %s\n", spread(r.NoAnswerTop)) + fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max) + fmt.Fprintf(&b, " by lang: %s\n", renderStats(r.ByLang)) + fmt.Fprintf(&b, " by tag: %s\n", renderStats(r.ByTag)) + return b.String() +} + +// Failures — per-case detail, sorted by ID so two runs diff cleanly. +func (r Report) Failures() string { + var b strings.Builder + out := append([]Outcome(nil), r.Outcomes...) + sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID }) + for _, o := range out { + if o.Pass { + continue + } + fmt.Fprintf(&b, " %s %q: %s\n", o.Case.ID, o.Case.Query, strings.Join(o.Reasons, "; ")) + } + return b.String() +} + +// spread — min / median / max of a sorted score list. Three numbers is enough +// to see whether two distributions overlap, which is the only question a +// threshold can answer. +func spread(sorted []float64) string { + if len(sorted) == 0 { + return "n/a" + } + return fmt.Sprintf("min %.3f median %.3f max %.3f (n=%d)", + sorted[0], sorted[len(sorted)/2], sorted[len(sorted)-1], len(sorted)) +} + +func renderStats(m map[string]TagStat) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + s := m[k] + parts = append(parts, fmt.Sprintf("%s %d/%d", k, s.Passed, s.Total)) + } + return strings.Join(parts, " ") +} diff --git a/internal/memory/recalleval/recalleval_test.go b/internal/memory/recalleval/recalleval_test.go new file mode 100644 index 0000000..04dc361 --- /dev/null +++ b/internal/memory/recalleval/recalleval_test.go @@ -0,0 +1,290 @@ +package recalleval + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "unicode" + + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/memory" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" +) + +// dim 1024 for the hash embedder: it is bag-of-words, so a narrower space +// collides tokens between unrelated notes and would measure the hash. +const hashDim = 1024 + +func TestLoadFixture(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(f.Cases) < 25 { + t.Errorf("%d cases, want >= 25", len(f.Cases)) + } + // Filler is what stops recall@3 being free: three case notes and a top-3 + // search would put the wanted note in the top 3 every time. + if len(f.Filler) < 10 { + t.Errorf("%d filler notes, want >= 10", len(f.Filler)) + } + seen := map[string]bool{} + silent, en := 0, 0 + for _, c := range f.Cases { + if c.ID == "" || seen[c.ID] { + t.Errorf("case %q: empty or duplicate id", c.ID) + } + seen[c.ID] = true + if c.Lang != "ru" && c.Lang != "en" { + t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang) + } + if c.Lang == "en" { + en++ + } + if strings.TrimSpace(c.Query) == "" { + t.Errorf("%s: empty query", c.ID) + } + // Fewer than three notes and a wrong answer has nowhere to come from, + // so recall@1 would be near-free. + if len(c.Notes) < 3 { + t.Errorf("%s: %d notes, want >= 3", c.ID, len(c.Notes)) + } + ids := map[string]bool{} + for _, n := range c.Notes { + if n.ID == "" || ids[n.ID] { + t.Errorf("%s: note %q empty or duplicate id", c.ID, n.ID) + } + ids[n.ID] = true + if strings.TrimSpace(n.Text) == "" { + t.Errorf("%s: note %q empty text", c.ID, n.ID) + } + if n.Kind != "note" && n.Kind != "fact" { + t.Errorf("%s: note %q kind %q, want note|fact", c.ID, n.ID, n.Kind) + } + } + if !c.Answerable() { + silent++ + continue + } + if !ids[c.Want] { + t.Errorf("%s: want %q is not one of the case's notes", c.ID, c.Want) + } + } + // Both lanes need enough cases that a rate means something. + if silent < 5 { + t.Errorf("%d must-be-silent cases, want >= 5", silent) + } + if en < 5 { + t.Errorf("%d English cases, want >= 5", en) + } +} + +// TestFixtureIsParaphrased — the fixture's claim to measuring recall at all. If +// a query repeats its note's words, cosine over a bag-of-words embedder gets it +// for free and the score says nothing about semantic recall. Half the query's +// words is the line: some shared vocabulary is natural ("nginx", "чай"), a copy +// is the failure. +func TestFixtureIsParaphrased(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + for _, c := range f.Cases { + if !c.Answerable() { + continue + } + var want string + for _, n := range c.Notes { + if n.ID == c.Want { + want = n.Text + } + } + q := words(c.Query) + if len(q) == 0 { + continue + } + inNote := map[string]bool{} + for _, w := range words(want) { + inNote[w] = true + } + shared := 0 + for _, w := range q { + if inNote[w] { + shared++ + } + } + if frac := float64(shared) / float64(len(q)); frac > 0.5 { + t.Errorf("%s: query shares %.0f%% of its words with the note — not a paraphrase\n query: %q\n note: %q", + c.ID, 100*frac, c.Query, want) + } + } +} + +// words — lowercased words of two runes or more, matching how the hash +// embedder tokenizes. +func words(s string) []string { + var out []string + for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) { + if len([]rune(w)) > 1 { + out = append(out, w) + } + } + return out +} + +// TestBestRecallMatchesDaemon — the harness duplicates bestRecall from +// cmd/mavend/recall.go (package main is not importable). This pins the copy to +// the original's three rules: no hits, below the gate, or no text ⇒ silence. +func TestBestRecallMatchesDaemon(t *testing.T) { + if got := bestRecall(nil, 0.55); got != "" { + t.Errorf("no hits: got %q, want silence", got) + } + low := []memory.Result{{ID: "a", Score: 0.4, Meta: map[string]string{"text": "чай"}}} + if got := bestRecall(low, 0.55); got != "" { + t.Errorf("below gate: got %q, want silence", got) + } + noText := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{}}} + if got := bestRecall(noText, 0.55); got != "" { + t.Errorf("no text: got %q, want silence", got) + } + ok := []memory.Result{{ID: "a", Score: 0.9, Meta: map[string]string{"text": "чай"}}} + if got := bestRecall(ok, 0.55); got != "чай" { + t.Errorf("above gate: got %q, want %q", got, "чай") + } +} + +// TestHashRecallBaseline — the CI ratchet. HashEmbedder, so it needs no model +// files and is byte-for-byte reproducible. +// +// It is a floor, not a target. The hash embedder is lexical, so most of this +// fixture is unwinnable for it by construction; the number worth moving is +// TestONNXRecall's. Never compare a hash-embedder number to an ONNX one. +func TestHashRecallBaseline(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + rep, err := Score(context.Background(), "recall+hash", router.NewHashEmbedder(hashDim), InMemory, + config.DefaultQueryMinScore, f) + if err != nil { + t.Fatalf("Score: %v", err) + } + t.Log("\n" + rep.String() + rep.Failures()) + t.Log("\ngate sweep:\n" + sweep(t, router.NewHashEmbedder(hashDim), f)) + + // 0.32 sits under the observed 0.360 recall@1. + const floorRecall1 = 0.32 + if rep.Recall1() < floorRecall1 { + t.Errorf("recall@1 %.3f below ratchet %.2f — note recall regressed", rep.Recall1(), floorRecall1) + } + // The dangerous direction, asserted tightly and separately: answering from + // the wrong note is worse than a gap. Observed 0 under the hash floor. + if rep.FalseRecall > 1 { + t.Errorf("%d false recalls, want <= 1:\n%s", rep.FalseRecall, rep.Failures()) + } +} + +// TestPersistentStoreScoresTheSame — the deployed store is sqlite-backed +// (store.MemoryStore via st.VectorMemory()), not the in-memory fallback. Its +// Search is a separate implementation of the same cosine scan, so it gets its +// own run: a divergence here would mean recall quality depends on whether a +// database was configured. +func TestPersistentStoreScoresTheSame(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + emb := router.NewHashEmbedder(hashDim) + inMem, err := Score(context.Background(), "recall+hash+memory", emb, InMemory, config.DefaultQueryMinScore, f) + if err != nil { + t.Fatalf("Score in-memory: %v", err) + } + persistent, err := Score(context.Background(), "recall+hash+sqlite", emb, sqliteStores(t), config.DefaultQueryMinScore, f) + if err != nil { + t.Fatalf("Score sqlite: %v", err) + } + t.Log("\n" + persistent.String()) + if persistent.Rank1 != inMem.Rank1 || persistent.FalseRecall != inMem.FalseRecall { + t.Errorf("sqlite recall@1 %d/%d fr %d, in-memory %d/%d fr %d — the two backends disagree", + persistent.Rank1, persistent.Answerable, persistent.FalseRecall, + inMem.Rank1, inMem.Answerable, inMem.FalseRecall) + } +} + +// sqliteStores returns a NewStore that hands each case its own plaintext +// database file, so cases stay isolated the way they are with InMemory. +func sqliteStores(t *testing.T) NewStore { + t.Helper() + dir := t.TempDir() + n := 0 + return func() (memory.Store, func(), error) { + n++ + st, err := store.Open(context.Background(), filepath.Join(dir, fmt.Sprintf("recall-%d.db", n))) + if err != nil { + return nil, nil, err + } + return st.VectorMemory(), func() { _ = st.Close() }, nil + } +} + +// TestONNXRecall — the number that matters: the multilingual embedder homesrv +// actually runs. Opt-in via MAVEN_ONNX_LIB because deps/ is gitignored, exactly +// like TestONNXBaseline in internal/router/eval. `make eval-recall` points it at +// the vendored runtime. +// +// Reports rather than asserts. The gate sweep is the point: it prints +// answered-vs-false-recall at a range of query_min_score values, so the right +// threshold is read off data instead of guessed. +func TestONNXRecall(t *testing.T) { + lib := os.Getenv("MAVEN_ONNX_LIB") + if lib == "" { + t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") + } + model := filepath.Join("../../..", "models/embedder/model.onnx") + tok := filepath.Join("../../..", "models/embedder/tokenizer.json") + for _, p := range []string{lib, model, tok} { + if _, err := os.Stat(p); err != nil { + t.Skipf("missing %s: %v", p, err) + } + } + emb, err := router.NewONNXEmbedder(model, tok, lib) + if err != nil { + t.Skipf("onnx embedder unavailable: %v", err) + } + defer emb.Close() + + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + rep, err := Score(context.Background(), "recall+onnx", emb, InMemory, config.DefaultQueryMinScore, f) + if err != nil { + t.Fatalf("Score: %v", err) + } + t.Log("\n" + rep.String() + rep.Failures()) + t.Log("\ngate sweep:\n" + sweep(t, emb, f)) +} + +// sweep scores the fixture at a range of gates and renders one line each. Two +// columns matter: how many real questions get answered, and how many made-up +// ones get answered anyway. A gate is only defensible if some value keeps the +// first high and the second at zero. +func sweep(t *testing.T, emb router.Embedder, f Fixture) string { + t.Helper() + var b strings.Builder + for _, gate := range []float64{0.0, 0.30, 0.40, 0.50, 0.55, 0.60, 0.70, 0.80, 0.90} { + rep, err := Score(context.Background(), "sweep", emb, InMemory, gate, f) + if err != nil { + t.Fatalf("sweep at %.2f: %v", gate, err) + } + fmt.Fprintf(&b, " gate %.2f: answered %d/%d (%.0f%%) false recall %d/%d\n", + gate, rep.Rank1-rep.Gated, rep.Answerable, 100*rep.Answered(), rep.FalseRecall, rep.NoAnswer) + } + return b.String() +} diff --git a/internal/memory/recalleval/ru_recall_v1.json b/internal/memory/recalleval/ru_recall_v1.json new file mode 100644 index 0000000..fd7c7f4 --- /dev/null +++ b/internal/memory/recalleval/ru_recall_v1.json @@ -0,0 +1,392 @@ +{ + "schema_version": 1, + "name": "ru_recall_v1", + "notes": [ + "Held-out note-recall fixture. Each case is a fresh semantic store: insert every note, embed the query, take the top 3 — the same read path cmd/mavend/voice.go runs for IntentQuery.", + "Queries paraphrase their note on purpose. A query that repeats the note's words measures string matching, not recall. TestFixtureIsParaphrased enforces a ceiling on word overlap.", + "want:\"\" means the query must recall NOTHING. Those cases measure false recall — the direction the spec calls out (a confident wrong fact is worse than a known gap).", + "The distractor tag marks cases where a second note is plausible and only one is right. The hard tag marks cases with little or no shared vocabulary.", + "Content is written for this operator: his preferences, his homelab, things he said once and would expect Maven to remember weeks later." + ], + "filler": [ + {"id": "f1", "text": "в субботу ходил в баню", "kind": "note"}, + {"id": "f2", "text": "купил новые кроссовки сорок третьего размера", "kind": "note"}, + {"id": "f3", "text": "сериал закончился на третьем сезоне", "kind": "note"}, + {"id": "f4", "text": "сосед сверху делает ремонт", "kind": "note"}, + {"id": "f5", "text": "билеты в театр брал заранее", "kind": "note"}, + {"id": "f6", "text": "выучил пару аккордов на гитаре", "kind": "note"}, + {"id": "f7", "text": "записался к стоматологу", "kind": "note"}, + {"id": "f8", "text": "поменял лампочку в коридоре", "kind": "note"}, + {"id": "f9", "text": "погулял вдоль реки", "kind": "note"}, + {"id": "f10", "text": "the balcony door sticks in winter", "kind": "note"}, + {"id": "f11", "text": "the neighbour's dog barks at cyclists", "kind": "note"}, + {"id": "f12", "text": "i finished the book about volcanoes", "kind": "note"} + ], + "cases": [ + { + "id": "ru-pref-001", + "lang": "ru", + "tags": ["preference", "paraphrase"], + "query": "какой кофе мне наливать", + "want": "n1", + "notes": [ + {"id": "n1", "text": "я пью кофе без сахара", "kind": "note"}, + {"id": "n2", "text": "по утрам бегаю в парке", "kind": "note"}, + {"id": "n3", "text": "не люблю громкую музыку", "kind": "note"} + ] + }, + { + "id": "ru-pref-002", + "lang": "ru", + "tags": ["preference", "homelab", "paraphrase", "hard"], + "query": "когда запускать резервное копирование", + "want": "n1", + "note": "The DESIGN.md preference-seam example, phrased as the operator would ask it later.", + "notes": [ + {"id": "n1", "text": "бэкапы лучше делать ночью в три часа", "kind": "note"}, + {"id": "n2", "text": "обновления ставлю по субботам", "kind": "note"}, + {"id": "n3", "text": "логи храню месяц", "kind": "note"} + ] + }, + { + "id": "ru-home-003", + "lang": "ru", + "tags": ["homelab", "paraphrase"], + "query": "что помогло от мерцания монитора", + "want": "n1", + "notes": [ + {"id": "n1", "text": "мерцание экрана прошло после обновления драйвера amdgpu", "kind": "note"}, + {"id": "n2", "text": "вентилятор шумит на полной нагрузке", "kind": "note"}, + {"id": "n3", "text": "поставил новый ssd в ноутбук", "kind": "note"} + ] + }, + { + "id": "ru-home-004", + "lang": "ru", + "tags": ["homelab", "distractor", "hard"], + "query": "адрес домашнего сервера", + "want": "n2", + "note": "Two notes carry an IP. Only one is the server.", + "notes": [ + {"id": "n1", "text": "роутер живёт на 192.168.1.1", "kind": "note"}, + {"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"}, + {"id": "n3", "text": "принтер подключен по usb", "kind": "note"} + ] + }, + { + "id": "ru-home-005", + "lang": "ru", + "tags": ["homelab"], + "query": "где искать настройки nginx", + "want": "n1", + "notes": [ + {"id": "n1", "text": "конфиг nginx лежит в /etc/nginx/sites-enabled", "kind": "note"}, + {"id": "n2", "text": "сертификаты обновляет certbot по расписанию", "kind": "note"}, + {"id": "n3", "text": "порт 8080 занят вебкой", "kind": "note"} + ] + }, + { + "id": "ru-pref-006", + "lang": "ru", + "tags": ["preference", "distractor", "hard"], + "query": "что мне нельзя есть", + "want": "n1", + "note": "The Friday-meat note is a plausible second answer but it is a habit, not a restriction.", + "notes": [ + {"id": "n1", "text": "у меня аллергия на орехи", "kind": "note"}, + {"id": "n2", "text": "не ем мясо по пятницам", "kind": "note"}, + {"id": "n3", "text": "люблю острую еду", "kind": "note"} + ] + }, + { + "id": "ru-pers-007", + "lang": "ru", + "tags": ["distractor", "paraphrase"], + "query": "когда мамин праздник", + "want": "n1", + "notes": [ + {"id": "n1", "text": "день рождения мамы четырнадцатого марта", "kind": "note"}, + {"id": "n2", "text": "у брата день рождения в июле", "kind": "note"}, + {"id": "n3", "text": "годовщина в сентябре", "kind": "note"} + ] + }, + { + "id": "ru-home-008", + "lang": "ru", + "tags": ["homelab", "hard", "paraphrase"], + "query": "чем ускоряется языковая модель", + "want": "n1", + "notes": [ + {"id": "n1", "text": "модель крутится на встройке через vulkan", "kind": "note"}, + {"id": "n2", "text": "whisper работает на процессоре", "kind": "note"}, + {"id": "n3", "text": "голос у piper русский", "kind": "note"} + ] + }, + { + "id": "ru-pref-009", + "lang": "ru", + "tags": ["preference", "distractor"], + "query": "во сколько я обычно засыпаю", + "want": "n1", + "notes": [ + {"id": "n1", "text": "ложусь спать около часа ночи", "kind": "note"}, + {"id": "n2", "text": "встаю в семь утра", "kind": "note"}, + {"id": "n3", "text": "днём не сплю", "kind": "note"} + ] + }, + { + "id": "ru-home-010", + "lang": "ru", + "tags": ["homelab", "paraphrase"], + "query": "где у меня хранятся пароли", + "want": "n1", + "notes": [ + {"id": "n1", "text": "пароли держу в keepassxc", "kind": "note"}, + {"id": "n2", "text": "двухфакторку сделал через totp", "kind": "note"}, + {"id": "n3", "text": "ssh ключи лежат на юбикее", "kind": "note"} + ] + }, + { + "id": "ru-home-011", + "lang": "ru", + "tags": ["homelab", "hard", "paraphrase"], + "query": "из-за чего кончилось место", + "want": "n1", + "notes": [ + {"id": "n1", "text": "диск забился логами докера в июне", "kind": "note"}, + {"id": "n2", "text": "рейд собрал из двух дисков", "kind": "note"}, + {"id": "n3", "text": "бэкап на внешний диск раз в неделю", "kind": "note"} + ] + }, + { + "id": "ru-pref-012", + "lang": "ru", + "tags": ["preference", "distractor"], + "query": "какой чай мне нравится", + "want": "n1", + "notes": [ + {"id": "n1", "text": "чай пью только зелёный", "kind": "note"}, + {"id": "n2", "text": "кофе пью без сахара", "kind": "note"}, + {"id": "n3", "text": "воду пью из фильтра", "kind": "note"} + ] + }, + { + "id": "ru-silent-013", + "lang": "ru", + "tags": ["silent"], + "query": "какая погода будет в пятницу", + "want": "", + "notes": [ + {"id": "n1", "text": "роутер живёт на 192.168.1.1", "kind": "note"}, + {"id": "n2", "text": "бэкапы лучше делать ночью", "kind": "note"}, + {"id": "n3", "text": "у меня аллергия на орехи", "kind": "note"} + ] + }, + { + "id": "ru-silent-014", + "lang": "ru", + "tags": ["silent"], + "query": "как зовут сестру моего коллеги", + "want": "", + "notes": [ + {"id": "n1", "text": "конфиг nginx лежит в /etc/nginx/sites-enabled", "kind": "note"}, + {"id": "n2", "text": "порт 8080 занят вебкой", "kind": "note"}, + {"id": "n3", "text": "сертификаты обновляет certbot", "kind": "note"} + ] + }, + { + "id": "ru-silent-015", + "lang": "ru", + "tags": ["silent"], + "query": "сколько я заплатил за машину", + "want": "", + "notes": [ + {"id": "n1", "text": "чай пью только зелёный", "kind": "note"}, + {"id": "n2", "text": "ложусь спать около часа ночи", "kind": "note"}, + {"id": "n3", "text": "не люблю громкую музыку", "kind": "note"} + ] + }, + { + "id": "ru-home-016", + "lang": "ru", + "tags": ["homelab", "paraphrase"], + "query": "откуда берётся токен бота", + "want": "n1", + "notes": [ + {"id": "n1", "text": "токен телеграма лежит в deploy/telegram.env", "kind": "note"}, + {"id": "n2", "text": "вебхуки не использую, только long-poll", "kind": "note"}, + {"id": "n3", "text": "уведомления приходят в личку", "kind": "note"} + ] + }, + { + "id": "ru-hard-017", + "lang": "ru", + "tags": ["hard", "paraphrase", "homelab"], + "query": "как я восстановил конфиги", + "want": "n1", + "note": "No shared word between query and note beyond none at all. This is the case a lexical embedder cannot win.", + "notes": [ + {"id": "n1", "text": "после переустановки системы вернул все настройки из git", "kind": "note"}, + {"id": "n2", "text": "разделы на диске резал вручную", "kind": "note"}, + {"id": "n3", "text": "загрузчик поставил заново", "kind": "note"} + ] + }, + { + "id": "ru-dist-018", + "lang": "ru", + "tags": ["distractor"], + "query": "чем кормить кота", + "want": "n1", + "notes": [ + {"id": "n1", "text": "кот ест только сухой корм", "kind": "note"}, + {"id": "n2", "text": "собаке даю мясо", "kind": "note"}, + {"id": "n3", "text": "рыбок кормлю раз в день", "kind": "note"} + ] + }, + { + "id": "ru-home-019", + "lang": "ru", + "tags": ["homelab", "hard", "paraphrase"], + "query": "как контейнер получает доступ к видеокарте", + "want": "n1", + "notes": [ + {"id": "n1", "text": "docker compose пробрасывает /dev/dri внутрь", "kind": "note"}, + {"id": "n2", "text": "контейнеры рестартуют сами", "kind": "note"}, + {"id": "n3", "text": "образы чищу вручную", "kind": "note"} + ] + }, + { + "id": "ru-pref-020", + "lang": "ru", + "tags": ["preference", "distractor"], + "query": "когда мне нельзя звонить", + "want": "n1", + "notes": [ + {"id": "n1", "text": "не звони мне после десяти вечера", "kind": "note"}, + {"id": "n2", "text": "утром не трогай меня до кофе", "kind": "note"}, + {"id": "n3", "text": "по выходным не работаю", "kind": "note"} + ] + }, + { + "id": "en-pref-021", + "lang": "en", + "tags": ["preference", "paraphrase"], + "query": "which colour scheme do i like", + "want": "n1", + "notes": [ + {"id": "n1", "text": "i prefer dark theme everywhere", "kind": "note"}, + {"id": "n2", "text": "font size 14 is fine", "kind": "note"}, + {"id": "n3", "text": "i use vim keybindings", "kind": "note"} + ] + }, + { + "id": "en-home-022", + "lang": "en", + "tags": ["homelab", "distractor"], + "query": "where is the big disk mounted", + "want": "n1", + "notes": [ + {"id": "n1", "text": "the nas drive is mounted at /mnt/hdd1", "kind": "note"}, + {"id": "n2", "text": "models live on the ssd", "kind": "note"}, + {"id": "n3", "text": "backups go to the nas nightly", "kind": "note"} + ] + }, + { + "id": "en-silent-023", + "lang": "en", + "tags": ["silent"], + "query": "what is my bank account number", + "want": "", + "notes": [ + {"id": "n1", "text": "the nas drive is mounted at /mnt/hdd1", "kind": "note"}, + {"id": "n2", "text": "i prefer dark theme everywhere", "kind": "note"}, + {"id": "n3", "text": "the router runs openwrt", "kind": "note"} + ] + }, + { + "id": "en-hard-024", + "lang": "en", + "tags": ["hard", "paraphrase"], + "query": "what fixed the screen problem", + "want": "n1", + "notes": [ + {"id": "n1", "text": "the flicker went away once i swapped the display cable", "kind": "note"}, + {"id": "n2", "text": "the laptop fan is loud", "kind": "note"}, + {"id": "n3", "text": "the second monitor is 1440p", "kind": "note"} + ] + }, + { + "id": "en-pref-025", + "lang": "en", + "tags": ["preference", "hard"], + "query": "should i be offered wine", + "want": "n1", + "notes": [ + {"id": "n1", "text": "i do not drink alcohol", "kind": "note"}, + {"id": "n2", "text": "i skip breakfast", "kind": "note"}, + {"id": "n3", "text": "i like spicy food", "kind": "note"} + ] + }, + { + "id": "ru-home-026", + "lang": "ru", + "tags": ["homelab", "paraphrase"], + "query": "какая модель распознавания речи мне подходит", + "want": "n1", + "notes": [ + {"id": "n1", "text": "whisper модель small хватает для русского", "kind": "note"}, + {"id": "n2", "text": "голос ирина звучит лучше остальных", "kind": "note"}, + {"id": "n3", "text": "слово активации маven", "kind": "note"} + ] + }, + { + "id": "ru-fact-027", + "lang": "ru", + "tags": ["distractor", "hard", "paraphrase"], + "query": "когда я последний раз обслуживал машину", + "want": "n1", + "note": "A fact, not a note — both share the vector index, so a fact can win a recall.", + "notes": [ + {"id": "n1", "text": "последний раз менял масло в мае", "kind": "fact"}, + {"id": "n2", "text": "шины поменял осенью", "kind": "fact"}, + {"id": "n3", "text": "страховка до декабря", "kind": "fact"} + ] + }, + { + "id": "ru-pref-028", + "lang": "ru", + "tags": ["preference", "hard", "paraphrase"], + "query": "как мне присылать оповещения", + "want": "n1", + "notes": [ + {"id": "n1", "text": "терпеть не могу уведомления со звуком", "kind": "note"}, + {"id": "n2", "text": "вибрацию оставь включённой", "kind": "note"}, + {"id": "n3", "text": "письма читаю вечером", "kind": "note"} + ] + }, + { + "id": "ru-silent-029", + "lang": "ru", + "tags": ["silent"], + "query": "во сколько отходит поезд", + "want": "", + "notes": [ + {"id": "n1", "text": "кот ест только сухой корм", "kind": "note"}, + {"id": "n2", "text": "люблю острую еду", "kind": "note"}, + {"id": "n3", "text": "пароли держу в keepassxc", "kind": "note"} + ] + }, + { + "id": "en-home-030", + "lang": "en", + "tags": ["homelab", "paraphrase"], + "query": "what firmware is on the router", + "want": "n1", + "notes": [ + {"id": "n1", "text": "the router runs openwrt", "kind": "note"}, + {"id": "n2", "text": "wifi channel is 6", "kind": "note"}, + {"id": "n3", "text": "the guest network is off", "kind": "note"} + ] + } + ] +}