From 751c2a705fbff16161ec14b51a251007852813c3 Mon Sep 17 00:00:00 2001 From: kami Date: Fri, 31 Jul 2026 11:38:07 +0400 Subject: [PATCH 1/3] Embed a question and a stored note differently (Vikunja #371) Note recall is asymmetric: a short question goes in, a longer note comes out. Adds EmbedQuery/EmbedPassage helpers and the e5 prefixes, and points the note/fact write path at the passage side and the query path at the query side. Reviewers: the three call sites in voice.go. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ --- cmd/mavend/voice.go | 6 ++-- internal/router/embedder.go | 31 +++++++++++++++++ internal/router/embedder_test.go | 57 ++++++++++++++++++++++++++++++++ internal/router/onnxembedder.go | 29 +++++++++++++++- 4 files changed, 119 insertions(+), 4 deletions(-) diff --git a/cmd/mavend/voice.go b/cmd/mavend/voice.go index 9bbd226..df318f4 100644 --- a/cmd/mavend/voice.go +++ b/cmd/mavend/voice.go @@ -567,7 +567,7 @@ func (h *reactiveHandler) applyAction(ctx context.Context, dec router.Decision) // fail the fact write). Facts aren't in the notes table, so this is the // only recall path for them — "когда я пил воду?" reads back from here. if h.memStore != nil { - if vec, err := h.embedder.Embed(ctx, dec.Utterance); err != nil { + if vec, err := router.EmbedPassage(ctx, h.embedder, dec.Utterance); err != nil { log.Printf("voice: embed fact for memory: %v", err) } else if err := h.memStore.Insert(ctx, "fact:"+dec.Slots.Key+":"+strconv.FormatInt(now.Unix(), 10), vec, map[string]string{ "source": "voice", @@ -681,7 +681,7 @@ func (h *reactiveHandler) applyAction(ctx context.Context, dec router.Decision) // embed the note text with the same model the classifier uses, persist // via CoreAPI (source=tap:voice). Semantic recall lives in `notes`, not // facts — no predicate reads it (spec's two-memory split). - vec, err := h.embedder.Embed(ctx, dec.Utterance) + vec, err := router.EmbedPassage(ctx, h.embedder, dec.Utterance) if err != nil { log.Printf("voice: embed note: %v", err) return "не получилось сохранить заметку." @@ -758,7 +758,7 @@ func (h *reactiveHandler) applyAction(ctx context.Context, dec router.Decision) return fmt.Sprintf("в %s сейчас %.0f градусов, %s.", w.Location, w.Temperature, w.Condition) } - vec, err := h.embedder.Embed(ctx, dec.Utterance) + vec, err := router.EmbedQuery(ctx, h.embedder, dec.Utterance) if err != nil { log.Printf("voice: embed query: %v", err) return "не получилось найти ответ." diff --git a/internal/router/embedder.go b/internal/router/embedder.go index e9c0de1..a69f535 100644 --- a/internal/router/embedder.go +++ b/internal/router/embedder.go @@ -19,6 +19,37 @@ type Embedder interface { Close() error } +// AsymmetricEmbedder — an embedder that wants to know whether a text is a +// search query or a stored passage. Recall is asymmetric: a short question +// goes in, a longer note comes out. The e5 family is trained for exactly that +// and needs the side written into the text ("query: " / "passage: "). +// +// Optional on purpose: HashEmbedder has no such notion, so callers go through +// EmbedQuery and EmbedPassage below, which fall back to plain Embed. +type AsymmetricEmbedder interface { + Embedder + EmbedQuery(ctx context.Context, text string) ([]float32, error) + EmbedPassage(ctx context.Context, text string) ([]float32, error) +} + +// EmbedQuery embeds text that is being searched WITH — a question. +func EmbedQuery(ctx context.Context, e Embedder, text string) ([]float32, error) { + if a, ok := e.(AsymmetricEmbedder); ok { + return a.EmbedQuery(ctx, text) + } + return e.Embed(ctx, text) +} + +// EmbedPassage embeds text that is being searched FOR — a note or a fact on +// its way into the store. Store and lookup must use these two calls, not one +// of them twice, or the asymmetry buys nothing. +func EmbedPassage(ctx context.Context, e Embedder, text string) ([]float32, error) { + if a, ok := e.(AsymmetricEmbedder); ok { + return a.EmbedPassage(ctx, text) + } + return e.Embed(ctx, text) +} + // HashEmbedder — a deterministic bag-of-words embedder used for tests and as a // non-zero default floor. NOT semantically meaningful across languages; the // real classifier swaps in the multilingual ONNX model wholesale. diff --git a/internal/router/embedder_test.go b/internal/router/embedder_test.go index 5f815b1..f4ef243 100644 --- a/internal/router/embedder_test.go +++ b/internal/router/embedder_test.go @@ -37,3 +37,60 @@ func TestHashEmbedderCyrillic(t *testing.T) { t.Fatalf("cosine(shared)=%.3f not > cosine(disjoint)=%.3f", cosine(a, b), cosine(a, c)) } } + +// recordingEmbedder — an asymmetric embedder that only remembers which side +// was asked for. Enough to pin the dispatch; real vectors need the model. +type recordingEmbedder struct{ calls []string } + +func (r *recordingEmbedder) Dim() int { return 2 } +func (r *recordingEmbedder) Close() error { return nil } + +func (r *recordingEmbedder) Embed(_ context.Context, _ string) ([]float32, error) { + r.calls = append(r.calls, "embed") + return []float32{1, 0}, nil +} + +func (r *recordingEmbedder) EmbedQuery(_ context.Context, _ string) ([]float32, error) { + r.calls = append(r.calls, "query") + return []float32{1, 0}, nil +} + +func (r *recordingEmbedder) EmbedPassage(_ context.Context, _ string) ([]float32, error) { + r.calls = append(r.calls, "passage") + return []float32{0, 1}, nil +} + +// TestEmbedQueryAndPassageSplit — a question and a stored note must not take +// the same path. If both ended up on the same call the asymmetric model buys +// nothing, which is the whole reason for the swap. +func TestEmbedQueryAndPassageSplit(t *testing.T) { + rec := &recordingEmbedder{} + if _, err := EmbedQuery(context.Background(), rec, "где логи?"); err != nil { + t.Fatalf("EmbedQuery: %v", err) + } + if _, err := EmbedPassage(context.Background(), rec, "логи в /var/log"); err != nil { + t.Fatalf("EmbedPassage: %v", err) + } + if len(rec.calls) != 2 || rec.calls[0] != "query" || rec.calls[1] != "passage" { + t.Errorf("calls %v, want [query passage]", rec.calls) + } +} + +// TestEmbedFallsBackToPlainEmbed — HashEmbedder has no sides, so both helpers +// must still work and give the same vector. +func TestEmbedFallsBackToPlainEmbed(t *testing.T) { + h := NewHashEmbedder(64) + q, err := EmbedQuery(context.Background(), h, "text") + if err != nil { + t.Fatalf("EmbedQuery: %v", err) + } + p, err := EmbedPassage(context.Background(), h, "text") + if err != nil { + t.Fatalf("EmbedPassage: %v", err) + } + for i := range q { + if q[i] != p[i] { + t.Fatalf("hash embedder gave two different vectors for the same text") + } + } +} diff --git a/internal/router/onnxembedder.go b/internal/router/onnxembedder.go index a7356ee..770790b 100644 --- a/internal/router/onnxembedder.go +++ b/internal/router/onnxembedder.go @@ -12,6 +12,15 @@ import ( "golang.org/x/text/unicode/norm" ) +// The deployed model is multilingual-e5-small. e5 was trained with these two +// words glued to the front of every text, and it scores badly without them — +// they are part of the model, not a style choice. Swapping back to a symmetric +// paraphrase model means dropping them again. +const ( + queryPrefix = "query: " + passagePrefix = "passage: " +) + const ( padTokenID = 1 unkTokenID = 3 @@ -54,7 +63,25 @@ func NewONNXEmbedder(modelPath, tokenizerPath, libPath string) (*onnxEmbedder, e func (e *onnxEmbedder) Dim() int { return embedDim } +// Embed treats the text as a query. The classifier compares one short +// utterance to another short seed phrase, so both sides get the same prefix +// and the comparison stays fair. The recall path must call EmbedQuery and +// EmbedPassage instead. func (e *onnxEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { + return e.embed(ctx, queryPrefix+text) +} + +// EmbedQuery — the question the user just asked. +func (e *onnxEmbedder) EmbedQuery(ctx context.Context, text string) ([]float32, error) { + return e.embed(ctx, queryPrefix+text) +} + +// EmbedPassage — a note or fact being stored, or re-scored at lookup time. +func (e *onnxEmbedder) EmbedPassage(ctx context.Context, text string) ([]float32, error) { + return e.embed(ctx, passagePrefix+text) +} + +func (e *onnxEmbedder) embed(ctx context.Context, text string) ([]float32, error) { inputIDs, attentionMask, _ := e.tokenizer.Encode(text) inputShape := ort.NewShape(1, int64(maxLength)) @@ -313,4 +340,4 @@ func preTokenize(text string) []string { return out } -var _ Embedder = (*onnxEmbedder)(nil) +var _ AsymmetricEmbedder = (*onnxEmbedder)(nil) From f6d5a2a7a48b0c9b8978926ee26e23030befea5c Mon Sep 17 00:00:00 2001 From: kami Date: Fri, 31 Jul 2026 11:38:18 +0400 Subject: [PATCH 2/3] Swap the embedder to multilingual-e5-small (Vikunja #371, #372) The old model was a symmetric paraphrase model, so it scored "do these look alike" instead of "does this note answer this question". Also fixes the file mismatch: the Makefile, the deploy config and both evals now all name the same quantized file, and the quantized one is what gets measured. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ --- AGENTS.md | 13 +++++--- Makefile | 10 ++++-- deploy/mavend.json | 4 +-- internal/memory/recalleval/recalleval.go | 32 ++++++++++++++++--- internal/memory/recalleval/recalleval_test.go | 4 +-- internal/router/eval/eval_test.go | 4 +-- 6 files changed, 48 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 24de599..ad6c750 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,14 +43,17 @@ notes. Without it, the floor `HashEmbedder` is used — deterministic but weak (Russian recall rarely clears the confidence gate, many commands fall to "clarify"). -**Download the embedder** (ONNX, ~90 MB): +**Download the embedder** (ONNX, ~120 MB): ```sh make download-embedder ``` -This fetches `paraphrase-multilingual-MiniLM-L12-v2` (384-dim, 12-layer, -supports 50+ languages including Russian) to `models/embedder/`. +This fetches `multilingual-e5-small` (384-dim, 12-layer, Russian and English) +to `models/embedder/multilingual-e5-small/`. It is an asymmetric retrieval +model: the code puts `query: ` in front of a question and `passage: ` in front +of a stored note, which is how e5 was trained. The quantized file is the one +that is downloaded, deployed and measured. **Also need ONNX Runtime** (`libonnxruntime.so`): @@ -64,8 +67,8 @@ sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/ ```json "voice": { "embedder": { - "model_path": "models/embedder/model_quantized.onnx", - "tokenizer_path": "models/embedder/tokenizer.json", + "model_path": "models/embedder/multilingual-e5-small/model_quantized.onnx", + "tokenizer_path": "models/embedder/multilingual-e5-small/tokenizer.json", "lib_path": "/usr/local/lib/libonnxruntime.so" } } diff --git a/Makefile b/Makefile index 6c4f8e6..9bff320 100644 --- a/Makefile +++ b/Makefile @@ -115,9 +115,13 @@ deps-piper: -o /tmp/piper.tar.gz tar -xzf /tmp/piper.tar.gz -C deps/ -EMBEDDER_DIR := $(shell pwd)/models/embedder -EMBEDDER_MODEL_URL := https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2/resolve/main/onnx/model_quantized.onnx -EMBEDDER_TOKENIZER_URL := https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2/resolve/main/tokenizer.json +# multilingual-e5-small: an asymmetric retrieval model. It is trained to match +# a short question against a longer passage, which is what note recall is. +# The quantized file is the one we download, deploy and measure — see +# RECALL-EVAL-31-07-2026.md. +EMBEDDER_DIR := $(shell pwd)/models/embedder/multilingual-e5-small +EMBEDDER_MODEL_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/onnx/model_quantized.onnx +EMBEDDER_TOKENIZER_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/tokenizer.json download-embedder: mkdir -p $(EMBEDDER_DIR) diff --git a/deploy/mavend.json b/deploy/mavend.json index 3a8c122..1328412 100644 --- a/deploy/mavend.json +++ b/deploy/mavend.json @@ -36,8 +36,8 @@ "stt": { "socket": "/run/maven/stt.sock", "lang": "ru" }, "tts": { "socket": "/run/maven/tts.sock", "lang": "ru" }, "embedder": { - "model_path": "/opt/maven/models/embedder/model.onnx", - "tokenizer_path": "/opt/maven/models/embedder/tokenizer.json", + "model_path": "/opt/maven/models/embedder/multilingual-e5-small/model_quantized.onnx", + "tokenizer_path": "/opt/maven/models/embedder/multilingual-e5-small/tokenizer.json", "lib_path": "/opt/maven/lib/libonnxruntime.so" }, "tool_timeout": "30s", diff --git a/internal/memory/recalleval/recalleval.go b/internal/memory/recalleval/recalleval.go index 1202683..c0307cc 100644 --- a/internal/memory/recalleval/recalleval.go +++ b/internal/memory/recalleval/recalleval.go @@ -117,18 +117,40 @@ type cachingEmbedder struct { seen map[string][]float32 } +var _ router.AsymmetricEmbedder = (*cachingEmbedder)(nil) + func (c *cachingEmbedder) Dim() int { return c.inner.Dim() } func (c *cachingEmbedder) Close() error { return nil } // the caller owns inner func (c *cachingEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { - if v, ok := c.seen[text]; ok { + return c.cached(ctx, "embed:"+text, func() ([]float32, error) { + return c.inner.Embed(ctx, text) + }) +} + +// The two sides of an asymmetric embedder give different vectors for the same +// string, so the cache key has to say which side asked. +func (c *cachingEmbedder) EmbedQuery(ctx context.Context, text string) ([]float32, error) { + return c.cached(ctx, "query:"+text, func() ([]float32, error) { + return router.EmbedQuery(ctx, c.inner, text) + }) +} + +func (c *cachingEmbedder) EmbedPassage(ctx context.Context, text string) ([]float32, error) { + return c.cached(ctx, "passage:"+text, func() ([]float32, error) { + return router.EmbedPassage(ctx, c.inner, text) + }) +} + +func (c *cachingEmbedder) cached(_ context.Context, key string, embed func() ([]float32, error)) ([]float32, error) { + if v, ok := c.seen[key]; ok { return v, nil } - v, err := c.inner.Embed(ctx, text) + v, err := embed() if err != nil { return nil, err } - c.seen[text] = v + c.seen[key] = v return v, nil } @@ -305,7 +327,7 @@ func scoreCase(ctx context.Context, emb router.Embedder, newStore NewStore, minS all := append(append([]StoredNote(nil), c.Notes...), filler...) for _, n := range all { - vec, err := emb.Embed(ctx, n.Text) + vec, err := router.EmbedPassage(ctx, emb, n.Text) if err != nil { return Outcome{}, fmt.Errorf("%s: embed note %s: %w", c.ID, n.ID, err) } @@ -317,7 +339,7 @@ func scoreCase(ctx context.Context, emb router.Embedder, newStore NewStore, minS o := Outcome{Case: c} start := time.Now() - qvec, err := emb.Embed(ctx, c.Query) + qvec, err := router.EmbedQuery(ctx, emb, c.Query) if err != nil { o.Latency = time.Since(start) o.Err = err diff --git a/internal/memory/recalleval/recalleval_test.go b/internal/memory/recalleval/recalleval_test.go index 9d80f2e..c12cdf7 100644 --- a/internal/memory/recalleval/recalleval_test.go +++ b/internal/memory/recalleval/recalleval_test.go @@ -246,8 +246,8 @@ func TestONNXRecall(t *testing.T) { if lib == "" { t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") } - model := filepath.Join("../../..", "models/embedder/model.onnx") - tok := filepath.Join("../../..", "models/embedder/tokenizer.json") + model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx") + tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json") for _, p := range []string{lib, model, tok} { if _, err := os.Stat(p); err != nil { t.Skipf("missing %s: %v", p, err) diff --git a/internal/router/eval/eval_test.go b/internal/router/eval/eval_test.go index a92b537..6f0a1a6 100644 --- a/internal/router/eval/eval_test.go +++ b/internal/router/eval/eval_test.go @@ -185,8 +185,8 @@ func TestONNXBaseline(t *testing.T) { if lib == "" { t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") } - model := filepath.Join("../../..", "models/embedder/model.onnx") - tok := filepath.Join("../../..", "models/embedder/tokenizer.json") + model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx") + tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json") for _, p := range []string{lib, model, tok} { if _, err := os.Stat(p); err != nil { t.Skipf("missing %s: %v", p, err) From 1d48755d12fbef00a3637fcb762300569d9d9808 Mon Sep 17 00:00:00 2001 From: kami Date: Fri, 31 Jul 2026 11:38:18 +0400 Subject: [PATCH 3/3] Record the recall numbers after the embedder swap recall@1 60% to 72%, answered 48% to 72%, latency 3x better. But false recall went 1/5 to 5/5: e5 packs every score into a narrow high band, so the 0.55 gate now admits everything. Left the gate alone as instructed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ --- RECALL-EVAL-31-07-2026.md | 60 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/RECALL-EVAL-31-07-2026.md b/RECALL-EVAL-31-07-2026.md index 45852b9..9330f81 100644 --- a/RECALL-EVAL-31-07-2026.md +++ b/RECALL-EVAL-31-07-2026.md @@ -83,6 +83,66 @@ sqlite-backed `store.MemoryStore` and `memory.InMemoryStore` identically — bot (`internal/store/memory.go:64`) at ~150µs over 42 rows against a ~59ms query embed. An ANN index is not the problem to solve. +## Re-measured after the embedder swap — 31-07-2026, later the same day + +Changed: `models/embedder/` is now **multilingual-e5-small** (quantized, 118MB), with `query: ` in +front of a question and `passage: ` in front of a stored note (Vikunja #371). `deploy/mavend.json` +and `make download-embedder` now name the same file, and it is the quantized one — that is what the +column below measures (Vikunja #372). Everything else is unchanged: same fixture, same store, same +0.55 gate. The old column is the baseline and is left as it was. + +| | recall+onnx, MiniLM (baseline) | recall+onnx, e5-small (new) | +|---|---|---| +| **recall@1** | 60.0% (15/25) | **72.0% (18/25)** | +| recall@3 | 80.0% (20/25) | 84.0% (21/25) | +| **answered after the 0.55 gate** | 48.0% (12/25) | **72.0% (18/25)** | +| wrong note on top / tie on top | 10 / 0 | 7 / 0 | +| ranked first, then silenced by the gate | 3 | 0 | +| **false recall** | 1/5 (20%) | **5/5 (100%)** | +| top-1 score when right, min / median | 0.559 / 0.678 | 0.791 / 0.857 | +| top-1 when it must stay silent, median / max | 0.470 / 0.567 | 0.815 / 0.835 | +| RU / EN / `hard` cases passed | 13/24 / 3/6 / 2/11 | 14/24 / 4/6 / 5/11 | +| latency p50 / p95 / max | 59ms / 148ms / 194ms | 18ms / 37ms / 49ms | + +### What moved + +Ranking got better and got faster. Half the previously-unwinnable `hard` cases now pass (2/11 → +5/11), the guitar note no longer beats the docker-logs note, and the gate stops silencing notes that +already ranked first. The quantized e5 is also ~3x quicker than the fp32 MiniLM it replaces. + +### What got worse: the gate is now a no-op + +e5 packs every cosine into a narrow high band. Right-note scores start at 0.791; must-stay-silent +scores reach 0.835. **The distributions still overlap, and now they overlap above the gate**, so +0.55 admits everything and false recall goes from 1/5 to 5/5. The sweep: + +``` +gate 0.50–0.70: answered 18/25 (72%) false recall 5/5 +gate 0.80: answered 17/25 (68%) false recall 4/5 +gate 0.90: answered 0/25 ( 0%) false recall 0/5 +``` + +There is no value that keeps real recall and rejects made-up questions — same conclusion as before, +now with a wider band and no room at all. `query_min_score` was left at 0.55 as instructed. **The +recommendation is to leave it there and stop tuning it**: any number under ~0.79 is a no-op and +anything above starts cutting real recall long before it stops the false ones. The fix is a margin +gate (`top1 − top2 > δ`), next-steps item 3, which is now the top item. + +### The prefixes did not do the work + +A control run with both prefixes set to the empty string scored the **same** recall@1 (72%), a +slightly better recall@3 (88%) and the same 5/5 false recall. So on this fixture the gain comes from +the model, not from the `query:` / `passage:` split. The prefixes are kept because they are how e5 +was trained and the split is the right shape for the read path, but they are not worth defending on +this evidence — a bigger fixture may say otherwise. + +### Stored vectors from the old model are now junk + +Cosine between a MiniLM vector and an e5 vector means nothing. Every row already in `notes` and in +the vector memory table was written by the old model, so after this deploy they will score as noise +against a new query. A live database needs every note and fact re-embedded before recall works at +all. Filed as its own task. + ## Next steps — ordered by value-to-risk; nothing here is a decision 1. **Swap the embedder to `multilingual-e5-small` with `query:`/`passage:` prefixes.** One config