From f6d5a2a7a48b0c9b8978926ee26e23030befea5c Mon Sep 17 00:00:00 2001 From: kami Date: Fri, 31 Jul 2026 11:38:18 +0400 Subject: [PATCH] Swap the embedder to multilingual-e5-small (Vikunja #371, #372) The old model was a symmetric paraphrase model, so it scored "do these look alike" instead of "does this note answer this question". Also fixes the file mismatch: the Makefile, the deploy config and both evals now all name the same quantized file, and the quantized one is what gets measured. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ --- AGENTS.md | 13 +++++--- Makefile | 10 ++++-- deploy/mavend.json | 4 +-- internal/memory/recalleval/recalleval.go | 32 ++++++++++++++++--- internal/memory/recalleval/recalleval_test.go | 4 +-- internal/router/eval/eval_test.go | 4 +-- 6 files changed, 48 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 24de599..ad6c750 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,14 +43,17 @@ notes. Without it, the floor `HashEmbedder` is used — deterministic but weak (Russian recall rarely clears the confidence gate, many commands fall to "clarify"). -**Download the embedder** (ONNX, ~90 MB): +**Download the embedder** (ONNX, ~120 MB): ```sh make download-embedder ``` -This fetches `paraphrase-multilingual-MiniLM-L12-v2` (384-dim, 12-layer, -supports 50+ languages including Russian) to `models/embedder/`. +This fetches `multilingual-e5-small` (384-dim, 12-layer, Russian and English) +to `models/embedder/multilingual-e5-small/`. It is an asymmetric retrieval +model: the code puts `query: ` in front of a question and `passage: ` in front +of a stored note, which is how e5 was trained. The quantized file is the one +that is downloaded, deployed and measured. **Also need ONNX Runtime** (`libonnxruntime.so`): @@ -64,8 +67,8 @@ sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/ ```json "voice": { "embedder": { - "model_path": "models/embedder/model_quantized.onnx", - "tokenizer_path": "models/embedder/tokenizer.json", + "model_path": "models/embedder/multilingual-e5-small/model_quantized.onnx", + "tokenizer_path": "models/embedder/multilingual-e5-small/tokenizer.json", "lib_path": "/usr/local/lib/libonnxruntime.so" } } diff --git a/Makefile b/Makefile index 6c4f8e6..9bff320 100644 --- a/Makefile +++ b/Makefile @@ -115,9 +115,13 @@ deps-piper: -o /tmp/piper.tar.gz tar -xzf /tmp/piper.tar.gz -C deps/ -EMBEDDER_DIR := $(shell pwd)/models/embedder -EMBEDDER_MODEL_URL := https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2/resolve/main/onnx/model_quantized.onnx -EMBEDDER_TOKENIZER_URL := https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2/resolve/main/tokenizer.json +# multilingual-e5-small: an asymmetric retrieval model. It is trained to match +# a short question against a longer passage, which is what note recall is. +# The quantized file is the one we download, deploy and measure — see +# RECALL-EVAL-31-07-2026.md. +EMBEDDER_DIR := $(shell pwd)/models/embedder/multilingual-e5-small +EMBEDDER_MODEL_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/onnx/model_quantized.onnx +EMBEDDER_TOKENIZER_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/tokenizer.json download-embedder: mkdir -p $(EMBEDDER_DIR) diff --git a/deploy/mavend.json b/deploy/mavend.json index 3a8c122..1328412 100644 --- a/deploy/mavend.json +++ b/deploy/mavend.json @@ -36,8 +36,8 @@ "stt": { "socket": "/run/maven/stt.sock", "lang": "ru" }, "tts": { "socket": "/run/maven/tts.sock", "lang": "ru" }, "embedder": { - "model_path": "/opt/maven/models/embedder/model.onnx", - "tokenizer_path": "/opt/maven/models/embedder/tokenizer.json", + "model_path": "/opt/maven/models/embedder/multilingual-e5-small/model_quantized.onnx", + "tokenizer_path": "/opt/maven/models/embedder/multilingual-e5-small/tokenizer.json", "lib_path": "/opt/maven/lib/libonnxruntime.so" }, "tool_timeout": "30s", diff --git a/internal/memory/recalleval/recalleval.go b/internal/memory/recalleval/recalleval.go index 1202683..c0307cc 100644 --- a/internal/memory/recalleval/recalleval.go +++ b/internal/memory/recalleval/recalleval.go @@ -117,18 +117,40 @@ type cachingEmbedder struct { seen map[string][]float32 } +var _ router.AsymmetricEmbedder = (*cachingEmbedder)(nil) + func (c *cachingEmbedder) Dim() int { return c.inner.Dim() } func (c *cachingEmbedder) Close() error { return nil } // the caller owns inner func (c *cachingEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { - if v, ok := c.seen[text]; ok { + return c.cached(ctx, "embed:"+text, func() ([]float32, error) { + return c.inner.Embed(ctx, text) + }) +} + +// The two sides of an asymmetric embedder give different vectors for the same +// string, so the cache key has to say which side asked. +func (c *cachingEmbedder) EmbedQuery(ctx context.Context, text string) ([]float32, error) { + return c.cached(ctx, "query:"+text, func() ([]float32, error) { + return router.EmbedQuery(ctx, c.inner, text) + }) +} + +func (c *cachingEmbedder) EmbedPassage(ctx context.Context, text string) ([]float32, error) { + return c.cached(ctx, "passage:"+text, func() ([]float32, error) { + return router.EmbedPassage(ctx, c.inner, text) + }) +} + +func (c *cachingEmbedder) cached(_ context.Context, key string, embed func() ([]float32, error)) ([]float32, error) { + if v, ok := c.seen[key]; ok { return v, nil } - v, err := c.inner.Embed(ctx, text) + v, err := embed() if err != nil { return nil, err } - c.seen[text] = v + c.seen[key] = v return v, nil } @@ -305,7 +327,7 @@ func scoreCase(ctx context.Context, emb router.Embedder, newStore NewStore, minS all := append(append([]StoredNote(nil), c.Notes...), filler...) for _, n := range all { - vec, err := emb.Embed(ctx, n.Text) + vec, err := router.EmbedPassage(ctx, emb, n.Text) if err != nil { return Outcome{}, fmt.Errorf("%s: embed note %s: %w", c.ID, n.ID, err) } @@ -317,7 +339,7 @@ func scoreCase(ctx context.Context, emb router.Embedder, newStore NewStore, minS o := Outcome{Case: c} start := time.Now() - qvec, err := emb.Embed(ctx, c.Query) + qvec, err := router.EmbedQuery(ctx, emb, c.Query) if err != nil { o.Latency = time.Since(start) o.Err = err diff --git a/internal/memory/recalleval/recalleval_test.go b/internal/memory/recalleval/recalleval_test.go index 9d80f2e..c12cdf7 100644 --- a/internal/memory/recalleval/recalleval_test.go +++ b/internal/memory/recalleval/recalleval_test.go @@ -246,8 +246,8 @@ func TestONNXRecall(t *testing.T) { if lib == "" { t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") } - model := filepath.Join("../../..", "models/embedder/model.onnx") - tok := filepath.Join("../../..", "models/embedder/tokenizer.json") + model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx") + tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json") for _, p := range []string{lib, model, tok} { if _, err := os.Stat(p); err != nil { t.Skipf("missing %s: %v", p, err) diff --git a/internal/router/eval/eval_test.go b/internal/router/eval/eval_test.go index a92b537..6f0a1a6 100644 --- a/internal/router/eval/eval_test.go +++ b/internal/router/eval/eval_test.go @@ -185,8 +185,8 @@ func TestONNXBaseline(t *testing.T) { if lib == "" { t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") } - model := filepath.Join("../../..", "models/embedder/model.onnx") - tok := filepath.Join("../../..", "models/embedder/tokenizer.json") + model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx") + tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json") for _, p := range []string{lib, model, tok} { if _, err := os.Stat(p); err != nil { t.Skipf("missing %s: %v", p, err)