claude
feabf9f350
encodeWord backtracks the Viterbi path from the end of the word and prepends each piece, which puts them back in reading order. A second reverse after that loop undid it. So "query: вода" tokenized to [0 12 1294 41 12489 2] where the reference tokenizer gives [0 41 1294 12 12489 2], and every multi-piece Russian word reached the model with its pieces in the wrong order. Measured on the recall fixture, same 27 cases either way: recall@1 70.4% -> 77.8% recall@3 85.2% -> 96.3% answered after gate 63.0% -> 66.7% false recall 0/5 -> 1/5 The classifier barely moves, 76.0% to 75.0% on the routing fixture, because seeds and queries were mangled the same way and cosine survived it. Recall is where it cost, because a stored passage and a live query are different lengths and break differently. The embedder id now names a tokenizer revision. Stored vectors were written under rev 1 and no longer sit in the same space as a query embedded now, and the model file's name never moved, so nothing would have triggered ReembedAll. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ptwopxyo3Z2kwFckHkLvN
35 lines
1.1 KiB
Go
35 lines
1.1 KiB
Go
package router
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestEmbedderIDFromModelPath(t *testing.T) {
|
|
got := modelIDFromPath("/opt/maven/models/embedder/multilingual-e5-small.onnx")
|
|
if got != "multilingual-e5-small@384/tok2" {
|
|
t.Fatalf("modelIDFromPath = %q", got)
|
|
}
|
|
// A different model file must produce a different id, even at 384 dim.
|
|
old := modelIDFromPath("/opt/maven/models/embedder/paraphrase-multilingual-MiniLM-L12-v2.onnx")
|
|
if old == got {
|
|
t.Fatal("two different models share one id")
|
|
}
|
|
// The tokenizer is half of what makes a vector, and it changes under a
|
|
// model file whose name never moves (V-664). An id that ignored it would
|
|
// leave stored passages in one space and every new query in another, with
|
|
// nothing to trigger the re-embed.
|
|
if !strings.Contains(got, "/tok") {
|
|
t.Fatalf("id %q does not name the tokenizer revision", got)
|
|
}
|
|
}
|
|
|
|
func TestEmbedderIDIncludesDim(t *testing.T) {
|
|
if id := EmbedderID(NewHashEmbedder(1024)); id != "hash@1024" {
|
|
t.Fatalf("EmbedderID = %q", id)
|
|
}
|
|
if EmbedderID(NewHashEmbedder(1024)) == EmbedderID(NewHashEmbedder(384)) {
|
|
t.Fatal("dimension not part of the id")
|
|
}
|
|
}
|