claude
86817d6d06
"что я говорил про бэкапы?" is his data by definition, and nothing outside the box has ever heard him say anything. The boundary matched possession words only, so the question walked past it into SearXNG and came back answered out of a Habr article about somebody else's backups. A speech-verb marker class was written first and dropped. Russian gives every verb a dozen surface forms and the "как я говорил, ..." preamble list has no end, so each form the lexicon missed was one more question reaching the world, and a missing verb looks exactly like no bug. The boundary now embeds two frozen seed sets and scores the turn's own query vector, already computed upstream, against both. Nearest side wins. The possession markers stay as the offline floor for a handler with no embedder. 19/19 held-out utterances correct against multilingual-e5-small; see docs/evals/2026-08-03-personal-boundary.md. The live probe on the deployed box is not done. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
95 lines
3.7 KiB
Go
95 lines
3.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/kami/maven/internal/router"
|
|
)
|
|
|
|
// A handler with no embedder never loads the seeds, so the boundary falls back
|
|
// to the possession markers. That is the offline floor and it must keep working
|
|
// — an embedder that fails to load must not open the boundary.
|
|
func TestBoundaryFallsBackToMarkersWithNoEmbedder(t *testing.T) {
|
|
h := personalHandler()
|
|
if !h.isPersonalTurn(context.Background(), &queryTurn{
|
|
dec: router.Decision{Utterance: "во сколько у меня встреча"},
|
|
}) {
|
|
t.Error("no embedder: a possession question must still be personal")
|
|
}
|
|
if h.isPersonalTurn(context.Background(), &queryTurn{
|
|
dec: router.Decision{Utterance: "почему небо синее"},
|
|
}) {
|
|
t.Error("no embedder: a world question must still pass")
|
|
}
|
|
}
|
|
|
|
// TestONNXPersonalBoundary — the number that matters, scored against the
|
|
// embedder homesrv actually runs. Opt-in via MAVEN_ONNX_LIB, exactly like
|
|
// TestONNXRecall in internal/memory/recalleval.
|
|
//
|
|
// Every case here is held out: none of these strings is a seed. The #495
|
|
// regression is the first row — "что я говорил про бэкапы?" reached SearXNG and
|
|
// was answered from a Habr article, and no possession word appears in it.
|
|
func TestONNXPersonalBoundary(t *testing.T) {
|
|
lib := os.Getenv("MAVEN_ONNX_LIB")
|
|
if lib == "" {
|
|
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
|
}
|
|
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
|
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
|
if err != nil {
|
|
t.Skipf("onnx embedder unavailable: %v", err)
|
|
}
|
|
defer emb.Close()
|
|
|
|
cases := []struct {
|
|
utterance string
|
|
personal bool
|
|
}{
|
|
{"что я говорил про бэкапы?", true},
|
|
{"что я сказал вчера про отпуск", true},
|
|
{"я писал что-нибудь про сервер", true},
|
|
{"я упоминал про конференцию?", true},
|
|
{"что я отмечал по поводу переезда", true},
|
|
{"я рассказывал тебе про новую работу?", true},
|
|
{"во сколько у меня встреча", true},
|
|
{"когда мой следующий отпуск", true},
|
|
{"what did i say about backups", true},
|
|
{"did i tell you about the doctor", true},
|
|
{"как я говорил, почему небо синее", false},
|
|
{"как уже я говорил, какая столица франции", false},
|
|
{"почему трава зелёная", false},
|
|
{"столица франции", false},
|
|
{"как мне сварить борщ", false},
|
|
{"что мне посмотреть вечером", false},
|
|
{"я хочу узнать про рим", false},
|
|
{"кто такой гагарин", false},
|
|
{"how do i boil an egg", false},
|
|
}
|
|
|
|
h := &reactiveHandler{embedder: emb}
|
|
ctx := context.Background()
|
|
wrong := 0
|
|
for _, c := range cases {
|
|
vec, err := router.EmbedQuery(ctx, emb, c.utterance)
|
|
if err != nil {
|
|
t.Fatalf("embed %q: %v", c.utterance, err)
|
|
}
|
|
turn := &queryTurn{dec: router.Decision{Utterance: c.utterance}, vec: vec}
|
|
got := h.isPersonalTurn(ctx, turn)
|
|
p, w, ok := h.boundary.score(vec)
|
|
if !ok {
|
|
t.Fatal("seeds did not load with a working embedder")
|
|
}
|
|
if got != c.personal {
|
|
wrong++
|
|
t.Errorf("%q: personal=%v want %v (personal %.4f world %.4f)", c.utterance, got, c.personal, p, w)
|
|
}
|
|
t.Logf("personal=%-5v personal %.4f world %.4f delta %+.4f %s", got, p, w, p-w, c.utterance)
|
|
}
|
|
t.Logf("personal boundary: %d/%d held-out utterances correct", len(cases)-wrong, len(cases))
|
|
}
|