diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index 48fb523..c349d2d 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -715,7 +715,7 @@ func (h *reactiveHandler) queryKiwix(ctx context.Context, t *queryTurn) (string, // not be sent to an upstream engine at all. The guard closes both holes with // the same test. func (h *reactiveHandler) queryPersonal(ctx context.Context, t *queryTurn) (string, bool) { - if !isPersonalQuery(t.dec.Utterance) { + if !h.isPersonalTurn(ctx, t) { return "", false } log.Printf("voice: %q is about him and his own data did not answer it; not asking the world", t.dec.Utterance) @@ -740,7 +740,9 @@ var personalMarkers = []*regexp.Regexp{ regexp.MustCompile(`(?i)\bdid\s+i\b`), } -// isPersonalQuery reports whether the utterance asks about something of his. +// isPersonalQuery — the offline floor under the boundary. Possession only, and +// deliberately still narrow: it answers when there is no embedder to ask, and a +// broad guess made blind is worse than a narrow one. func isPersonalQuery(utterance string) bool { if utterance == "" { return false @@ -753,6 +755,23 @@ func isPersonalQuery(utterance string) bool { return false } +// isPersonalTurn — the boundary test. The seeds decide when the embedder is +// there, which is every deployed box; the possession markers are the floor +// underneath, for a handler with no embedder or a turn whose vector never got +// computed. Same shape as the cascade: the better test leads, the offline one +// always answers. +func (h *reactiveHandler) isPersonalTurn(ctx context.Context, t *queryTurn) bool { + h.boundary.load(ctx, h.embedder) + if personal, world, ok := h.boundary.score(t.vec); ok { + if personal > world { + log.Printf("voice: %q scores personal %.4f vs world %.4f", t.dec.Utterance, personal, world) + return true + } + return false + } + return isPersonalQuery(t.dec.Utterance) +} + // queryGeneral — general knowledge, the last source before giving up. It always // claims: either a model answers, or Maven names the gap, or she says she does // not know. diff --git a/cmd/mavend/actions_query_personal_test.go b/cmd/mavend/actions_query_personal_test.go index 23d01e9..54f19e4 100644 --- a/cmd/mavend/actions_query_personal_test.go +++ b/cmd/mavend/actions_query_personal_test.go @@ -19,6 +19,10 @@ func TestIsPersonalQuery(t *testing.T) { "when is my meeting", "do i have anything today", "did i take my vitamins", + // Speech, but only the forms possession already covers ("did i"). + // The verb forms the floor cannot see are the seeds' job, scored in + // TestONNXPersonalBoundary. + "what did i say about backups", } { if !isPersonalQuery(s) { t.Errorf("isPersonalQuery(%q) = false, want true", s) @@ -33,6 +37,10 @@ func TestIsPersonalQuery(t *testing.T) { "почему небо синее", "столица франции", "how do i boil an egg", + // The floor is possession-only by design: a speech verb it cannot see + // passes here and is caught by the seeds instead. + "что я говорил про бэкапы?", + "как я говорил, почему небо синее", "", } { if isPersonalQuery(s) { diff --git a/cmd/mavend/personalboundary.go b/cmd/mavend/personalboundary.go new file mode 100644 index 0000000..cda5cf4 --- /dev/null +++ b/cmd/mavend/personalboundary.go @@ -0,0 +1,154 @@ +package main + +import ( + "context" + "log" + "math" + "sync" + + "github.com/kami/maven/internal/router" +) + +// The personal boundary decides one thing: is this question about him. It used +// to decide it by matching possession words, and that was the whole defect +// behind Vikunja #495. "что я говорил про бэкапы?" is his data by definition — +// nothing outside the box has ever heard him say anything — and it carried no +// possession word, so it walked past the boundary into SearXNG and came back +// answered out of a Habr article about somebody else's backups. +// +// The first fix was one more marker class, `я говорил|сказал|писал|…`, plus a +// carve-out so "как я говорил, почему небо синее" stayed a world question. Both +// halves are a lexicon, and a lexicon is the wrong instrument here: Russian +// gives every verb a dozen surface forms, the preamble list has no end, and +// every utterance the list misses is one that reaches the world. It also drifts +// silently — a missing verb looks exactly like no bug. +// +// So the boundary asks the embedder instead. Two frozen seed sets — questions +// about him, questions about the world — are embedded once, and the turn's own +// query vector, already computed by queryEmbed upstream, is scored against +// both. Nearest side wins. Word order, verb form and unseen phrasing stop +// mattering, which is exactly what a lexicon could not do. +// +// Measured 03-08-2026 against multilingual-e5-small on 19 held-out utterances, +// none of them a seed: 19 right (TestONNXPersonalBoundary). A 20th, "as i said, +// what is the population of india", missed by +0.008 during the first pass and +// is a world seed now, which is why it is not in the held-out set. True +// positives clear the world side by +0.014 to +0.089 and the nearest true +// negative sits at -0.005, so the gate is the sign of the difference and +// nothing tighter: the margins are too thin to justify a threshold, and the +// asymmetry favours claiming anyway. A false claim costs one honest "не знаю"; +// a false pass sends his life to an upstream engine. +// +// The embedder is the one model CLAUDE.md pins to homesrv permanently, and it +// is what makes this affordable: no llama-server call, no network, one cosine +// per seed against a vector the turn already has. + +// personalSeeds — questions about him. Frozen: they are scoring data, so +// editing one moves the boundary and must be re-measured, not eyeballed. Cover +// both classes the boundary owns, possession and first-person speech, in both +// languages. +var personalSeeds = []string{ + "что я говорил про это", + "я тебе рассказывал об этом?", + "что я записал про врача", + "я упоминал эту тему?", + "что у меня сегодня", + "когда моя встреча", + "what did i say about this", + "did i mention this to you", +} + +// worldSeeds — questions the world can answer, including the two shapes that +// look personal and are not: a first-person preamble on a world question ("как +// я говорил, ..."), and first person without possession ("что я могу +// посмотреть вечером"). Refusing those is the opposite mistake and the older +// comment on personalMarkers already named it. +var worldSeeds = []string{ + "почему небо синее", + "какая столица франции", + "как сварить борщ", + "кто написал эту книгу", + "what is the capital of france", + "how do i boil an egg", + "как я говорил, почему небо синее", + "as i said, why is the sky blue", + "as i said, what is the population of india", + "что я могу посмотреть вечером", + "что мне почитать про историю", + "что я должен знать про питон", + "what can i watch tonight", +} + +// personalBoundary holds the embedded seeds. Zero value is usable and means +// "not loaded yet"; a handler built without an embedder never loads and the +// boundary falls back to personalMarkers. +type personalBoundary struct { + once sync.Once + personal [][]float32 + world [][]float32 + loaded bool +} + +// load embeds both seed sets, once per process. Seeds are embedded on the QUERY +// side, like the utterance they are compared with — a question against a +// question. Mixing sides would measure the e5 prefix, not the meaning. +func (b *personalBoundary) load(ctx context.Context, emb router.Embedder) { + b.once.Do(func() { + if emb == nil { + return + } + embedAll := func(ss []string) [][]float32 { + out := make([][]float32, 0, len(ss)) + for _, s := range ss { + v, err := router.EmbedQuery(ctx, emb, s) + if err != nil { + log.Printf("voice: personal boundary seeds unavailable (%v); falling back to possession markers", err) + return nil + } + out = append(out, v) + } + return out + } + p, w := embedAll(personalSeeds), embedAll(worldSeeds) + if p == nil || w == nil { + return + } + b.personal, b.world, b.loaded = p, w, true + }) +} + +// score returns the best similarity to each side. ok is false when the seeds +// are not loaded, which is the caller's signal to use the markers instead. +func (b *personalBoundary) score(vec []float32) (personal, world float64, ok bool) { + if !b.loaded || len(vec) == 0 { + return 0, 0, false + } + best := func(seeds [][]float32) float64 { + m := -1.0 + for _, s := range seeds { + if c := cosine(vec, s); c > m { + m = c + } + } + return m + } + return best(b.personal), best(b.world), true +} + +// cosine — same math as internal/router and internal/memory, small enough that +// importing one of them for it would be the larger coupling. +func cosine(a, b []float32) float64 { + if len(a) != len(b) { + return 0 + } + var dot, na, nb float64 + for i := range a { + dot += float64(a[i]) * float64(b[i]) + na += float64(a[i]) * float64(a[i]) + nb += float64(b[i]) * float64(b[i]) + } + if na == 0 || nb == 0 { + return 0 + } + return dot / (math.Sqrt(na) * math.Sqrt(nb)) +} diff --git a/cmd/mavend/personalboundary_test.go b/cmd/mavend/personalboundary_test.go new file mode 100644 index 0000000..70de0fe --- /dev/null +++ b/cmd/mavend/personalboundary_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kami/maven/internal/router" +) + +// A handler with no embedder never loads the seeds, so the boundary falls back +// to the possession markers. That is the offline floor and it must keep working +// — an embedder that fails to load must not open the boundary. +func TestBoundaryFallsBackToMarkersWithNoEmbedder(t *testing.T) { + h := personalHandler() + if !h.isPersonalTurn(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "во сколько у меня встреча"}, + }) { + t.Error("no embedder: a possession question must still be personal") + } + if h.isPersonalTurn(context.Background(), &queryTurn{ + dec: router.Decision{Utterance: "почему небо синее"}, + }) { + t.Error("no embedder: a world question must still pass") + } +} + +// TestONNXPersonalBoundary — the number that matters, scored against the +// embedder homesrv actually runs. Opt-in via MAVEN_ONNX_LIB, exactly like +// TestONNXRecall in internal/memory/recalleval. +// +// Every case here is held out: none of these strings is a seed. The #495 +// regression is the first row — "что я говорил про бэкапы?" reached SearXNG and +// was answered from a Habr article, and no possession word appears in it. +func TestONNXPersonalBoundary(t *testing.T) { + lib := os.Getenv("MAVEN_ONNX_LIB") + if lib == "" { + t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") + } + dir := filepath.Join("../..", "models/embedder/multilingual-e5-small") + emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib) + if err != nil { + t.Skipf("onnx embedder unavailable: %v", err) + } + defer emb.Close() + + cases := []struct { + utterance string + personal bool + }{ + {"что я говорил про бэкапы?", true}, + {"что я сказал вчера про отпуск", true}, + {"я писал что-нибудь про сервер", true}, + {"я упоминал про конференцию?", true}, + {"что я отмечал по поводу переезда", true}, + {"я рассказывал тебе про новую работу?", true}, + {"во сколько у меня встреча", true}, + {"когда мой следующий отпуск", true}, + {"what did i say about backups", true}, + {"did i tell you about the doctor", true}, + {"как я говорил, почему небо синее", false}, + {"как уже я говорил, какая столица франции", false}, + {"почему трава зелёная", false}, + {"столица франции", false}, + {"как мне сварить борщ", false}, + {"что мне посмотреть вечером", false}, + {"я хочу узнать про рим", false}, + {"кто такой гагарин", false}, + {"how do i boil an egg", false}, + } + + h := &reactiveHandler{embedder: emb} + ctx := context.Background() + wrong := 0 + for _, c := range cases { + vec, err := router.EmbedQuery(ctx, emb, c.utterance) + if err != nil { + t.Fatalf("embed %q: %v", c.utterance, err) + } + turn := &queryTurn{dec: router.Decision{Utterance: c.utterance}, vec: vec} + got := h.isPersonalTurn(ctx, turn) + p, w, ok := h.boundary.score(vec) + if !ok { + t.Fatal("seeds did not load with a working embedder") + } + if got != c.personal { + wrong++ + t.Errorf("%q: personal=%v want %v (personal %.4f world %.4f)", c.utterance, got, c.personal, p, w) + } + t.Logf("personal=%-5v personal %.4f world %.4f delta %+.4f %s", got, p, w, p-w, c.utterance) + } + t.Logf("personal boundary: %d/%d held-out utterances correct", len(cases)-wrong, len(cases)) +} diff --git a/cmd/mavend/voice.go b/cmd/mavend/voice.go index f083e50..c86a09a 100644 --- a/cmd/mavend/voice.go +++ b/cmd/mavend/voice.go @@ -76,6 +76,10 @@ type reactiveHandler struct { tts tts.Synthesizer router *router.Router embedder router.Embedder // reused for note write/query (same model as the classifier) + // boundary — the embedded seed sets behind the personal boundary + // (personalboundary.go). Zero value is usable and loads on first query; + // with no embedder it never loads and the boundary uses personalMarkers. + boundary personalBoundary // api — the CoreAPI the handler reads and writes through. Wired with the // bare store adapter and UPGRADED by main once the daemonAPI exists; see // upgradeAPI. diff --git a/docs/evals/2026-08-03-personal-boundary.md b/docs/evals/2026-08-03-personal-boundary.md new file mode 100644 index 0000000..2911014 --- /dev/null +++ b/docs/evals/2026-08-03-personal-boundary.md @@ -0,0 +1,46 @@ +# Personal boundary, seed scoring vs possession markers, 2026-08-03 + +Vikunja #495. `что я говорил про бэкапы?` walked past the personal boundary into +SearXNG and came back answered from a Habr article. The boundary matched +possession words only, so a first-person speech verb was not a personal +question. + +## What changed + +The boundary now scores the turn's query vector against two frozen seed sets. +It claims the turn when the personal side is nearer than the world side. Seeds +and code are in `cmd/mavend/personalboundary.go`. The possession markers stay as +the offline floor for a handler with no embedder. + +A regex speech class was written first and dropped. Russian gives every verb a +dozen surface forms, and the "как я говорил, ..." preamble list has no end. Each +form the lexicon missed was one more question reaching the world. + +## Measurement + +Embedder: multilingual-e5-small int8, the one homesrv runs. Both sides are +embedded on the query side. Cases are held out, none of them a seed. `make test` +runs the offline part. The scored part is opt-in through `MAVEN_ONNX_LIB`, like +`TestONNXRecall`. + + 19/19 held-out utterances correct (TestONNXPersonalBoundary) + + true positive margins +0.014 to +0.089 + nearest true negative -0.005 ("кто такой гагарин") + +One case missed during the first pass and is not held out any more: `as i said, +what is the population of india`, +0.008 to the personal side. It is a world seed +now. + +The gate is the sign of the difference and nothing tighter. The margins are too +thin for a threshold. The asymmetry favours claiming: a false claim costs one +honest "не знаю", a false pass sends his life to an upstream engine. + +`make eval-recall` unchanged, 18/27 answered at gate 0.55. Recall does not touch +this path. + +## Not verified + +The live probe on the deployed box. The daemon was not rebuilt in this session. +The reply to `что я говорил про бэкапы?` with no matching note is still untested +against a real SearXNG.