claude
5815f0b8f3
"что ты умеешь" reached the personal boundary, which claimed it as his and said "не знаю — не нашла у тебя такой записи" about her own description. Letting it past would be no better: SearXNG answers about somebody else's assistant. A self query source above the boundary, reading one frozen description. It is NOT a note — notes are his, and a note about her would come back for "что я записал", would be fed to the digestion worker as something he said, and would be recalled by proximity for questions that are not about her. The description names only what this box does. Everything that depends on config — the house, the LAN, the feeds, the list, weather, telegram — is named as depending on what he allowed, and a test pins that split: inventing a capability here is the same defect as inventing a fact. topicSelf is scored like every other topic, with a narrow keyword floor for the no-embedder case. "что ты умеешь" moved off topicOther, where it had been sitting so an attention question had something to lose to — a phrasing on two sides never clears the margin. TestONNXTopics 38/38 -> 43/43 on held-out utterances.
171 lines
8.6 KiB
Go
171 lines
8.6 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"os"
|
||
"path/filepath"
|
||
"testing"
|
||
|
||
"github.com/kami/maven/internal/router"
|
||
)
|
||
|
||
// TestTopicFloorAnswersWithoutSeeds — a handler with no embedder never loads the
|
||
// seeds, and every topic source has to keep working. This is the case that used
|
||
// to be the only one, so a regression here is all four recognisers going silent
|
||
// on a box with no embedder at all.
|
||
func TestTopicFloorAnswersWithoutSeeds(t *testing.T) {
|
||
h := &reactiveHandler{}
|
||
for _, tc := range []struct {
|
||
utterance string
|
||
label topicLabel
|
||
floor func(string) bool
|
||
want bool
|
||
}{
|
||
{"какая сегодня погода", topicWeather, isWeatherQuery, true},
|
||
{"что включено в доме?", topicHome, isHomeQuery, true},
|
||
{"какие устройства в сети?", topicNetwork, isNetworkQuery, true},
|
||
{"что требует внимания?", topicAttend, isAttentionQuery, true},
|
||
{"что нового в лентах?", topicFeed, feedFloor, true},
|
||
{"что в списке покупок?", topicList, listFloor, true},
|
||
{"почему небо синее", topicWeather, isWeatherQuery, false},
|
||
{"я дома", topicHome, isHomeQuery, false},
|
||
{"интернет не работает", topicNetwork, isNetworkQuery, false},
|
||
} {
|
||
turn := &queryTurn{dec: router.Decision{Utterance: tc.utterance}}
|
||
if got := h.turnIsAbout(context.Background(), turn, tc.label, tc.floor); got != tc.want {
|
||
t.Errorf("turnIsAbout(%q, %s) = %v, want %v", tc.utterance, tc.label, got, tc.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestONNXTopics — the number that matters, scored against the embedder homesrv
|
||
// actually runs. Opt-in via MAVEN_ONNX_LIB, like TestONNXPersonalBoundary.
|
||
//
|
||
// Every case is held out: none of these strings is a seed. It asserts what the
|
||
// gate does, not what the raw scorer says — a label under topicMargin is handed
|
||
// to the keyword floor, and one case turns on exactly that.
|
||
//
|
||
// The first three rows are the collisions the old regexes needed hand-written
|
||
// bail-outs for: the temperature pair that made isHomeQuery exclude weather
|
||
// words, and the "посетил" substring that made isNetworkQuery match "сети" as a
|
||
// whole token.
|
||
func TestONNXTopics(t *testing.T) {
|
||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||
if lib == "" {
|
||
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||
}
|
||
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
||
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
||
if err != nil {
|
||
t.Skipf("onnx embedder unavailable: %v", err)
|
||
}
|
||
defer emb.Close()
|
||
|
||
cases := []struct {
|
||
utterance string
|
||
want topicLabel
|
||
floor func(string) bool
|
||
}{
|
||
{"какая температура на улице?", topicWeather, isWeatherQuery},
|
||
{"какая температура в доме?", topicHome, isHomeQuery},
|
||
{"сколько машин я посетил?", topicOther, nil},
|
||
{"сколько сейчас градусов", topicWeather, isWeatherQuery},
|
||
{"дождь будет вечером?", topicWeather, isWeatherQuery},
|
||
{"тепло сегодня на улице?", topicWeather, isWeatherQuery},
|
||
{"свет на кухне включен?", topicHome, isHomeQuery},
|
||
{"что сейчас включено дома", topicHome, isHomeQuery},
|
||
{"датчики в квартире что показывают", topicHome, isHomeQuery},
|
||
{"просканируй сеть", topicNetwork, isNetworkQuery},
|
||
{"сколько устройств в локальной сети", topicNetwork, isNetworkQuery},
|
||
{"кто сейчас в сетке", topicNetwork, isNetworkQuery},
|
||
// The case the margin exists for. It reads as network by 0.0055, and
|
||
// isNetworkQuery says no, so it stays the complaint it is — a scan she
|
||
// runs unasked is the behaviour the bounds prevent.
|
||
{"вайфай опять отвалился", topicOther, isNetworkQuery},
|
||
{"я уже приехал домой", topicOther, nil},
|
||
{"что я говорил про погоду в москве", topicOther, nil},
|
||
{"напомни полить цветы", topicOther, nil},
|
||
{"что требует моего внимания сейчас", topicAttend, isAttentionQuery},
|
||
{"что не так с базой данных", topicAttend, isAttentionQuery},
|
||
{"есть что-то срочное на сегодня", topicAttend, isAttentionQuery},
|
||
{"что нового в ленте за сегодня", topicFeed, feedFloor},
|
||
{"какие сегодня заголовки", topicFeed, feedFloor},
|
||
{"что нового про искусственный интеллект", topicFeed, feedFloor},
|
||
// The greeting. It has to lose to topicOther, or fall thin enough that
|
||
// ParseFeedQuery — which declines a vague noun with no topic — answers.
|
||
{"что нового?", topicOther, feedFloor},
|
||
{"что мне надо купить в магазине", topicList, listFloor},
|
||
{"прочитай мне список", topicList, listFloor},
|
||
{"что там в аптеке нужно взять", topicList, listFloor},
|
||
// A task read-back is not a list read-back, and the two collide on
|
||
// "что у меня".
|
||
{"какие у меня сейчас задачи", topicOther, listFloor},
|
||
// World questions that name a day (Vikunja #553). Weather was the only
|
||
// topic carrying day words, so all of these read as weather and two of
|
||
// them cleared the margin: the gate asked "для какого города?" about
|
||
// the dollar. The last two are far from any seed on purpose — the
|
||
// first three are close enough to the new topicOther seeds that they
|
||
// would pass on similarity alone.
|
||
{"какой сегодня курс доллара", topicOther, isWeatherQuery},
|
||
{"какой сегодня праздник", topicOther, isWeatherQuery},
|
||
{"что интересного произошло сегодня в мире", topicOther, isWeatherQuery},
|
||
{"во сколько завтра открывается музей", topicOther, isWeatherQuery},
|
||
{"кто сегодня играет в лиге чемпионов", topicOther, isWeatherQuery},
|
||
// The control the seeds above must not cost: real weather still reads
|
||
// as weather, including the two that lean on the keyword floor.
|
||
{"будет ли завтра дождь в москве", topicWeather, isWeatherQuery},
|
||
{"какая температура завтра утром", topicWeather, isWeatherQuery},
|
||
// The same shape one topic over, seen on the box (Vikunja #554): a
|
||
// device has a history, and asking about it is not asking what is
|
||
// plugged in. "кто изобрёл телефон" answered "нашла 3 устройства".
|
||
// Held out from the seeds, which name the telephone and the computer.
|
||
{"кто придумал радио", topicOther, isNetworkQuery},
|
||
{"когда изобрели телевизор", topicOther, isNetworkQuery},
|
||
{"как устроен телефон внутри", topicOther, isNetworkQuery},
|
||
// The control: a real scan is still a scan.
|
||
{"какие устройства подключены к вайфаю", topicNetwork, isNetworkQuery},
|
||
// Questions about her (Vikunja #555), held out from selfSeeds.
|
||
{"а что ты вообще умеешь делать", topicSelf, selfFloor},
|
||
{"какие у тебя навыки", topicSelf, selfFloor},
|
||
{"расскажи мне о себе", topicSelf, selfFloor},
|
||
{"what are you able to do", topicSelf, selfFloor},
|
||
// The control: an attention question is about the state of his things,
|
||
// and it is the neighbour these seeds could have taken.
|
||
{"что требует внимания у меня в сервисах", topicAttend, isAttentionQuery},
|
||
}
|
||
|
||
h := &reactiveHandler{recall: recallWiring{embedder: emb}}
|
||
ctx := context.Background()
|
||
h.recall.topics.load(ctx, emb)
|
||
if !h.recall.topics.loaded {
|
||
t.Fatal("topic seeds did not load")
|
||
}
|
||
right := 0
|
||
for _, tc := range cases {
|
||
vec, err := router.EmbedQuery(ctx, emb, tc.utterance)
|
||
if err != nil {
|
||
t.Fatalf("embed %q: %v", tc.utterance, err)
|
||
}
|
||
label, margin, ok := h.recall.topics.best(vec)
|
||
if !ok {
|
||
t.Fatalf("best(%q) not ok", tc.utterance)
|
||
}
|
||
// What the gate would do, which is the thing under test: a label that
|
||
// does not clear the margin is handed to that source's keyword floor.
|
||
got := label
|
||
if margin < topicMargin {
|
||
got = topicOther
|
||
if tc.floor != nil && tc.floor(tc.utterance) {
|
||
got = label
|
||
}
|
||
}
|
||
if got == tc.want {
|
||
right++
|
||
} else {
|
||
t.Errorf("%q: %s by %.4f, want %s", tc.utterance, label, margin, tc.want)
|
||
}
|
||
t.Logf(" %-40s -> %-8s margin %.4f", tc.utterance, label, margin)
|
||
}
|
||
t.Logf("topics: %d/%d", right, len(cases))
|
||
}
|