Make locative recall prove identity, not overlap (V-719)
The spare-key note scored 0.832 to 0.867 against a spare passport, a blue shirt, a blue document box and a car key. Score and margin cannot separate those: the right note runs 0.817 to 0.892 and the silent cases 0.787 to 0.874, so the ranges overlap and structure has to decide. RecallAllowed now takes two structural facts from the router. A locative question must corroborate every identity term against the candidate's subject, read up to its first dictionary-proven verb, so a location object in the note cannot answer for the thing being located. A turn that is not question-shaped needs a named shared topic even when it ends in '?', which is what "я отменил напоминание про молоко" lacked when it recalled an unrelated note at 0.825 with no runner-up to fail the margin. query_min_score moves 0.55 to 0.80 for tokenizer rev 2. The held-out fixture answers 14/27 real recalls and 0/14 false ones. LocativeAnswerVerifier is the resident-model second opinion, kept behind the deterministic gate and wired into nothing. The measurement that says why is docs/evals/2026-08-15-locative-answerability-verifier.md. --no-verify: master is the working branch this session by the owner's call. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -641,10 +641,11 @@ const (
|
||||
// небо синее?", because the right-note and must-be-silent score ranges overlap
|
||||
// and no threshold sits between them.
|
||||
func recallOnTopic(utterance, text string) bool {
|
||||
if memory.RecallAllowed(utterance, text) {
|
||||
if memory.RecallAllowed(utterance, text,
|
||||
router.IsOpenQuestionShaped(utterance), router.IsLocativeQuestionShaped(utterance)) {
|
||||
return true
|
||||
}
|
||||
log.Printf("voice: recall %q rejected for %q: a world question and no shared topic word", text, utterance)
|
||||
log.Printf("voice: recall %q rejected for %q: no structural ask with a shared topic, or a world/locative question with no shared topic", text, utterance)
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -193,9 +193,9 @@ func TestQueryRecallNoteCanWin(t *testing.T) {
|
||||
t.Run("the better-matching fact answers", func(t *testing.T) {
|
||||
h, _ := buildRecallHandler(t, q, []recallCase{
|
||||
{text: "молоко стоит в холодильнике", score: 0.80, kind: "note"},
|
||||
{text: "купил молоко в среду", score: 0.95, kind: "fact"},
|
||||
{text: "молоко было в холодильнике в среду", score: 0.95, kind: "fact"},
|
||||
})
|
||||
if reply := askQuery(t, h, q); reply != "купил молоко в среду" {
|
||||
if reply := askQuery(t, h, q); reply != "молоко было в холодильнике в среду" {
|
||||
t.Errorf("reply %q, want the fact read back", reply)
|
||||
}
|
||||
})
|
||||
@@ -212,3 +212,63 @@ func TestQueryRecallNoteCanWin(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestQueryRecallRequiresStructuralOrTopicEvidence — the whole-assistant
|
||||
// cold-start regression. The routing heads called an ordinary past-tense
|
||||
// report a query; with one note in the store the margin gate has no runner-up,
|
||||
// and cosine 0.825 was enough to speak a completely unrelated spare-key note.
|
||||
// A bare question mark does not turn the proposition into an open information
|
||||
// question, negation must not weaken the refusal, and a locative question must
|
||||
// corroborate the target it asks Maven to locate (V-719).
|
||||
func TestQueryRecallRequiresStructuralOrTopicEvidence(t *testing.T) {
|
||||
const unrelated = "запомни: запасной ключ лежит в синей коробке"
|
||||
for _, tc := range []struct {
|
||||
query string
|
||||
score float64
|
||||
}{
|
||||
{"я отменил напоминание про молоко", 0.825031306},
|
||||
{"я отменил напоминание про молоко?", 0.825031306},
|
||||
{"я не отменил напоминание про молоко", 0.825031306},
|
||||
{"я не отменил напоминание про молоко?", 0.825031306},
|
||||
{"где мой паспорт?", 0.817210},
|
||||
{"где я отменил напоминание про молоко?", 0.805800},
|
||||
{"где лежит синяя рубашка?", 0.837694},
|
||||
{"где лежит синяя папка?", 0.837472},
|
||||
{"где мой запасной паспорт?", 0.831662},
|
||||
{"где лежит запасная флешка?", 0.838980},
|
||||
{"где находится синяя коробка с документами?", 0.866553},
|
||||
{"где лежит ключ от машины?", 0.843853},
|
||||
{"где синяя коробка?", 0.90},
|
||||
} {
|
||||
t.Run(tc.query, func(t *testing.T) {
|
||||
h, phr := buildRecallHandler(t, tc.query, []recallCase{
|
||||
{text: unrelated, score: tc.score, kind: "note"},
|
||||
})
|
||||
reply := askQuery(t, h, tc.query)
|
||||
if strings.Contains(reply, "запасной ключ") {
|
||||
t.Fatalf("unrelated note escaped into reply %q", reply)
|
||||
}
|
||||
if len(phr.notes) != 0 {
|
||||
t.Fatalf("unrelated note reached the phraser: %q", phr.notes)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Voice punctuation is optional. A nominal request with no interrogative
|
||||
// still works when the candidate itself corroborates the named topic.
|
||||
const nominal = "адрес домашнего сервера"
|
||||
h, _ := buildRecallHandler(t, nominal, []recallCase{
|
||||
{text: "домашний сервер на 192.168.1.104", score: 0.90, kind: "note"},
|
||||
})
|
||||
if reply := askQuery(t, h, nominal); !strings.Contains(reply, "домашний сервер") {
|
||||
t.Fatalf("nominal recall lost its shared-topic answer: %q", reply)
|
||||
}
|
||||
|
||||
const locative = "где лежит запасной ключ?"
|
||||
h, _ = buildRecallHandler(t, locative, []recallCase{
|
||||
{text: "запасной ключ лежит в синей коробке", score: 0.90, kind: "note"},
|
||||
})
|
||||
if reply := askQuery(t, h, locative); !strings.Contains(reply, "запасной ключ") {
|
||||
t.Fatalf("locative recall lost its corroborated target: %q", reply)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/memory"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/store"
|
||||
"github.com/kami/maven/internal/tool"
|
||||
@@ -84,12 +85,76 @@ func TestReactiveNotesReminders(t *testing.T) {
|
||||
t.Fatal("expected at least one note, got none")
|
||||
}
|
||||
last := notes[0]
|
||||
if last.Text != "запомни что кофе закончился" {
|
||||
t.Errorf("note text = %q, want %q", last.Text, "запомни что кофе закончился")
|
||||
if last.Text != "кофе закончился" {
|
||||
t.Errorf("note text = %q, want %q", last.Text, "кофе закончился")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestRunTurnExplicitNoteStoresOnlyTheBody pins the live failure end to end:
|
||||
// a routed text turn reaches actionNote, stores only the dictated body in both
|
||||
// durable and vector memory, and cannot ask the resident model to choose the
|
||||
// acknowledgement's grammatical gender (V-721).
|
||||
func TestRunTurnExplicitNoteStoresOnlyTheBody(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
api := ipc.NewStoreAPI(st)
|
||||
now := time.Date(2026, 8, 15, 8, 0, 0, 0, time.FixedZone("+04", 4*60*60))
|
||||
emb := router.NewHashEmbedder(1024)
|
||||
mem := memory.NewInMemoryStore()
|
||||
rtr := router.New(router.Config{
|
||||
Grammars: []router.Grammar{{
|
||||
Name: "explicit-note-test",
|
||||
Decide: func(string) (router.Decision, bool) {
|
||||
return router.Decision{
|
||||
Stage: 0, Intent: router.IntentNote, Confidence: 1,
|
||||
// Deliberately hostile model slot: neither persistence nor
|
||||
// acknowledgement may use it.
|
||||
Slots: router.Slots{Text: "ты поедешь на дачу"},
|
||||
}, true
|
||||
},
|
||||
}},
|
||||
Threshold: 0.55,
|
||||
})
|
||||
model := &countingCompleter{out: `{"response":"Хорошо, сохранил.","mood":"neutral"}`}
|
||||
h := &reactiveHandler{
|
||||
api: api, router: rtr,
|
||||
recall: recallWiring{embedder: emb, memStore: mem},
|
||||
replier: newLLMReplier(model, nil),
|
||||
now: func() time.Time { return now },
|
||||
dataStore: st,
|
||||
}
|
||||
|
||||
const utterance = "запомни: запасной ключ лежит в синей коробке"
|
||||
if reply := h.runTurn(ctx, utterance, sourceText); reply != "сохранила заметку." {
|
||||
t.Fatalf("reply = %q, want the fixed feminine acknowledgement", reply)
|
||||
}
|
||||
if model.calls != 0 {
|
||||
t.Fatalf("resident model was called %d time(s) for a note acknowledgement", model.calls)
|
||||
}
|
||||
notes, err := st.RecentNotes(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentNotes: %v", err)
|
||||
}
|
||||
const body = "запасной ключ лежит в синей коробке"
|
||||
if len(notes) != 1 || notes[0].Text != body || notes[0].Source != "tap:voice" || !notes[0].Ts.Equal(now) {
|
||||
t.Fatalf("stored notes = %+v, want one exact body at the turn time", notes)
|
||||
}
|
||||
records, err := mem.ByPrefix(ctx, "note:")
|
||||
if err != nil {
|
||||
t.Fatalf("vector catalog: %v", err)
|
||||
}
|
||||
if len(records) != 1 || records[0].Meta["text"] != body {
|
||||
t.Fatalf("vector records = %+v, want the same extracted body", records)
|
||||
}
|
||||
if records[0].Meta["text"] == utterance || records[0].Meta["text"] == "ты поедешь на дачу" {
|
||||
t.Fatalf("vector metadata used a command or model rewrite: %+v", records[0].Meta)
|
||||
}
|
||||
if !phraser.IsAck(phraser.AckNote, nil, "сохранила заметку.") {
|
||||
t.Fatal("fixed acknowledgement is not registered as the note acknowledgement")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSpokenTaskCaptureFilesATask — the whole path, from the utterance to the
|
||||
// task table. It went dead when the router started claiming the marker as an
|
||||
// act: capture rides the note intent, so nothing below actionNote was ever
|
||||
|
||||
@@ -56,7 +56,7 @@ type recallWiring struct {
|
||||
// minScore — the note-recall confidence gate. Top cosine below this ⇒
|
||||
// "I don't know" instead of a guess. Tuned for the ONNX embedder; a knob,
|
||||
// not load-bearing math (same posture as the presence thresholds). Set by
|
||||
// wireVoice from VoiceConfig; default 0.55.
|
||||
// wireVoice from VoiceConfig; default 0.80.
|
||||
minScore float64
|
||||
|
||||
// minMargin — the second half of that gate: how far the top hit must beat
|
||||
|
||||
+1
-1
@@ -259,7 +259,7 @@
|
||||
"heads_path": "/opt/maven/models/embedder/router-heads/router_heads.onnx"
|
||||
},
|
||||
"llm_router": true,
|
||||
"query_min_score": 0.55,
|
||||
"query_min_score": 0.80,
|
||||
"query_min_margin": 0.008,
|
||||
"clarify_max_attempts": 3,
|
||||
"tool_timeout": "30s",
|
||||
|
||||
@@ -52,6 +52,14 @@ func TestDeployConfigLoads(t *testing.T) {
|
||||
if cfg.Voice.RouterThreshold <= 0 {
|
||||
t.Error("router threshold did not get its default")
|
||||
}
|
||||
// The recall thresholds are a measured pair. An explicit deployment value
|
||||
// silently overriding a retuned default would make the eval and the box run
|
||||
// different safety gates, so pin both directions here.
|
||||
if cfg.Voice.QueryMinScore != DefaultQueryMinScore || cfg.Voice.QueryMinMargin != DefaultQueryMinMargin {
|
||||
t.Errorf("deploy recall gate is %.3f/%.3f, defaults are %.3f/%.3f",
|
||||
cfg.Voice.QueryMinScore, cfg.Voice.QueryMinMargin,
|
||||
DefaultQueryMinScore, DefaultQueryMinMargin)
|
||||
}
|
||||
|
||||
// The second reach (V-649). The token is a ${VAR} that CI cannot resolve, so
|
||||
// the committed deployment makes the dark state explicit. Removing disabled
|
||||
|
||||
@@ -77,11 +77,15 @@ type ToolConfig struct {
|
||||
// Voice defaults, applied in normaliseVoice.
|
||||
const (
|
||||
DefaultRouterThreshold = 0.55
|
||||
DefaultQueryMinScore = 0.55
|
||||
// Read off the margin sweep in internal/memory/recalleval on the e5
|
||||
// embedder: 0.008 answers 68% of real questions (down from 72%) and cuts
|
||||
// false recall from 5/5 to 1/5. Every larger delta costs real recall
|
||||
// without removing that last one until 0.020, which drops recall to 44%.
|
||||
// The absolute half of the recall gate, recalibrated after tokenizer rev 2
|
||||
// changed the e5 score distribution. At 0.80 with the independent 0.008
|
||||
// margin and structural eligibility, the held-out fixture answers 14/27
|
||||
// real recalls and 0/14 false ones. The score distributions still overlap
|
||||
// (true minimum 0.817, silent maximum 0.874), so structure remains decisive.
|
||||
DefaultQueryMinScore = 0.80
|
||||
// The runner-up half is intentionally separate. Keep 0.008 as the ambiguity
|
||||
// floor measured before the structural gate; a single-hit store has no
|
||||
// runner-up, which is why score and structure are both required.
|
||||
DefaultQueryMinMargin = 0.008
|
||||
// DefaultClarifyMaxAttempts — see dialogue.DefaultMaxAttempts.
|
||||
DefaultClarifyMaxAttempts = 3
|
||||
@@ -258,8 +262,8 @@ type VoiceConfig struct {
|
||||
LLMRouter *bool `json:"llm_router,omitempty"`
|
||||
|
||||
// QueryMinScore — the note-recall confidence gate. Top cosine below this
|
||||
// ⇒ "I don't know" instead of a guess. Tuned for the ONNX embedder (0.55);
|
||||
// the HashEmbedder floor scores lexically and may never clear it. 0.55
|
||||
// ⇒ "I don't know" instead of a guess. Tuned for the corrected ONNX
|
||||
// embedder (0.80); the HashEmbedder floor scores lexically and may never clear it. 0.80
|
||||
// default if unset.
|
||||
QueryMinScore float64 `json:"query_min_score,omitempty"`
|
||||
|
||||
|
||||
@@ -68,3 +68,16 @@ func voiceConfigWith(t *testing.T, model, heads string) error {
|
||||
_, err := Load(writeConfig(t, body))
|
||||
return err
|
||||
}
|
||||
|
||||
func TestVoiceRecallGateDefaults(t *testing.T) {
|
||||
cfg, err := Load(writeConfig(t, `{"voice":{"enabled":true,"bind":"127.0.0.1:9100"}}`))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if got := cfg.Voice.QueryMinScore; got != DefaultQueryMinScore {
|
||||
t.Errorf("query score default = %.3f, want %.3f", got, DefaultQueryMinScore)
|
||||
}
|
||||
if got := cfg.Voice.QueryMinMargin; got != DefaultQueryMinMargin {
|
||||
t.Errorf("query margin default = %.3f, want %.3f", got, DefaultQueryMinMargin)
|
||||
}
|
||||
}
|
||||
|
||||
+135
-11
@@ -3,8 +3,19 @@ package memory
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/kami/maven/internal/lexicon"
|
||||
"github.com/kami/maven/internal/morph"
|
||||
)
|
||||
|
||||
var recallCaptureVerbs = func() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, w := range lexicon.CaptureVerbs() {
|
||||
out[w] = true
|
||||
}
|
||||
return out
|
||||
}()
|
||||
|
||||
// stopwords — words that carry no topic. A question and a note that share only
|
||||
// these share nothing: "почему небо синее" and "сеть какая-то медленная" both
|
||||
// contain "какая"-shaped filler and are about different worlds.
|
||||
@@ -49,18 +60,35 @@ var firstPerson = map[string]bool{
|
||||
|
||||
// RecallAllowed is the second half of the recall gate (#470). A hit that
|
||||
// cleared the score and margin gate may still be about something else
|
||||
// entirely: the held-out fixture puts the right note at 0.791-0.890 and the
|
||||
// must-be-silent cases at 0.795-0.835, so no threshold sits between them, and
|
||||
// entirely: the held-out fixture puts the right note at 0.817-0.892 and the
|
||||
// must-be-silent cases at 0.787-0.874, so no threshold sits between them, and
|
||||
// a note about his slow network answered "почему небо синее?".
|
||||
//
|
||||
// The veto applies only to a question that mentions nothing of his. That
|
||||
// The broad veto applies only to a question that mentions nothing of his. That
|
||||
// restriction is what keeps the fix from costing more than it saves: recall
|
||||
// exists to find the note whose words he no longer remembers, and demanding a
|
||||
// shared word of every recall silenced four true recalls on the fixture to
|
||||
// kill one false one. A question about his own life keeps the embedder alone
|
||||
// shared word of every recall silences several right-note paraphrases on the
|
||||
// fixture. A question about his own life keeps the embedder alone
|
||||
// as its judge. A question about the world has to name something the memory
|
||||
// actually mentions.
|
||||
//
|
||||
// openQuestion is the caller's structural evidence that the words ask for
|
||||
// information: an interrogative or narrative request, not punctuation alone.
|
||||
// It closes a different hole: a model can mis-route an ordinary first-person
|
||||
// report as a query. With only one stored note there is no runner-up for the
|
||||
// margin gate, so "я отменил напоминание про молоко" recalled an unrelated note
|
||||
// at cosine 0.825. A report or polar proposition therefore needs a named topic
|
||||
// shared with the hit, even when it ends in '?'. Nominal requests still work —
|
||||
// "адрес домашнего сервера" shares its topic — while a statistical route alone
|
||||
// cannot make an unrelated personal statement into a recall request.
|
||||
//
|
||||
// requireNamedTopic is the stricter locative frame: an answer to "where is X"
|
||||
// must corroborate every identity term in X. One overlapping modifier or
|
||||
// predicate is not enough: the spare-key note scores 0.832-0.867 for a spare
|
||||
// passport, blue shirt, blue document box, and car key, while sharing words
|
||||
// such as "spare", "blue", "box", or "key" (V-719). Verbs are grammar, not
|
||||
// identity, and are excluded using the embedded morphology dictionary.
|
||||
//
|
||||
// The veto's price was re-measured on 2026-08-03 (#496,
|
||||
// docs/evals/2026-08-03-recall-topic-veto.md). It costs one true recall and
|
||||
// buys one false one, and the fixture pass count is the same either way. The
|
||||
@@ -68,11 +96,89 @@ var firstPerson = map[string]bool{
|
||||
// reported as, and the fixture has no cross-language case at all. Do not add a
|
||||
// script test or a bilingual stem map for it — both are no-ops here. The
|
||||
// separating signal is semantic and belongs in a reranker, not in this file.
|
||||
func RecallAllowed(query, text string) bool {
|
||||
func RecallAllowed(query, text string, openQuestion, requireNamedTopic bool) bool {
|
||||
shared := sharesNamedTopic(query, text)
|
||||
if requireNamedTopic && !corroboratesNamedIdentity(query, text) {
|
||||
return false
|
||||
}
|
||||
if !openQuestion && !shared {
|
||||
return false
|
||||
}
|
||||
if mentionsHim(query) {
|
||||
return true
|
||||
}
|
||||
return SharesContentWord(query, text)
|
||||
return shared || len(contentWords(query)) == 0
|
||||
}
|
||||
|
||||
// corroboratesNamedIdentity requires every non-frame, non-verb query term to
|
||||
// occur in the candidate proposition's subject as the same dictionary word.
|
||||
// Searching the whole candidate is unsafe: in "the spare key lies in the blue
|
||||
// box", the box is a location, not the thing being located, and cannot answer
|
||||
// "where is the blue box?". Locative predicates are deliberately excluded, so
|
||||
// two memories do not become the same target merely because both things "lie"
|
||||
// somewhere. Requiring all remaining terms preserves qualifiers too: a key is
|
||||
// not a car key and a box is not a box of documents.
|
||||
func corroboratesNamedIdentity(query, text string) bool {
|
||||
q := identityWords(query)
|
||||
if len(q) == 0 {
|
||||
return false
|
||||
}
|
||||
t := propositionSubjectIdentity(text)
|
||||
if len(t) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, want := range q {
|
||||
found := false
|
||||
for _, got := range t {
|
||||
if want == got || morph.SameWord(want, got) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// propositionSubjectIdentity returns the identity phrase before the first
|
||||
// dictionary-proven main verb. A leading capture imperative is storage frame,
|
||||
// not the proposition: "remember: the spare key lies ..." has "spare key" as
|
||||
// its subject. If no subject/predicate boundary can be proved, it returns nil;
|
||||
// strict locative recall then abstains instead of treating a location object or
|
||||
// incidental modifier anywhere in the note as the requested entity.
|
||||
func propositionSubjectIdentity(s string) []string {
|
||||
toks := wordTokens(s)
|
||||
start := 0
|
||||
for start < len(toks) && recallCaptureVerbs[toks[start]] {
|
||||
start++
|
||||
}
|
||||
predicate := -1
|
||||
for i := start; i < len(toks); i++ {
|
||||
if morph.IsVerbForm(toks[i]) {
|
||||
predicate = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if predicate <= start {
|
||||
return nil
|
||||
}
|
||||
return identityTokens(toks[start:predicate])
|
||||
}
|
||||
|
||||
func identityWords(s string) []string {
|
||||
return identityTokens(contentWords(s))
|
||||
}
|
||||
|
||||
func identityTokens(words []string) []string {
|
||||
out := make([]string, 0, len(words))
|
||||
for _, w := range words {
|
||||
if !stopwords[w] && !morph.IsVerbForm(w) {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mentionsHim(query string) bool {
|
||||
@@ -96,7 +202,21 @@ func SharesContentWord(query, text string) bool {
|
||||
// gate is then the only judge it can have.
|
||||
return true
|
||||
}
|
||||
t := contentWords(text)
|
||||
return sharesContentWords(q, contentWords(text))
|
||||
}
|
||||
|
||||
// sharesNamedTopic is the stricter form used for a non-question-shaped turn:
|
||||
// an utterance made entirely of frame words names no topic, so it cannot use
|
||||
// the score gate as its only evidence that a stored note should be spoken.
|
||||
func sharesNamedTopic(query, text string) bool {
|
||||
q := contentWords(query)
|
||||
if len(q) == 0 {
|
||||
return false
|
||||
}
|
||||
return sharesContentWords(q, contentWords(text))
|
||||
}
|
||||
|
||||
func sharesContentWords(q, t []string) bool {
|
||||
for _, a := range q {
|
||||
for _, b := range t {
|
||||
if a == b || sameStem(a, b) {
|
||||
@@ -109,9 +229,7 @@ func SharesContentWord(query, text string) bool {
|
||||
|
||||
func contentWords(s string) []string {
|
||||
var out []string
|
||||
for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||||
}) {
|
||||
for _, w := range wordTokens(s) {
|
||||
if !stopwords[w] {
|
||||
out = append(out, w)
|
||||
}
|
||||
@@ -119,6 +237,12 @@ func contentWords(s string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func wordTokens(s string) []string {
|
||||
return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||||
})
|
||||
}
|
||||
|
||||
// sameStem is inflection and derivation tolerance: Russian marks case and
|
||||
// tense on the ending, and the note and the question rarely use the same form.
|
||||
// "воду" and "вода" are the same water, "кормить" and "корм" the same feeding.
|
||||
|
||||
@@ -6,27 +6,56 @@ func TestRecallAllowed(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
query, text string
|
||||
openQuestion, requireNamedTopic bool
|
||||
want bool
|
||||
}{
|
||||
// The #470 shape: a world question and a note about his box.
|
||||
{"world question, unrelated note", "почему небо синее", "сеть какая-то медленная", false},
|
||||
{"world question, unrelated fact", "какая столица Франции", "какая последняя версия языка Go", false},
|
||||
{"silent fixture case", "во сколько отходит поезд", "бэкап запускается в три ночи", false},
|
||||
{"world question, unrelated note", "почему небо синее", "сеть какая-то медленная", true, false, false},
|
||||
{"world question, unrelated fact", "какая столица Франции", "какая последняя версия языка Go", true, false, false},
|
||||
{"silent fixture case", "во сколько отходит поезд", "бэкап запускается в три ночи", true, false, false},
|
||||
|
||||
// A world question that does name the topic keeps its answer.
|
||||
{"world question, same topic", "какой поезд идёт в Минск", "поезда в Минск ходят утром", true},
|
||||
{"world question, same topic", "какой поезд идёт в Минск", "поезда в Минск ходят утром", true, false, true},
|
||||
|
||||
// A question about his own life is judged by the embedder alone,
|
||||
// because recall exists for words he no longer remembers.
|
||||
{"about him, no shared word", "во сколько я обычно засыпаю", "ложусь около одиннадцати", true},
|
||||
{"about him, english", "which colour scheme do i like", "тёмная тема везде", true},
|
||||
{"about him, no shared word", "во сколько я обычно засыпаю", "ложусь около одиннадцати", true, false, true},
|
||||
{"about him, english", "which colour scheme do i like", "тёмная тема везде", true, false, true},
|
||||
|
||||
// A locative asks for the named object's or event's location. Even a
|
||||
// personal question must corroborate every identity term in that target;
|
||||
// an incidental adjective or noun is insufficient (V-719).
|
||||
{"locative, unrelated high hit", "где мой паспорт?", "запомни: запасной ключ лежит в синей коробке", true, true, false},
|
||||
{"locative event, unrelated high hit", "где я отменил напоминание про молоко?", "запомни: запасной ключ лежит в синей коробке", true, true, false},
|
||||
{"locative, shared adjective only", "где лежит синяя рубашка?", "запасной ключ лежит в синей коробке", true, true, false},
|
||||
{"locative, shared modifier only", "где мой запасной паспорт?", "запасной ключ лежит в синей коробке", true, true, false},
|
||||
{"locative, shared head missing qualifier", "где лежит ключ от машины?", "запасной ключ лежит в синей коробке", true, true, false},
|
||||
{"locative, shared phrase missing complement", "где находится синяя коробка с документами?", "запасной ключ лежит в синей коробке", true, true, false},
|
||||
{"locative, location object is not subject", "где синяя коробка?", "запасной ключ лежит в синей коробке", true, true, false},
|
||||
{"locative, shared target", "где лежит запасной ключ?", "запасной ключ лежит в синей коробке", true, true, true},
|
||||
{"locative, capture prefix is not subject", "где лежит запасной ключ?", "запомни: запасной ключ лежит в синей коробке", true, true, true},
|
||||
|
||||
// Inflection must not break a match.
|
||||
{"inflected", "чем кормить кота", "корм для кота в шкафу", true},
|
||||
{"inflected", "чем кормить кота", "корм для кота в шкафу", true, false, true},
|
||||
|
||||
// A model-routed query without structural question evidence must still
|
||||
// name the hit's topic. This is the cold-start false recall seen in the
|
||||
// whole-assistant E2E, and its negated neighbour.
|
||||
{"declarative report is not a recall", "я отменил напоминание про молоко", "запомни: запасной ключ лежит в синей коробке", false, false, false},
|
||||
{"polar punctuation does not waive the topic", "я отменил напоминание про молоко?", "запомни: запасной ключ лежит в синей коробке", false, false, false},
|
||||
{"negated declarative report is not a recall", "я не отменил напоминание про молоко", "запомни: запасной ключ лежит в синей коробке", false, false, false},
|
||||
{"negated polar report is not a recall", "я не отменил напоминание про молоко?", "запомни: запасной ключ лежит в синей коробке", false, false, false},
|
||||
|
||||
// Question marks are optional in voice transcripts. A nominal request
|
||||
// and an intonational personal query remain eligible when their stored
|
||||
// answer corroborates the named topic.
|
||||
{"nominal request with a topic", "адрес домашнего сервера", "домашний сервер на 192.168.1.104", false, false, true},
|
||||
{"intonational query with a topic", "я сегодня вообще пил воду", "выпил стакан воды утром", false, false, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := RecallAllowed(c.query, c.text); got != c.want {
|
||||
t.Errorf("%s: RecallAllowed(%q, %q) = %v, want %v", c.name, c.query, c.text, got, c.want)
|
||||
if got := RecallAllowed(c.query, c.text, c.openQuestion, c.requireNamedTopic); got != c.want {
|
||||
t.Errorf("%s: RecallAllowed(%q, %q, open-question=%t, require-topic=%t) = %v, want %v",
|
||||
c.name, c.query, c.text, c.openQuestion, c.requireNamedTopic, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -39,10 +68,10 @@ func TestRecallAllowed(t *testing.T) {
|
||||
// which puts false recall back to 1/5. Anyone loosening the veto has to move
|
||||
// the first line without moving the second.
|
||||
func TestRecallVetoTradeIsPinned(t *testing.T) {
|
||||
if RecallAllowed("what fixed the screen problem", "the flicker went away once i swapped the display cable") {
|
||||
if RecallAllowed("what fixed the screen problem", "the flicker went away once i swapped the display cable", true, false) {
|
||||
t.Error("en-hard-024 is expected to stay vetoed — if this passes now, re-measure false recall before celebrating")
|
||||
}
|
||||
if RecallAllowed("во сколько отходит поезд", "погулял вдоль реки") {
|
||||
if RecallAllowed("во сколько отходит поезд", "погулял вдоль реки", true, false) {
|
||||
t.Error("ru-silent-029 must stay vetoed — this is the false recall the veto exists to stop")
|
||||
}
|
||||
}
|
||||
@@ -50,7 +79,7 @@ func TestRecallVetoTradeIsPinned(t *testing.T) {
|
||||
// A question made only of filler has no topic word to match on, and the score
|
||||
// gate is then the only judge it can have.
|
||||
func TestRecallAllowedFallsBackWhenNothingToCompare(t *testing.T) {
|
||||
if !RecallAllowed("что это", "сеть какая-то медленная") {
|
||||
if !RecallAllowed("что это", "сеть какая-то медленная", true, false) {
|
||||
t.Error("a question with no content word must not be vetoed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/kami/maven/internal/llm"
|
||||
)
|
||||
|
||||
// LocativeCompleter is the narrow resident-model seam the verifier needs.
|
||||
// *llm.Client satisfies it. Keeping the interface here lets the safety and
|
||||
// malformed-output paths run without a server in unit tests.
|
||||
type LocativeCompleter interface {
|
||||
Complete(context.Context, llm.Req) (string, error)
|
||||
}
|
||||
|
||||
// LocativeAnswerVerifier is the model-backed second opinion for candidates the
|
||||
// deterministic locative identity gate rejected. It never replaces that gate:
|
||||
// exact structural accepts do not call it, and a missing model, timeout, error,
|
||||
// or malformed verdict remains an abstention in the daemon.
|
||||
type LocativeAnswerVerifier struct {
|
||||
c LocativeCompleter
|
||||
}
|
||||
|
||||
// LocativeVerdict keeps the two extracted referents as auditable evidence.
|
||||
// The daemon consumes only Answerable; the live eval records all three fields
|
||||
// so a yes/no score cannot hide what the model thought it was comparing.
|
||||
type LocativeVerdict struct {
|
||||
Target string `json:"target"`
|
||||
MemorySubject string `json:"memory_subject"`
|
||||
Answerable bool `json:"-"`
|
||||
Raw string `json:"-"`
|
||||
}
|
||||
|
||||
// NewLocativeAnswerVerifier returns nil when no resident completion seam exists.
|
||||
// That is the ordinary no-model deployment and deliberately means abstain.
|
||||
func NewLocativeAnswerVerifier(c LocativeCompleter) *LocativeAnswerVerifier {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return &LocativeAnswerVerifier{c: c}
|
||||
}
|
||||
|
||||
// locativeVerifierGrammar fixes both the shape and every variable-width field.
|
||||
// target and memory_subject come before answer deliberately: the small resident
|
||||
// model must identify the two referents before choosing the verdict instead of
|
||||
// emitting an unconstrained first-token hunch.
|
||||
const locativeVerifierGrammar = `
|
||||
root ::= "{" ws "\"target\"" ws ":" ws string "," ws "\"memory_subject\"" ws ":" ws string "," ws "\"answer\"" ws ":" ws answer ws "}"
|
||||
answer ::= "\"no\"" | "\"yes\""
|
||||
string ::= "\"" ([^"\\\x00-\x1F] | "\\" ["\\/bfnrt] | "\\u" [0-9a-fA-F]{4}){1,80} "\""
|
||||
ws ::= [ \t\n]{0,2}
|
||||
`
|
||||
|
||||
// locativeVerifierSystem teaches a relation, not the held-out fixture. The
|
||||
// examples use different entities from the measured nginx/token/disk/key and
|
||||
// adversarial passport/shirt/box cases. Two negative examples pin the dangerous
|
||||
// distinctions: a location object is not the proposition subject, and a shared
|
||||
// generic noun with a conflicting complement is not the same referent.
|
||||
const locativeVerifierSystem = `Ты — строгий классификатор логического следования для личной памяти. Вход — JSON с одним question и одним memory.
|
||||
Сначала выдели target: конкретный предмет/событие, чьё место или источник спрашивают. Затем memory_subject: предмет/событие, МЕСТО КОТОРОГО сообщает память. Предмет после слов места (в, на, под, рядом с, inside, at, under) — это место/контейнер, а НЕ memory_subject.
|
||||
answer=yes только если target и memory_subject — один и тот же конкретный референт и память прямо сообщает запрошенное место/источник. Настоящие синонимы и контекстные названия допустимы. Совпадение цвета, свойства, общего слова, контейнера, места или действия недостаточно. Уточнения принадлежности/состава не должны конфликтовать. Не используй внешние знания. Сомнение => no.
|
||||
|
||||
Примеры:
|
||||
input: {"question":"где красная тетрадь?","memory":"зарядка лежит на красной тетради"}
|
||||
output: {"target":"красная тетрадь","memory_subject":"зарядка","answer":"no"}
|
||||
input: {"question":"где ключ от гаража?","memory":"ключ от офиса лежит под ковриком"}
|
||||
output: {"target":"ключ от гаража","memory_subject":"ключ от офиса","answer":"no"}
|
||||
input: {"question":"где дубликат ключа от мастерской?","memory":"запасной ключ мастерской лежит в ящике"}
|
||||
output: {"target":"дубликат ключа от мастерской","memory_subject":"запасной ключ мастерской","answer":"yes"}
|
||||
input: {"question":"where are the database settings?","memory":"the database configuration is in /etc/db"}
|
||||
output: {"target":"database settings","memory_subject":"database configuration","answer":"yes"}
|
||||
|
||||
Верни только JSON требуемой формы.`
|
||||
|
||||
const (
|
||||
locativeVerifierTimeout = 8 * time.Second
|
||||
locativeVerifierMaxTokens = 128
|
||||
locativeVerifierMaxField = 80
|
||||
)
|
||||
|
||||
// Answerable implements the daemon's deliberately tiny verifier interface.
|
||||
func (v *LocativeAnswerVerifier) Answerable(ctx context.Context, question, candidate string) (bool, error) {
|
||||
verdict, err := v.Evaluate(ctx, question, candidate)
|
||||
return verdict.Answerable, err
|
||||
}
|
||||
|
||||
// Evaluate returns the bounded model verdict and its extracted referents.
|
||||
// Callers must treat every error as false; it never manufactures a fallback.
|
||||
func (v *LocativeAnswerVerifier) Evaluate(ctx context.Context, question, candidate string) (LocativeVerdict, error) {
|
||||
if v == nil || v.c == nil {
|
||||
return LocativeVerdict{}, errors.New("locative verifier: resident model unavailable")
|
||||
}
|
||||
input, err := json.Marshal(struct {
|
||||
Question string `json:"question"`
|
||||
Memory string `json:"memory"`
|
||||
}{Question: question, Memory: candidate})
|
||||
if err != nil {
|
||||
return LocativeVerdict{}, fmt.Errorf("locative verifier: encode input: %w", err)
|
||||
}
|
||||
|
||||
callCtx, cancel := context.WithTimeout(ctx, locativeVerifierTimeout)
|
||||
defer cancel()
|
||||
raw, err := v.c.Complete(callCtx, llm.Req{
|
||||
System: locativeVerifierSystem,
|
||||
User: string(input),
|
||||
Grammar: locativeVerifierGrammar,
|
||||
MaxTokens: locativeVerifierMaxTokens,
|
||||
RepeatPenalty: 1.1,
|
||||
})
|
||||
if err != nil {
|
||||
return LocativeVerdict{}, fmt.Errorf("locative verifier: complete: %w", err)
|
||||
}
|
||||
return parseLocativeVerdict(raw)
|
||||
}
|
||||
|
||||
func parseLocativeVerdict(raw string) (LocativeVerdict, error) {
|
||||
trimmed := strings.TrimSpace(raw)
|
||||
var wire struct {
|
||||
Target string `json:"target"`
|
||||
MemorySubject string `json:"memory_subject"`
|
||||
Answer string `json:"answer"`
|
||||
}
|
||||
dec := json.NewDecoder(strings.NewReader(trimmed))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&wire); err != nil {
|
||||
return LocativeVerdict{}, fmt.Errorf("locative verifier: parse %q: %w", boundedRaw(trimmed), err)
|
||||
}
|
||||
var trailing any
|
||||
if err := dec.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
if err == nil {
|
||||
return LocativeVerdict{}, fmt.Errorf("locative verifier: trailing JSON in %q", boundedRaw(trimmed))
|
||||
}
|
||||
return LocativeVerdict{}, fmt.Errorf("locative verifier: trailing data in %q: %w", boundedRaw(trimmed), err)
|
||||
}
|
||||
wire.Target = strings.TrimSpace(wire.Target)
|
||||
wire.MemorySubject = strings.TrimSpace(wire.MemorySubject)
|
||||
if wire.Target == "" || wire.MemorySubject == "" ||
|
||||
utf8.RuneCountInString(wire.Target) > locativeVerifierMaxField ||
|
||||
utf8.RuneCountInString(wire.MemorySubject) > locativeVerifierMaxField {
|
||||
return LocativeVerdict{}, fmt.Errorf("locative verifier: empty or oversized referent in %q", boundedRaw(trimmed))
|
||||
}
|
||||
var answerable bool
|
||||
switch wire.Answer {
|
||||
case "yes":
|
||||
answerable = true
|
||||
case "no":
|
||||
answerable = false
|
||||
default:
|
||||
return LocativeVerdict{}, fmt.Errorf("locative verifier: invalid answer %q", wire.Answer)
|
||||
}
|
||||
return LocativeVerdict{
|
||||
Target: wire.Target,
|
||||
MemorySubject: wire.MemorySubject,
|
||||
Answerable: answerable,
|
||||
Raw: trimmed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func boundedRaw(s string) string {
|
||||
const max = 240
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + "…"
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/llm"
|
||||
)
|
||||
|
||||
type locativeCompleteFunc func(context.Context, llm.Req) (string, error)
|
||||
|
||||
func (f locativeCompleteFunc) Complete(ctx context.Context, req llm.Req) (string, error) {
|
||||
return f(ctx, req)
|
||||
}
|
||||
|
||||
func TestLocativeAnswerVerifierRequestAndVerdict(t *testing.T) {
|
||||
var got llm.Req
|
||||
v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(_ context.Context, req llm.Req) (string, error) {
|
||||
got = req
|
||||
return `{"target":"настройки nginx","memory_subject":"конфиг nginx","answer":"yes"}`, nil
|
||||
}))
|
||||
verdict, err := v.Evaluate(context.Background(), "где настройки nginx?", "конфиг nginx лежит в /etc/nginx")
|
||||
if err != nil {
|
||||
t.Fatalf("Evaluate: %v", err)
|
||||
}
|
||||
if !verdict.Answerable || verdict.Target != "настройки nginx" || verdict.MemorySubject != "конфиг nginx" {
|
||||
t.Fatalf("verdict = %+v", verdict)
|
||||
}
|
||||
if got.Grammar != locativeVerifierGrammar || got.MaxTokens != locativeVerifierMaxTokens || got.RepeatPenalty != 1.1 {
|
||||
t.Fatalf("request bounds drifted: %+v", got)
|
||||
}
|
||||
var input map[string]string
|
||||
if err := json.Unmarshal([]byte(got.User), &input); err != nil {
|
||||
t.Fatalf("user input is not JSON: %v", err)
|
||||
}
|
||||
if len(input) != 2 || input["question"] != "где настройки nginx?" || input["memory"] != "конфиг nginx лежит в /etc/nginx" {
|
||||
t.Fatalf("model saw fields outside question+memory: %#v", input)
|
||||
}
|
||||
if strings.Contains(got.System, "nginx") {
|
||||
t.Fatal("held-out entity leaked into the static verifier prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocativeAnswerVerifierNoAndMalformedFailClosed(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
raw string
|
||||
want bool
|
||||
err bool
|
||||
}{
|
||||
{"no", `{"target":"паспорт","memory_subject":"ключ","answer":"no"}`, false, false},
|
||||
{"bare yes", `yes`, false, true},
|
||||
{"unknown answer", `{"target":"a","memory_subject":"b","answer":"maybe"}`, false, true},
|
||||
{"empty target", `{"target":"","memory_subject":"b","answer":"yes"}`, false, true},
|
||||
{"extra field", `{"target":"a","memory_subject":"b","answer":"yes","why":"guess"}`, false, true},
|
||||
{"trailing object", `{"target":"a","memory_subject":"b","answer":"yes"}{}`, false, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(context.Context, llm.Req) (string, error) {
|
||||
return tc.raw, nil
|
||||
}))
|
||||
got, err := v.Answerable(context.Background(), "q", "m")
|
||||
if got != tc.want || (err != nil) != tc.err {
|
||||
t.Fatalf("Answerable = %v, %v; want %v, err=%v", got, err, tc.want, tc.err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocativeAnswerVerifierUnavailableErrorAndTimeout(t *testing.T) {
|
||||
if v := NewLocativeAnswerVerifier(nil); v != nil {
|
||||
t.Fatal("nil resident model produced a verifier")
|
||||
}
|
||||
boom := errors.New("llama down")
|
||||
v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(context.Context, llm.Req) (string, error) {
|
||||
return "", boom
|
||||
}))
|
||||
if ok, err := v.Answerable(context.Background(), "q", "m"); ok || !errors.Is(err, boom) {
|
||||
t.Fatalf("model error = %v, %v; want false wrapping %v", ok, err, boom)
|
||||
}
|
||||
|
||||
v = NewLocativeAnswerVerifier(locativeCompleteFunc(func(ctx context.Context, _ llm.Req) (string, error) {
|
||||
<-ctx.Done()
|
||||
return "", ctx.Err()
|
||||
}))
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
|
||||
defer cancel()
|
||||
if ok, err := v.Answerable(ctx, "q", "m"); ok || !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("timeout = %v, %v; want false deadline", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocativeVerifierGrammarIsFullyBounded(t *testing.T) {
|
||||
if strings.Contains(locativeVerifierGrammar, "*") || strings.Contains(locativeVerifierGrammar, "+") {
|
||||
t.Fatalf("grammar contains an unbounded repetition:\n%s", locativeVerifierGrammar)
|
||||
}
|
||||
for _, bound := range []string{"{1,80}", "{0,2}"} {
|
||||
if !strings.Contains(locativeVerifierGrammar, bound) {
|
||||
t.Errorf("grammar missing bound %s", bound)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -473,7 +473,8 @@ func bestRecall(query string, results []memory.Result, minScore, minMargin float
|
||||
return ""
|
||||
}
|
||||
text := results[0].Meta["text"]
|
||||
if !memory.RecallAllowed(query, text) {
|
||||
if !memory.RecallAllowed(query, text,
|
||||
router.IsOpenQuestionShaped(query), router.IsLocativeQuestionShaped(query)) {
|
||||
return ""
|
||||
}
|
||||
return text
|
||||
|
||||
@@ -75,8 +75,8 @@ func TestLoadFixture(t *testing.T) {
|
||||
}
|
||||
}
|
||||
// Both lanes need enough cases that a rate means something.
|
||||
if silent < 5 {
|
||||
t.Errorf("%d must-be-silent cases, want >= 5", silent)
|
||||
if silent < 15 {
|
||||
t.Errorf("%d must-be-silent cases, want >= 15", silent)
|
||||
}
|
||||
if en < 5 {
|
||||
t.Errorf("%d English cases, want >= 5", en)
|
||||
@@ -176,6 +176,23 @@ func TestBestRecallMatchesDaemon(t *testing.T) {
|
||||
if got := bestRecall("почему небо синее", offTopic, 0.55, 0); got != "" {
|
||||
t.Errorf("off topic: got %q, want silence", got)
|
||||
}
|
||||
// A statistical query route is not by itself evidence that an ordinary
|
||||
// first-person report asks for a stored note. This is deliberately one hit:
|
||||
// a fresh Maven has no runner-up, so only the absolute and structural gates
|
||||
// can stop the live cold-start false recall.
|
||||
report := []memory.Result{{ID: "a", Score: 0.825031306, Meta: map[string]string{"text": "запомни: запасной ключ лежит в синей коробке"}}}
|
||||
if got := bestRecall("я отменил напоминание про молоко", report, 0.55, 0.008); got != "" {
|
||||
t.Errorf("declarative report: got %q, want silence", got)
|
||||
}
|
||||
if got := bestRecall("я отменил напоминание про молоко?", report, 0.55, 0.008); got != "" {
|
||||
t.Errorf("polar report: got %q, want silence", got)
|
||||
}
|
||||
// A voice transcript can omit punctuation. A nominal request that shares
|
||||
// the answer's topic stays eligible.
|
||||
nominal := []memory.Result{{ID: "a", Score: 0.90, Meta: map[string]string{"text": "домашний сервер на 192.168.1.104"}}}
|
||||
if got := bestRecall("адрес домашнего сервера", nominal, 0.80, 0.008); got != "домашний сервер на 192.168.1.104" {
|
||||
t.Errorf("nominal topic request: got %q, want the server note", got)
|
||||
}
|
||||
clear := []memory.Result{
|
||||
{ID: "a", Score: 0.86, Meta: map[string]string{"text": "чай"}},
|
||||
{ID: "b", Score: 0.70, Meta: map[string]string{"text": "кофе"}},
|
||||
@@ -265,9 +282,9 @@ func sqliteStores(t *testing.T) NewStore {
|
||||
// like TestONNXBaseline in internal/router/eval. `make eval-recall` points it at
|
||||
// the vendored runtime.
|
||||
//
|
||||
// Reports rather than asserts. The gate sweep is the point: it prints
|
||||
// answered-vs-false-recall at a range of query_min_score values, so the right
|
||||
// threshold is read off data instead of guessed.
|
||||
// Reports the full distributions and pins only the two operator-facing safety
|
||||
// ratchets: do not lose the measured answering floor, and never read a note on
|
||||
// a must-be-silent case. Exact scores stay observable rather than asserted.
|
||||
func TestONNXRecall(t *testing.T) {
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
@@ -295,11 +312,36 @@ func TestONNXRecall(t *testing.T) {
|
||||
t.Fatalf("Score: %v", err)
|
||||
}
|
||||
t.Log("\n" + rep.String() + rep.Failures())
|
||||
var semanticOnly []string
|
||||
for _, o := range rep.Outcomes {
|
||||
if !o.Case.Answerable() && len(o.Hits) > 0 {
|
||||
t.Logf("silent candidate %s score=%.6f margin=%.6f query=%q memory=%q",
|
||||
o.Case.ID, o.TopScor, o.Margin, o.Case.Query, o.Hits[0].Meta["text"])
|
||||
}
|
||||
if o.Case.Answerable() && o.Rank1 &&
|
||||
!memory.SharesContentWord(o.Case.Query, o.Hits[0].Meta["text"]) {
|
||||
semanticOnly = append(semanticOnly, o.Case.ID)
|
||||
}
|
||||
}
|
||||
t.Logf("right-note rank-1 cases with semantic-only (no lexical topic) evidence: %s",
|
||||
strings.Join(semanticOnly, ", "))
|
||||
// V-719 closes the single-hit locative failure class by requiring every
|
||||
// named identity term to be corroborated. The real e5 distributions overlap:
|
||||
// four true locative paraphrases and six false locative neighbours cannot be
|
||||
// separated by score or target-phrase cosine. The deterministic floor is 14;
|
||||
// recovering those four safely needs a separate answerability verifier, and
|
||||
// an absent or failed verifier must keep this fail-closed result.
|
||||
if answered := rep.Rank1 - rep.Gated; answered < 14 {
|
||||
t.Errorf("answered %d/%d, want at least 14 strict-floor recalls", answered, rep.Answerable)
|
||||
}
|
||||
if rep.FalseRecall != 0 {
|
||||
t.Errorf("false recall %d/%d, want zero:\n%s", rep.FalseRecall, rep.NoAnswer, rep.Failures())
|
||||
}
|
||||
// Cached for the sweeps only: the headline run above must pay the real
|
||||
// embedder cost so its latency numbers mean something.
|
||||
cached := Cache(emb)
|
||||
t.Log("\ngate sweep (margin off):\n" + sweep(t, cached, f))
|
||||
t.Log("\nmargin sweep (gate 0.55):\n" + marginSweep(t, cached, f))
|
||||
t.Logf("\nmargin sweep (gate %.2f):\n%s", config.DefaultQueryMinScore, marginSweep(t, cached, f))
|
||||
}
|
||||
|
||||
// sweep scores the fixture at a range of gates and renders one line each. Two
|
||||
@@ -321,9 +363,8 @@ func sweep(t *testing.T, emb router.Embedder, f Fixture) string {
|
||||
}
|
||||
|
||||
// marginSweep is the same idea for the margin gate (top1 − top2 > delta), with
|
||||
// the absolute gate held at its default. The absolute score cannot separate a
|
||||
// real hit from a made-up question under e5 — every score lands in one narrow
|
||||
// band — so this sweep is the one that picks a number.
|
||||
// the absolute gate held at its default. Neither axis separates every case on
|
||||
// its own under e5's narrow score band; the deployed pair is calibrated jointly.
|
||||
func marginSweep(t *testing.T, emb router.Embedder, f Fixture) string {
|
||||
t.Helper()
|
||||
var b strings.Builder
|
||||
|
||||
@@ -413,6 +413,136 @@
|
||||
{"id": "n1", "text": "на заправке у моста дешевле бензин", "kind": "note"},
|
||||
{"id": "n2", "text": "надо поменять зимние шины", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-033",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard"],
|
||||
"note": "Observed through the whole-assistant E2E with only n1 stored: routing called an ordinary past-tense report a query and the single-hit recall gate spoke n1 at cosine 0.825. A report is not a request to read a semantically nearby note.",
|
||||
"query": "я отменил напоминание про молоко",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-034",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard", "locative"],
|
||||
"note": "V-719 whole-assistant adversarial: with only n1 stored, the open personal question scored 0.817 against it. An explicit location request must not substitute a semantically nearby object's location for the named target.",
|
||||
"query": "где мой паспорт?",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-035",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard", "locative"],
|
||||
"note": "V-719 adversarial neighbour: a locative interrogative must not reopen the report-shaped false recall; the unrelated single note scored 0.806.",
|
||||
"query": "где я отменил напоминание про молоко?",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-036",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard", "locative", "incidental-overlap"],
|
||||
"note": "V-719 adversarial: blue is only an incidental shared modifier; shirt and spare key are different targets despite cosine 0.838.",
|
||||
"query": "где лежит синяя рубашка?",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-037",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard", "locative", "incidental-overlap"],
|
||||
"note": "V-719 adversarial: blue is only an incidental shared modifier; folder and spare key are different targets despite cosine 0.837.",
|
||||
"query": "где лежит синяя папка?",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-038",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard", "locative", "incidental-overlap"],
|
||||
"note": "V-719 adversarial: spare is only an incidental shared modifier; passport and key are different targets despite cosine 0.832.",
|
||||
"query": "где мой запасной паспорт?",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-039",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard", "locative", "incidental-overlap"],
|
||||
"note": "V-719 adversarial: spare is only an incidental shared modifier; flash drive and key are different targets despite cosine 0.839.",
|
||||
"query": "где лежит запасная флешка?",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-040",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard", "locative", "incidental-overlap"],
|
||||
"note": "V-719 adversarial: blue box overlaps, but the candidate does not corroborate the documents qualifier despite cosine 0.867.",
|
||||
"query": "где находится синяя коробка с документами?",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-041",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard", "locative", "incidental-overlap"],
|
||||
"note": "V-719 adversarial: key overlaps, but the candidate does not corroborate the car qualifier despite cosine 0.844.",
|
||||
"query": "где лежит ключ от машины?",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ru-silent-042",
|
||||
"lang": "ru",
|
||||
"tags": ["silent", "hard", "locative", "location-object"],
|
||||
"note": "V-719 adversarial: the requested blue box occurs in n1 only as the spare key's location, not as the proposition subject. Arbitrary whole-note overlap must not turn it into the box's own location.",
|
||||
"query": "где синяя коробка?",
|
||||
"want": "",
|
||||
"notes": [
|
||||
{"id": "n1", "text": "запомни: запасной ключ лежит в синей коробке", "kind": "note"},
|
||||
{"id": "n2", "text": "домашний сервер на 192.168.1.104", "kind": "note"},
|
||||
{"id": "n3", "text": "чай пью только зелёный", "kind": "note"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user