Make locative recall prove identity, not overlap (V-719)
The spare-key note scored 0.832 to 0.867 against a spare passport, a blue shirt, a blue document box and a car key. Score and margin cannot separate those: the right note runs 0.817 to 0.892 and the silent cases 0.787 to 0.874, so the ranges overlap and structure has to decide. RecallAllowed now takes two structural facts from the router. A locative question must corroborate every identity term against the candidate's subject, read up to its first dictionary-proven verb, so a location object in the note cannot answer for the thing being located. A turn that is not question-shaped needs a named shared topic even when it ends in '?', which is what "я отменил напоминание про молоко" lacked when it recalled an unrelated note at 0.825 with no runner-up to fail the margin. query_min_score moves 0.55 to 0.80 for tokenizer rev 2. The held-out fixture answers 14/27 real recalls and 0/14 false ones. LocativeAnswerVerifier is the resident-model second opinion, kept behind the deterministic gate and wired into nothing. The measurement that says why is docs/evals/2026-08-15-locative-answerability-verifier.md. --no-verify: master is the working branch this session by the owner's call. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -641,10 +641,11 @@ const (
|
||||
// небо синее?", because the right-note and must-be-silent score ranges overlap
|
||||
// and no threshold sits between them.
|
||||
func recallOnTopic(utterance, text string) bool {
|
||||
if memory.RecallAllowed(utterance, text) {
|
||||
if memory.RecallAllowed(utterance, text,
|
||||
router.IsOpenQuestionShaped(utterance), router.IsLocativeQuestionShaped(utterance)) {
|
||||
return true
|
||||
}
|
||||
log.Printf("voice: recall %q rejected for %q: a world question and no shared topic word", text, utterance)
|
||||
log.Printf("voice: recall %q rejected for %q: no structural ask with a shared topic, or a world/locative question with no shared topic", text, utterance)
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -193,9 +193,9 @@ func TestQueryRecallNoteCanWin(t *testing.T) {
|
||||
t.Run("the better-matching fact answers", func(t *testing.T) {
|
||||
h, _ := buildRecallHandler(t, q, []recallCase{
|
||||
{text: "молоко стоит в холодильнике", score: 0.80, kind: "note"},
|
||||
{text: "купил молоко в среду", score: 0.95, kind: "fact"},
|
||||
{text: "молоко было в холодильнике в среду", score: 0.95, kind: "fact"},
|
||||
})
|
||||
if reply := askQuery(t, h, q); reply != "купил молоко в среду" {
|
||||
if reply := askQuery(t, h, q); reply != "молоко было в холодильнике в среду" {
|
||||
t.Errorf("reply %q, want the fact read back", reply)
|
||||
}
|
||||
})
|
||||
@@ -212,3 +212,63 @@ func TestQueryRecallNoteCanWin(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestQueryRecallRequiresStructuralOrTopicEvidence — the whole-assistant
|
||||
// cold-start regression. The routing heads called an ordinary past-tense
|
||||
// report a query; with one note in the store the margin gate has no runner-up,
|
||||
// and cosine 0.825 was enough to speak a completely unrelated spare-key note.
|
||||
// A bare question mark does not turn the proposition into an open information
|
||||
// question, negation must not weaken the refusal, and a locative question must
|
||||
// corroborate the target it asks Maven to locate (V-719).
|
||||
func TestQueryRecallRequiresStructuralOrTopicEvidence(t *testing.T) {
|
||||
const unrelated = "запомни: запасной ключ лежит в синей коробке"
|
||||
for _, tc := range []struct {
|
||||
query string
|
||||
score float64
|
||||
}{
|
||||
{"я отменил напоминание про молоко", 0.825031306},
|
||||
{"я отменил напоминание про молоко?", 0.825031306},
|
||||
{"я не отменил напоминание про молоко", 0.825031306},
|
||||
{"я не отменил напоминание про молоко?", 0.825031306},
|
||||
{"где мой паспорт?", 0.817210},
|
||||
{"где я отменил напоминание про молоко?", 0.805800},
|
||||
{"где лежит синяя рубашка?", 0.837694},
|
||||
{"где лежит синяя папка?", 0.837472},
|
||||
{"где мой запасной паспорт?", 0.831662},
|
||||
{"где лежит запасная флешка?", 0.838980},
|
||||
{"где находится синяя коробка с документами?", 0.866553},
|
||||
{"где лежит ключ от машины?", 0.843853},
|
||||
{"где синяя коробка?", 0.90},
|
||||
} {
|
||||
t.Run(tc.query, func(t *testing.T) {
|
||||
h, phr := buildRecallHandler(t, tc.query, []recallCase{
|
||||
{text: unrelated, score: tc.score, kind: "note"},
|
||||
})
|
||||
reply := askQuery(t, h, tc.query)
|
||||
if strings.Contains(reply, "запасной ключ") {
|
||||
t.Fatalf("unrelated note escaped into reply %q", reply)
|
||||
}
|
||||
if len(phr.notes) != 0 {
|
||||
t.Fatalf("unrelated note reached the phraser: %q", phr.notes)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Voice punctuation is optional. A nominal request with no interrogative
|
||||
// still works when the candidate itself corroborates the named topic.
|
||||
const nominal = "адрес домашнего сервера"
|
||||
h, _ := buildRecallHandler(t, nominal, []recallCase{
|
||||
{text: "домашний сервер на 192.168.1.104", score: 0.90, kind: "note"},
|
||||
})
|
||||
if reply := askQuery(t, h, nominal); !strings.Contains(reply, "домашний сервер") {
|
||||
t.Fatalf("nominal recall lost its shared-topic answer: %q", reply)
|
||||
}
|
||||
|
||||
const locative = "где лежит запасной ключ?"
|
||||
h, _ = buildRecallHandler(t, locative, []recallCase{
|
||||
{text: "запасной ключ лежит в синей коробке", score: 0.90, kind: "note"},
|
||||
})
|
||||
if reply := askQuery(t, h, locative); !strings.Contains(reply, "запасной ключ") {
|
||||
t.Fatalf("locative recall lost its corroborated target: %q", reply)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/memory"
|
||||
"github.com/kami/maven/internal/phraser"
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/store"
|
||||
"github.com/kami/maven/internal/tool"
|
||||
@@ -84,12 +85,76 @@ func TestReactiveNotesReminders(t *testing.T) {
|
||||
t.Fatal("expected at least one note, got none")
|
||||
}
|
||||
last := notes[0]
|
||||
if last.Text != "запомни что кофе закончился" {
|
||||
t.Errorf("note text = %q, want %q", last.Text, "запомни что кофе закончился")
|
||||
if last.Text != "кофе закончился" {
|
||||
t.Errorf("note text = %q, want %q", last.Text, "кофе закончился")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestRunTurnExplicitNoteStoresOnlyTheBody pins the live failure end to end:
|
||||
// a routed text turn reaches actionNote, stores only the dictated body in both
|
||||
// durable and vector memory, and cannot ask the resident model to choose the
|
||||
// acknowledgement's grammatical gender (V-721).
|
||||
func TestRunTurnExplicitNoteStoresOnlyTheBody(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
api := ipc.NewStoreAPI(st)
|
||||
now := time.Date(2026, 8, 15, 8, 0, 0, 0, time.FixedZone("+04", 4*60*60))
|
||||
emb := router.NewHashEmbedder(1024)
|
||||
mem := memory.NewInMemoryStore()
|
||||
rtr := router.New(router.Config{
|
||||
Grammars: []router.Grammar{{
|
||||
Name: "explicit-note-test",
|
||||
Decide: func(string) (router.Decision, bool) {
|
||||
return router.Decision{
|
||||
Stage: 0, Intent: router.IntentNote, Confidence: 1,
|
||||
// Deliberately hostile model slot: neither persistence nor
|
||||
// acknowledgement may use it.
|
||||
Slots: router.Slots{Text: "ты поедешь на дачу"},
|
||||
}, true
|
||||
},
|
||||
}},
|
||||
Threshold: 0.55,
|
||||
})
|
||||
model := &countingCompleter{out: `{"response":"Хорошо, сохранил.","mood":"neutral"}`}
|
||||
h := &reactiveHandler{
|
||||
api: api, router: rtr,
|
||||
recall: recallWiring{embedder: emb, memStore: mem},
|
||||
replier: newLLMReplier(model, nil),
|
||||
now: func() time.Time { return now },
|
||||
dataStore: st,
|
||||
}
|
||||
|
||||
const utterance = "запомни: запасной ключ лежит в синей коробке"
|
||||
if reply := h.runTurn(ctx, utterance, sourceText); reply != "сохранила заметку." {
|
||||
t.Fatalf("reply = %q, want the fixed feminine acknowledgement", reply)
|
||||
}
|
||||
if model.calls != 0 {
|
||||
t.Fatalf("resident model was called %d time(s) for a note acknowledgement", model.calls)
|
||||
}
|
||||
notes, err := st.RecentNotes(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentNotes: %v", err)
|
||||
}
|
||||
const body = "запасной ключ лежит в синей коробке"
|
||||
if len(notes) != 1 || notes[0].Text != body || notes[0].Source != "tap:voice" || !notes[0].Ts.Equal(now) {
|
||||
t.Fatalf("stored notes = %+v, want one exact body at the turn time", notes)
|
||||
}
|
||||
records, err := mem.ByPrefix(ctx, "note:")
|
||||
if err != nil {
|
||||
t.Fatalf("vector catalog: %v", err)
|
||||
}
|
||||
if len(records) != 1 || records[0].Meta["text"] != body {
|
||||
t.Fatalf("vector records = %+v, want the same extracted body", records)
|
||||
}
|
||||
if records[0].Meta["text"] == utterance || records[0].Meta["text"] == "ты поедешь на дачу" {
|
||||
t.Fatalf("vector metadata used a command or model rewrite: %+v", records[0].Meta)
|
||||
}
|
||||
if !phraser.IsAck(phraser.AckNote, nil, "сохранила заметку.") {
|
||||
t.Fatal("fixed acknowledgement is not registered as the note acknowledgement")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSpokenTaskCaptureFilesATask — the whole path, from the utterance to the
|
||||
// task table. It went dead when the router started claiming the marker as an
|
||||
// act: capture rides the note intent, so nothing below actionNote was ever
|
||||
|
||||
@@ -56,7 +56,7 @@ type recallWiring struct {
|
||||
// minScore — the note-recall confidence gate. Top cosine below this ⇒
|
||||
// "I don't know" instead of a guess. Tuned for the ONNX embedder; a knob,
|
||||
// not load-bearing math (same posture as the presence thresholds). Set by
|
||||
// wireVoice from VoiceConfig; default 0.55.
|
||||
// wireVoice from VoiceConfig; default 0.80.
|
||||
minScore float64
|
||||
|
||||
// minMargin — the second half of that gate: how far the top hit must beat
|
||||
|
||||
Reference in New Issue
Block a user