5b0b29dfad
The spare-key note scored 0.832 to 0.867 against a spare passport, a blue shirt, a blue document box and a car key. Score and margin cannot separate those: the right note runs 0.817 to 0.892 and the silent cases 0.787 to 0.874, so the ranges overlap and structure has to decide. RecallAllowed now takes two structural facts from the router. A locative question must corroborate every identity term against the candidate's subject, read up to its first dictionary-proven verb, so a location object in the note cannot answer for the thing being located. A turn that is not question-shaped needs a named shared topic even when it ends in '?', which is what "я отменил напоминание про молоко" lacked when it recalled an unrelated note at 0.825 with no runner-up to fail the margin. query_min_score moves 0.55 to 0.80 for tokenizer rev 2. The held-out fixture answers 14/27 real recalls and 0/14 false ones. LocativeAnswerVerifier is the resident-model second opinion, kept behind the deterministic gate and wired into nothing. The measurement that says why is docs/evals/2026-08-15-locative-answerability-verifier.md. --no-verify: master is the working branch this session by the owner's call. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
86 lines
6.5 KiB
Go
86 lines
6.5 KiB
Go
package memory
|
|
|
|
import "testing"
|
|
|
|
func TestRecallAllowed(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
query, text string
|
|
openQuestion, requireNamedTopic bool
|
|
want bool
|
|
}{
|
|
// The #470 shape: a world question and a note about his box.
|
|
{"world question, unrelated note", "почему небо синее", "сеть какая-то медленная", true, false, false},
|
|
{"world question, unrelated fact", "какая столица Франции", "какая последняя версия языка Go", true, false, false},
|
|
{"silent fixture case", "во сколько отходит поезд", "бэкап запускается в три ночи", true, false, false},
|
|
|
|
// A world question that does name the topic keeps its answer.
|
|
{"world question, same topic", "какой поезд идёт в Минск", "поезда в Минск ходят утром", true, false, true},
|
|
|
|
// A question about his own life is judged by the embedder alone,
|
|
// because recall exists for words he no longer remembers.
|
|
{"about him, no shared word", "во сколько я обычно засыпаю", "ложусь около одиннадцати", true, false, true},
|
|
{"about him, english", "which colour scheme do i like", "тёмная тема везде", true, false, true},
|
|
|
|
// A locative asks for the named object's or event's location. Even a
|
|
// personal question must corroborate every identity term in that target;
|
|
// an incidental adjective or noun is insufficient (V-719).
|
|
{"locative, unrelated high hit", "где мой паспорт?", "запомни: запасной ключ лежит в синей коробке", true, true, false},
|
|
{"locative event, unrelated high hit", "где я отменил напоминание про молоко?", "запомни: запасной ключ лежит в синей коробке", true, true, false},
|
|
{"locative, shared adjective only", "где лежит синяя рубашка?", "запасной ключ лежит в синей коробке", true, true, false},
|
|
{"locative, shared modifier only", "где мой запасной паспорт?", "запасной ключ лежит в синей коробке", true, true, false},
|
|
{"locative, shared head missing qualifier", "где лежит ключ от машины?", "запасной ключ лежит в синей коробке", true, true, false},
|
|
{"locative, shared phrase missing complement", "где находится синяя коробка с документами?", "запасной ключ лежит в синей коробке", true, true, false},
|
|
{"locative, location object is not subject", "где синяя коробка?", "запасной ключ лежит в синей коробке", true, true, false},
|
|
{"locative, shared target", "где лежит запасной ключ?", "запасной ключ лежит в синей коробке", true, true, true},
|
|
{"locative, capture prefix is not subject", "где лежит запасной ключ?", "запомни: запасной ключ лежит в синей коробке", true, true, true},
|
|
|
|
// Inflection must not break a match.
|
|
{"inflected", "чем кормить кота", "корм для кота в шкафу", true, false, true},
|
|
|
|
// A model-routed query without structural question evidence must still
|
|
// name the hit's topic. This is the cold-start false recall seen in the
|
|
// whole-assistant E2E, and its negated neighbour.
|
|
{"declarative report is not a recall", "я отменил напоминание про молоко", "запомни: запасной ключ лежит в синей коробке", false, false, false},
|
|
{"polar punctuation does not waive the topic", "я отменил напоминание про молоко?", "запомни: запасной ключ лежит в синей коробке", false, false, false},
|
|
{"negated declarative report is not a recall", "я не отменил напоминание про молоко", "запомни: запасной ключ лежит в синей коробке", false, false, false},
|
|
{"negated polar report is not a recall", "я не отменил напоминание про молоко?", "запомни: запасной ключ лежит в синей коробке", false, false, false},
|
|
|
|
// Question marks are optional in voice transcripts. A nominal request
|
|
// and an intonational personal query remain eligible when their stored
|
|
// answer corroborates the named topic.
|
|
{"nominal request with a topic", "адрес домашнего сервера", "домашний сервер на 192.168.1.104", false, false, true},
|
|
{"intonational query with a topic", "я сегодня вообще пил воду", "выпил стакан воды утром", false, false, true},
|
|
}
|
|
for _, c := range cases {
|
|
if got := RecallAllowed(c.query, c.text, c.openQuestion, c.requireNamedTopic); got != c.want {
|
|
t.Errorf("%s: RecallAllowed(%q, %q, open-question=%t, require-topic=%t) = %v, want %v",
|
|
c.name, c.query, c.text, c.openQuestion, c.requireNamedTopic, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The known cost of the veto and the thing that pays for it, both measured on
|
|
// the held-out fixture with the real embedder (#496,
|
|
// docs/evals/2026-08-03-recall-topic-veto.md). The two are one lexical class:
|
|
// zero shared content words, no first-person marker, scores 0.826 against 0.835
|
|
// and margins 0.023 against 0.019. Recovering the first re-admits the second,
|
|
// which puts false recall back to 1/5. Anyone loosening the veto has to move
|
|
// the first line without moving the second.
|
|
func TestRecallVetoTradeIsPinned(t *testing.T) {
|
|
if RecallAllowed("what fixed the screen problem", "the flicker went away once i swapped the display cable", true, false) {
|
|
t.Error("en-hard-024 is expected to stay vetoed — if this passes now, re-measure false recall before celebrating")
|
|
}
|
|
if RecallAllowed("во сколько отходит поезд", "погулял вдоль реки", true, false) {
|
|
t.Error("ru-silent-029 must stay vetoed — this is the false recall the veto exists to stop")
|
|
}
|
|
}
|
|
|
|
// A question made only of filler has no topic word to match on, and the score
|
|
// gate is then the only judge it can have.
|
|
func TestRecallAllowedFallsBackWhenNothingToCompare(t *testing.T) {
|
|
if !RecallAllowed("что это", "сеть какая-то медленная", true, false) {
|
|
t.Error("a question with no content word must not be vetoed")
|
|
}
|
|
}
|