From aec94eb2e8b86ea23d53cc592dc5e46e200ad639 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 3 Aug 2026 13:50:54 +0400 Subject: [PATCH] memory: a world question must name what the memory mentions (V-470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The score gate cannot separate the right note from an unrelated one: the held-out fixture puts the right note at 0.791-0.890 and the must-be-silent cases at 0.795-0.835, so a note about his slow network answered 'почему небо синее?'. RecallAllowed adds a topic veto, and applies it only to a question that mentions nothing of his. That restriction is the whole design: demanding a shared word of every recall silenced four true recalls on the fixture to kill one false one, because recall exists to find the note whose words he no longer remembers. A question about his own life keeps the embedder as its only judge. --- internal/memory/lexical.go | 126 ++++++++++++++++++++++++++++++++ internal/memory/lexical_test.go | 40 ++++++++++ 2 files changed, 166 insertions(+) create mode 100644 internal/memory/lexical.go create mode 100644 internal/memory/lexical_test.go diff --git a/internal/memory/lexical.go b/internal/memory/lexical.go new file mode 100644 index 0000000..2dc3161 --- /dev/null +++ b/internal/memory/lexical.go @@ -0,0 +1,126 @@ +package memory + +import ( + "strings" + "unicode" +) + +// stopwords — words that carry no topic. A question and a note that share only +// these share nothing: "почему небо синее" and "сеть какая-то медленная" both +// contain "какая"-shaped filler and are about different worlds. +var stopwords = map[string]bool{ + // interrogatives and demonstratives + "что": true, "чего": true, "какой": true, "какая": true, "какое": true, + "какие": true, "каких": true, "кто": true, "кого": true, "кому": true, + "почему": true, "зачем": true, "где": true, "куда": true, "откуда": true, + "когда": true, "сколько": true, "как": true, "то": true, "это": true, + "этот": true, "тот": true, "там": true, "тут": true, "такой": true, + // pronouns — every sentence he says is about him, so "я" is not a topic + "я": true, "меня": true, "мне": true, "мой": true, "моя": true, "мои": true, + "ты": true, "тебя": true, "тебе": true, "твой": true, "он": true, "она": true, + "они": true, "мы": true, "себя": true, "свой": true, + // prepositions, conjunctions, particles, copulas + "в": true, "во": true, "на": true, "с": true, "со": true, "у": true, + "о": true, "об": true, "про": true, "за": true, "из": true, "по": true, + "до": true, "от": true, "для": true, "над": true, "под": true, "при": true, + "и": true, "а": true, "но": true, "или": true, "же": true, "ли": true, + "не": true, "ни": true, "бы": true, "был": true, "была": true, "было": true, + "быть": true, "есть": true, "был-ли": true, "уже": true, "ещё": true, + "еще": true, "так": true, "вот": true, "там-же": true, + // English filler, for the mixed utterances he does say + "the": true, "a": true, "an": true, "is": true, "are": true, "was": true, + "were": true, "be": true, "of": true, "in": true, "on": true, "at": true, + "to": true, "for": true, "about": true, "and": true, "or": true, "not": true, + "what": true, "who": true, "why": true, "when": true, "where": true, + "which": true, "how": true, "i": true, "my": true, "me": true, "it": true, + "this": true, "that": true, +} + +// firstPerson — the words that make an utterance a question about his own +// life. Not possession only: "как я восстановил конфиги" owns nothing and is +// still about him. +var firstPerson = map[string]bool{ + "я": true, "меня": true, "мне": true, "мной": true, "мой": true, + "моя": true, "моё": true, "мое": true, "мои": true, "моего": true, + "моей": true, "моих": true, "моим": true, "себя": true, "свой": true, + "своя": true, "свои": true, "своего": true, "мною": true, + "i": true, "me": true, "my": true, "mine": true, "myself": true, +} + +// RecallAllowed is the second half of the recall gate (#470). A hit that +// cleared the score and margin gate may still be about something else +// entirely: the held-out fixture puts the right note at 0.791-0.890 and the +// must-be-silent cases at 0.795-0.835, so no threshold sits between them, and +// a note about his slow network answered "почему небо синее?". +// +// The veto applies only to a question that mentions nothing of his. That +// restriction is what keeps the fix from costing more than it saves: recall +// exists to find the note whose words he no longer remembers, and demanding a +// shared word of every recall silenced four true recalls on the fixture to +// kill one false one. A question about his own life keeps the embedder alone +// as its judge. A question about the world has to name something the memory +// actually mentions. +func RecallAllowed(query, text string) bool { + if mentionsHim(query) { + return true + } + return SharesContentWord(query, text) +} + +func mentionsHim(query string) bool { + for _, w := range strings.FieldsFunc(strings.ToLower(query), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) { + if firstPerson[w] { + return true + } + } + return false +} + +// SharesContentWord reports whether query and text have at least one topic +// word in common, after dropping the words that carry no topic. Stems are +// compared, so the note and the question do not have to inflect alike. +func SharesContentWord(query, text string) bool { + q := contentWords(query) + if len(q) == 0 { + // Nothing to compare — a question made entirely of filler. The score + // gate is then the only judge it can have. + return true + } + t := contentWords(text) + for _, a := range q { + for _, b := range t { + if a == b || sameStem(a, b) { + return true + } + } + } + return false +} + +func contentWords(s string) []string { + var out []string + for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) { + if !stopwords[w] { + out = append(out, w) + } + } + return out +} + +// sameStem is inflection and derivation tolerance: Russian marks case and +// tense on the ending, and the note and the question rarely use the same form. +// "воду" and "вода" are the same water, "кормить" and "корм" the same feeding. +// All but the last rune of the shorter word must match, and never fewer than +// three, which is what keeps "сеть" clear of "сеанс". +func sameStem(a, b string) bool { + ar, br := []rune(a), []rune(b) + n := min(len(ar), len(br)) - 1 + if n < 3 { + return false + } + return string(ar[:n]) == string(br[:n]) +} diff --git a/internal/memory/lexical_test.go b/internal/memory/lexical_test.go new file mode 100644 index 0000000..a4a6315 --- /dev/null +++ b/internal/memory/lexical_test.go @@ -0,0 +1,40 @@ +package memory + +import "testing" + +func TestRecallAllowed(t *testing.T) { + cases := []struct { + name string + query, text string + want bool + }{ + // The #470 shape: a world question and a note about his box. + {"world question, unrelated note", "почему небо синее", "сеть какая-то медленная", false}, + {"world question, unrelated fact", "какая столица Франции", "какая последняя версия языка Go", false}, + {"silent fixture case", "во сколько отходит поезд", "бэкап запускается в три ночи", false}, + + // A world question that does name the topic keeps its answer. + {"world question, same topic", "какой поезд идёт в Минск", "поезда в Минск ходят утром", true}, + + // A question about his own life is judged by the embedder alone, + // because recall exists for words he no longer remembers. + {"about him, no shared word", "во сколько я обычно засыпаю", "ложусь около одиннадцати", true}, + {"about him, english", "which colour scheme do i like", "тёмная тема везде", true}, + + // Inflection must not break a match. + {"inflected", "чем кормить кота", "корм для кота в шкафу", true}, + } + for _, c := range cases { + if got := RecallAllowed(c.query, c.text); got != c.want { + t.Errorf("%s: RecallAllowed(%q, %q) = %v, want %v", c.name, c.query, c.text, got, c.want) + } + } +} + +// A question made only of filler has no topic word to match on, and the score +// gate is then the only judge it can have. +func TestRecallAllowedFallsBackWhenNothingToCompare(t *testing.T) { + if !RecallAllowed("что это", "сеть какая-то медленная") { + t.Error("a question with no content word must not be vetoed") + } +}