memory: a world question must name what the memory mentions (V-470)

The score gate cannot separate the right note from an unrelated one: the held-out fixture puts the right note at 0.791-0.890 and the must-be-silent cases at 0.795-0.835, so a note about his slow network answered 'почему небо синее?'. RecallAllowed adds a topic veto, and applies it only to a question that mentions nothing of his. That restriction is the whole design: demanding a shared word of every recall silenced four true recalls on the fixture to kill one false one, because recall exists to find the note whose words he no longer remembers. A question about his own life keeps the embedder as its only judge.
2026-08-03 13:50:54 +04:00
parent 4dfe106fe3
commit aec94eb2e8
2 changed files with 166 additions and 0 deletions
@@ -0,0 +1,126 @@
+package memory
+
+import (
+	"strings"
+	"unicode"
+)
+
+// stopwords — words that carry no topic. A question and a note that share only
+// these share nothing: "почему небо синее" and "сеть какая-то медленная" both
+// contain "какая"-shaped filler and are about different worlds.
+var stopwords = map[string]bool{
+	// interrogatives and demonstratives
+	"что": true, "чего": true, "какой": true, "какая": true, "какое": true,
+	"какие": true, "каких": true, "кто": true, "кого": true, "кому": true,
+	"почему": true, "зачем": true, "где": true, "куда": true, "откуда": true,
+	"когда": true, "сколько": true, "как": true, "то": true, "это": true,
+	"этот": true, "тот": true, "там": true, "тут": true, "такой": true,
+	// pronouns — every sentence he says is about him, so "я" is not a topic
+	"я": true, "меня": true, "мне": true, "мой": true, "моя": true, "мои": true,
+	"ты": true, "тебя": true, "тебе": true, "твой": true, "он": true, "она": true,
+	"они": true, "мы": true, "себя": true, "свой": true,
+	// prepositions, conjunctions, particles, copulas
+	"в": true, "во": true, "на": true, "с": true, "со": true, "у": true,
+	"о": true, "об": true, "про": true, "за": true, "из": true, "по": true,
+	"до": true, "от": true, "для": true, "над": true, "под": true, "при": true,
+	"и": true, "а": true, "но": true, "или": true, "же": true, "ли": true,
+	"не": true, "ни": true, "бы": true, "был": true, "была": true, "было": true,
+	"быть": true, "есть": true, "был-ли": true, "уже": true, "ещё": true,
+	"еще": true, "так": true, "вот": true, "там-же": true,
+	// English filler, for the mixed utterances he does say
+	"the": true, "a": true, "an": true, "is": true, "are": true, "was": true,
+	"were": true, "be": true, "of": true, "in": true, "on": true, "at": true,
+	"to": true, "for": true, "about": true, "and": true, "or": true, "not": true,
+	"what": true, "who": true, "why": true, "when": true, "where": true,
+	"which": true, "how": true, "i": true, "my": true, "me": true, "it": true,
+	"this": true, "that": true,
+}
+
+// firstPerson — the words that make an utterance a question about his own
+// life. Not possession only: "как я восстановил конфиги" owns nothing and is
+// still about him.
+var firstPerson = map[string]bool{
+	"я": true, "меня": true, "мне": true, "мной": true, "мой": true,
+	"моя": true, "моё": true, "мое": true, "мои": true, "моего": true,
+	"моей": true, "моих": true, "моим": true, "себя": true, "свой": true,
+	"своя": true, "свои": true, "своего": true, "мною": true,
+	"i": true, "me": true, "my": true, "mine": true, "myself": true,
+}
+
+// RecallAllowed is the second half of the recall gate (#470). A hit that
+// cleared the score and margin gate may still be about something else
+// entirely: the held-out fixture puts the right note at 0.791-0.890 and the
+// must-be-silent cases at 0.795-0.835, so no threshold sits between them, and
+// a note about his slow network answered "почему небо синее?".
+//
+// The veto applies only to a question that mentions nothing of his. That
+// restriction is what keeps the fix from costing more than it saves: recall
+// exists to find the note whose words he no longer remembers, and demanding a
+// shared word of every recall silenced four true recalls on the fixture to
+// kill one false one. A question about his own life keeps the embedder alone
+// as its judge. A question about the world has to name something the memory
+// actually mentions.
+func RecallAllowed(query, text string) bool {
+	if mentionsHim(query) {
+		return true
+	}
+	return SharesContentWord(query, text)
+}
+
+func mentionsHim(query string) bool {
+	for _, w := range strings.FieldsFunc(strings.ToLower(query), func(r rune) bool {
+		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+	}) {
+		if firstPerson[w] {
+			return true
+		}
+	}
+	return false
+}
+
+// SharesContentWord reports whether query and text have at least one topic
+// word in common, after dropping the words that carry no topic. Stems are
+// compared, so the note and the question do not have to inflect alike.
+func SharesContentWord(query, text string) bool {
+	q := contentWords(query)
+	if len(q) == 0 {
+		// Nothing to compare — a question made entirely of filler. The score
+		// gate is then the only judge it can have.
+		return true
+	}
+	t := contentWords(text)
+	for _, a := range q {
+		for _, b := range t {
+			if a == b || sameStem(a, b) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func contentWords(s string) []string {
+	var out []string
+	for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
+		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+	}) {
+		if !stopwords[w] {
+			out = append(out, w)
+		}
+	}
+	return out
+}
+
+// sameStem is inflection and derivation tolerance: Russian marks case and
+// tense on the ending, and the note and the question rarely use the same form.
+// "воду" and "вода" are the same water, "кормить" and "корм" the same feeding.
+// All but the last rune of the shorter word must match, and never fewer than
+// three, which is what keeps "сеть" clear of "сеанс".
+func sameStem(a, b string) bool {
+	ar, br := []rune(a), []rune(b)
+	n := min(len(ar), len(br)) - 1
+	if n < 3 {
+		return false
+	}
+	return string(ar[:n]) == string(br[:n])
+}
@@ -0,0 +1,40 @@
+package memory
+
+import "testing"
+
+func TestRecallAllowed(t *testing.T) {
+	cases := []struct {
+		name        string
+		query, text string
+		want        bool
+	}{
+		// The #470 shape: a world question and a note about his box.
+		{"world question, unrelated note", "почему небо синее", "сеть какая-то медленная", false},
+		{"world question, unrelated fact", "какая столица Франции", "какая последняя версия языка Go", false},
+		{"silent fixture case", "во сколько отходит поезд", "бэкап запускается в три ночи", false},
+
+		// A world question that does name the topic keeps its answer.
+		{"world question, same topic", "какой поезд идёт в Минск", "поезда в Минск ходят утром", true},
+
+		// A question about his own life is judged by the embedder alone,
+		// because recall exists for words he no longer remembers.
+		{"about him, no shared word", "во сколько я обычно засыпаю", "ложусь около одиннадцати", true},
+		{"about him, english", "which colour scheme do i like", "тёмная тема везде", true},
+
+		// Inflection must not break a match.
+		{"inflected", "чем кормить кота", "корм для кота в шкафу", true},
+	}
+	for _, c := range cases {
+		if got := RecallAllowed(c.query, c.text); got != c.want {
+			t.Errorf("%s: RecallAllowed(%q, %q) = %v, want %v", c.name, c.query, c.text, got, c.want)
+		}
+	}
+}
+
+// A question made only of filler has no topic word to match on, and the score
+// gate is then the only judge it can have.
+func TestRecallAllowedFallsBackWhenNothingToCompare(t *testing.T) {
+	if !RecallAllowed("что это", "сеть какая-то медленная") {
+		t.Error("a question with no content word must not be vetoed")
+	}
+}