Maven/internal/memory/lexical.go

package memory

import (
	"strings"
	"unicode"

	"github.com/kami/maven/internal/lexicon"
	"github.com/kami/maven/internal/morph"
)

var recallCaptureVerbs = func() map[string]bool {
	out := map[string]bool{}
	for _, w := range lexicon.CaptureVerbs() {
		out[w] = true
	}
	return out
}()

// stopwords — words that carry no topic. A question and a note that share only
// these share nothing: "почему небо синее" and "сеть какая-то медленная" both
// contain "какая"-shaped filler and are about different worlds.
var stopwords = map[string]bool{
	// interrogatives and demonstratives
	"что": true, "чего": true, "какой": true, "какая": true, "какое": true,
	"какие": true, "каких": true, "кто": true, "кого": true, "кому": true,
	"почему": true, "зачем": true, "где": true, "куда": true, "откуда": true,
	"когда": true, "сколько": true, "как": true, "то": true, "это": true,
	"этот": true, "тот": true, "там": true, "тут": true, "такой": true,
	// pronouns — every sentence he says is about him, so "я" is not a topic
	"я": true, "меня": true, "мне": true, "мой": true, "моя": true, "мои": true,
	"ты": true, "тебя": true, "тебе": true, "твой": true, "он": true, "она": true,
	"они": true, "мы": true, "себя": true, "свой": true,
	// prepositions, conjunctions, particles, copulas
	"в": true, "во": true, "на": true, "с": true, "со": true, "у": true,
	"о": true, "об": true, "про": true, "за": true, "из": true, "по": true,
	"до": true, "от": true, "для": true, "над": true, "под": true, "при": true,
	"и": true, "а": true, "но": true, "или": true, "же": true, "ли": true,
	"не": true, "ни": true, "бы": true, "был": true, "была": true, "было": true,
	"быть": true, "есть": true, "был-ли": true, "уже": true, "ещё": true,
	"еще": true, "так": true, "вот": true, "там-же": true,
	// English filler, for the mixed utterances he does say
	"the": true, "a": true, "an": true, "is": true, "are": true, "was": true,
	"were": true, "be": true, "of": true, "in": true, "on": true, "at": true,
	"to": true, "for": true, "about": true, "and": true, "or": true, "not": true,
	"what": true, "who": true, "why": true, "when": true, "where": true,
	"which": true, "how": true, "i": true, "my": true, "me": true, "it": true,
	"this": true, "that": true,
}

// firstPerson — the words that make an utterance a question about his own
// life. Not possession only: "как я восстановил конфиги" owns nothing and is
// still about him.
var firstPerson = map[string]bool{
	"я": true, "меня": true, "мне": true, "мной": true, "мой": true,
	"моя": true, "моё": true, "мое": true, "мои": true, "моего": true,
	"моей": true, "моих": true, "моим": true, "себя": true, "свой": true,
	"своя": true, "свои": true, "своего": true, "мною": true,
	"i": true, "me": true, "my": true, "mine": true, "myself": true,
}

// RecallAllowed is the second half of the recall gate (#470). A hit that
// cleared the score and margin gate may still be about something else
// entirely: the held-out fixture puts the right note at 0.817-0.892 and the
// must-be-silent cases at 0.787-0.874, so no threshold sits between them, and
// a note about his slow network answered "почему небо синее?".
//
// The broad veto applies only to a question that mentions nothing of his. That
// restriction is what keeps the fix from costing more than it saves: recall
// exists to find the note whose words he no longer remembers, and demanding a
// shared word of every recall silences several right-note paraphrases on the
// fixture. A question about his own life keeps the embedder alone
// as its judge. A question about the world has to name something the memory
// actually mentions.
//
// openQuestion is the caller's structural evidence that the words ask for
// information: an interrogative or narrative request, not punctuation alone.
// It closes a different hole: a model can mis-route an ordinary first-person
// report as a query. With only one stored note there is no runner-up for the
// margin gate, so "я отменил напоминание про молоко" recalled an unrelated note
// at cosine 0.825. A report or polar proposition therefore needs a named topic
// shared with the hit, even when it ends in '?'. Nominal requests still work —
// "адрес домашнего сервера" shares its topic — while a statistical route alone
// cannot make an unrelated personal statement into a recall request.
//
// requireNamedTopic is the stricter locative frame: an answer to "where is X"
// must corroborate every identity term in X. One overlapping modifier or
// predicate is not enough: the spare-key note scores 0.832-0.867 for a spare
// passport, blue shirt, blue document box, and car key, while sharing words
// such as "spare", "blue", "box", or "key" (V-719). Verbs are grammar, not
// identity, and are excluded using the embedded morphology dictionary.
//
// The veto's price was re-measured on 2026-08-03 (#496,
// docs/evals/2026-08-03-recall-topic-veto.md). It costs one true recall and
// buys one false one, and the fixture pass count is the same either way. The
// lost case is an English paraphrase, not the cross-language loss it was
// reported as, and the fixture has no cross-language case at all. Do not add a
// script test or a bilingual stem map for it — both are no-ops here. The
// separating signal is semantic and belongs in a reranker, not in this file.
func RecallAllowed(query, text string, openQuestion, requireNamedTopic bool) bool {
	shared := sharesNamedTopic(query, text)
	if requireNamedTopic && !corroboratesNamedIdentity(query, text) {
		return false
	}
	if !openQuestion && !shared {
		return false
	}
	if mentionsHim(query) {
		return true
	}
	return shared || len(contentWords(query)) == 0
}

// corroboratesNamedIdentity requires every non-frame, non-verb query term to
// occur in the candidate proposition's subject as the same dictionary word.
// Searching the whole candidate is unsafe: in "the spare key lies in the blue
// box", the box is a location, not the thing being located, and cannot answer
// "where is the blue box?". Locative predicates are deliberately excluded, so
// two memories do not become the same target merely because both things "lie"
// somewhere. Requiring all remaining terms preserves qualifiers too: a key is
// not a car key and a box is not a box of documents.
func corroboratesNamedIdentity(query, text string) bool {
	q := identityWords(query)
	if len(q) == 0 {
		return false
	}
	t := propositionSubjectIdentity(text)
	if len(t) == 0 {
		return false
	}
	for _, want := range q {
		found := false
		for _, got := range t {
			if want == got || morph.SameWord(want, got) {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}
	return true
}

// propositionSubjectIdentity returns the identity phrase before the first
// dictionary-proven main verb. A leading capture imperative is storage frame,
// not the proposition: "remember: the spare key lies ..." has "spare key" as
// its subject. If no subject/predicate boundary can be proved, it returns nil;
// strict locative recall then abstains instead of treating a location object or
// incidental modifier anywhere in the note as the requested entity.
func propositionSubjectIdentity(s string) []string {
	toks := wordTokens(s)
	start := 0
	for start < len(toks) && recallCaptureVerbs[toks[start]] {
		start++
	}
	predicate := -1
	for i := start; i < len(toks); i++ {
		if morph.IsVerbForm(toks[i]) {
			predicate = i
			break
		}
	}
	if predicate <= start {
		return nil
	}
	return identityTokens(toks[start:predicate])
}

func identityWords(s string) []string {
	return identityTokens(contentWords(s))
}

func identityTokens(words []string) []string {
	out := make([]string, 0, len(words))
	for _, w := range words {
		if !stopwords[w] && !morph.IsVerbForm(w) {
			out = append(out, w)
		}
	}
	return out
}

func mentionsHim(query string) bool {
	for _, w := range strings.FieldsFunc(strings.ToLower(query), func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
	}) {
		if firstPerson[w] {
			return true
		}
	}
	return false
}

// SharesContentWord reports whether query and text have at least one topic
// word in common, after dropping the words that carry no topic. Stems are
// compared, so the note and the question do not have to inflect alike.
func SharesContentWord(query, text string) bool {
	q := contentWords(query)
	if len(q) == 0 {
		// Nothing to compare — a question made entirely of filler. The score
		// gate is then the only judge it can have.
		return true
	}
	return sharesContentWords(q, contentWords(text))
}

// sharesNamedTopic is the stricter form used for a non-question-shaped turn:
// an utterance made entirely of frame words names no topic, so it cannot use
// the score gate as its only evidence that a stored note should be spoken.
func sharesNamedTopic(query, text string) bool {
	q := contentWords(query)
	if len(q) == 0 {
		return false
	}
	return sharesContentWords(q, contentWords(text))
}

func sharesContentWords(q, t []string) bool {
	for _, a := range q {
		for _, b := range t {
			if a == b || sameStem(a, b) {
				return true
			}
		}
	}
	return false
}

func contentWords(s string) []string {
	var out []string
	for _, w := range wordTokens(s) {
		if !stopwords[w] {
			out = append(out, w)
		}
	}
	return out
}

func wordTokens(s string) []string {
	return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
	})
}

// sameStem is inflection and derivation tolerance: Russian marks case and
// tense on the ending, and the note and the question rarely use the same form.
// "воду" and "вода" are the same water, "кормить" and "корм" the same feeding.
// All but the last rune of the shorter word must match, and never fewer than
// three, which is what keeps "сеть" clear of "сеанс".
func sameStem(a, b string) bool {
	ar, br := []rune(a), []rune(b)
	n := min(len(ar), len(br)) - 1
	if n < 3 {
		return false
	}
	return string(ar[:n]) == string(br[:n])
}