Maven/internal/memory/lexical.go

package memory

import (
	"strings"
	"unicode"
)

// stopwords — words that carry no topic. A question and a note that share only
// these share nothing: "почему небо синее" and "сеть какая-то медленная" both
// contain "какая"-shaped filler and are about different worlds.
var stopwords = map[string]bool{
	// interrogatives and demonstratives
	"что": true, "чего": true, "какой": true, "какая": true, "какое": true,
	"какие": true, "каких": true, "кто": true, "кого": true, "кому": true,
	"почему": true, "зачем": true, "где": true, "куда": true, "откуда": true,
	"когда": true, "сколько": true, "как": true, "то": true, "это": true,
	"этот": true, "тот": true, "там": true, "тут": true, "такой": true,
	// pronouns — every sentence he says is about him, so "я" is not a topic
	"я": true, "меня": true, "мне": true, "мой": true, "моя": true, "мои": true,
	"ты": true, "тебя": true, "тебе": true, "твой": true, "он": true, "она": true,
	"они": true, "мы": true, "себя": true, "свой": true,
	// prepositions, conjunctions, particles, copulas
	"в": true, "во": true, "на": true, "с": true, "со": true, "у": true,
	"о": true, "об": true, "про": true, "за": true, "из": true, "по": true,
	"до": true, "от": true, "для": true, "над": true, "под": true, "при": true,
	"и": true, "а": true, "но": true, "или": true, "же": true, "ли": true,
	"не": true, "ни": true, "бы": true, "был": true, "была": true, "было": true,
	"быть": true, "есть": true, "был-ли": true, "уже": true, "ещё": true,
	"еще": true, "так": true, "вот": true, "там-же": true,
	// English filler, for the mixed utterances he does say
	"the": true, "a": true, "an": true, "is": true, "are": true, "was": true,
	"were": true, "be": true, "of": true, "in": true, "on": true, "at": true,
	"to": true, "for": true, "about": true, "and": true, "or": true, "not": true,
	"what": true, "who": true, "why": true, "when": true, "where": true,
	"which": true, "how": true, "i": true, "my": true, "me": true, "it": true,
	"this": true, "that": true,
}

// firstPerson — the words that make an utterance a question about his own
// life. Not possession only: "как я восстановил конфиги" owns nothing and is
// still about him.
var firstPerson = map[string]bool{
	"я": true, "меня": true, "мне": true, "мной": true, "мой": true,
	"моя": true, "моё": true, "мое": true, "мои": true, "моего": true,
	"моей": true, "моих": true, "моим": true, "себя": true, "свой": true,
	"своя": true, "свои": true, "своего": true, "мною": true,
	"i": true, "me": true, "my": true, "mine": true, "myself": true,
}

// RecallAllowed is the second half of the recall gate (#470). A hit that
// cleared the score and margin gate may still be about something else
// entirely: the held-out fixture puts the right note at 0.791-0.890 and the
// must-be-silent cases at 0.795-0.835, so no threshold sits between them, and
// a note about his slow network answered "почему небо синее?".
//
// The veto applies only to a question that mentions nothing of his. That
// restriction is what keeps the fix from costing more than it saves: recall
// exists to find the note whose words he no longer remembers, and demanding a
// shared word of every recall silenced four true recalls on the fixture to
// kill one false one. A question about his own life keeps the embedder alone
// as its judge. A question about the world has to name something the memory
// actually mentions.
func RecallAllowed(query, text string) bool {
	if mentionsHim(query) {
		return true
	}
	return SharesContentWord(query, text)
}

func mentionsHim(query string) bool {
	for _, w := range strings.FieldsFunc(strings.ToLower(query), func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
	}) {
		if firstPerson[w] {
			return true
		}
	}
	return false
}

// SharesContentWord reports whether query and text have at least one topic
// word in common, after dropping the words that carry no topic. Stems are
// compared, so the note and the question do not have to inflect alike.
func SharesContentWord(query, text string) bool {
	q := contentWords(query)
	if len(q) == 0 {
		// Nothing to compare — a question made entirely of filler. The score
		// gate is then the only judge it can have.
		return true
	}
	t := contentWords(text)
	for _, a := range q {
		for _, b := range t {
			if a == b || sameStem(a, b) {
				return true
			}
		}
	}
	return false
}

func contentWords(s string) []string {
	var out []string
	for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
	}) {
		if !stopwords[w] {
			out = append(out, w)
		}
	}
	return out
}

// sameStem is inflection and derivation tolerance: Russian marks case and
// tense on the ending, and the note and the question rarely use the same form.
// "воду" and "вода" are the same water, "кормить" and "корм" the same feeding.
// All but the last rune of the shorter word must match, and never fewer than
// three, which is what keeps "сеть" clear of "сеанс".
func sameStem(a, b string) bool {
	ar, br := []rune(a), []rune(b)
	n := min(len(ar), len(br)) - 1
	if n < 3 {
		return false
	}
	return string(ar[:n]) == string(br[:n])
}