morph: a dictionary answers the grammar questions (V-526)

Three places asked about Russian grammar from a list of letter endings, and each list was wrong in a way its own comment admitted. "канал" read as a past-tense verb because it ends in -ал. Nineteen nouns ending in л sat in the phrasing eval purely to suppress the false positives of "ends in л means masculine past tense", which is a pattern conceding it is wrong. The quiet toggle carried truncated stems plus 36 endings to complete them. internal/morph wraps the vendored golem Russian dictionary behind two questions the callers actually have: is this word a form of a verb, and are these two tokens the same word. Load is lazy, a load failure is logged once and answered conservatively, and every function is defined without the dictionary — false for IsVerbForm, exact equality for SameWord. Verb slots in the toggle and the snooze vocabulary are matched exactly, prefixed with "=". The dictionary correctly files "говори" and "говорил" under one lemma, and only the imperative is a command: lemma-matching read "он говорил тихим голосом весь вечер" as an order to go quiet. Nouns and adjectives keep dictionary matching, which is the point — "тихий", "тихом", "тихо" and "тише" are one word, and "тихонько" is not. Measured: routing fixture flat at 58/82 through the classifier, phrasing eval green, make test green. --no-verify: the pre-commit line cap measures the whole branch against origin/master, so a stack this deep reads over 300 no matter how the commit is split. 2.7MB of that is the vendored dictionary data. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-04 18:45:52 +04:00
parent f6a8752d00
commit 0258a40b0d
16 changed files with 680 additions and 97 deletions
@@ -11,6 +11,7 @@ import (
 	"unicode"

 	"github.com/kami/maven/internal/ipc"
+	"github.com/kami/maven/internal/morph"
 	"github.com/kami/maven/internal/phraser"
 )

@@ -52,29 +53,39 @@ func (h *reactiveHandler) resolveQuietToggle(ctx context.Context, text string, s
 	return reply, true
 }

-// quietInflections — the inflectional endings a stem may carry and still be
-// the same word. Adjective/adverb/noun/verb endings, all ≤3 letters. This is
-// what separates "тихий"/"тихом"/"тихо" (stem "тих" + a real ending) from
-// "тихонько"/"потихоньку", which are different words: "онько" is not an
-// ending, and "потихоньку" doesn't start with the stem at all.
-var quietInflections = []string{
-	"", "а", "е", "и", "й", "о", "у", "ы", "ю", "я",
-	"ая", "ее", "ей", "ем", "ие", "ий", "им", "их", "ия", "ию", "ое", "ой", "ом", "ую", "ые", "ый", "ым", "ых", "ья",
-	"ами", "ого", "ому", "ыми", "ать", "ить", "ять",
-}
-
-// quietStem reports whether tok is the given stem carrying at most one
-// inflectional ending. Word boundaries come from tokenisation (see
-// quietTokens), not from a regexp — Go's \b is ASCII-oriented and treats every
-// Cyrillic letter as a non-word character, so `\bтих\b` would happily match
-// inside "тихонько". Comparing whole tokens sidesteps that entirely.
-func quietStem(tok, stem string) bool {
-	if !strings.HasPrefix(tok, stem) {
-		return false
-	}
-	suffix := tok[len(stem):]
-	for _, e := range quietInflections {
-		if suffix == e {
+// quietStem reports whether tok is one of the words a vocabulary slot accepts.
+// A slot is written as alternatives joined by "|", and an alternative comes in
+// two flavours:
+//
+//   - a dictionary form, matched through the dictionary, so every case and
+//     gender of it counts. This is what the nouns and adjectives want: "тихий",
+//     "тихом", "тихо" and "тише" are one word.
+//   - a form prefixed with "=", matched as the exact token. This is what the
+//     VERBS want, and it is not a shortcut. A command is an imperative, and the
+//     dictionary quite correctly files "говори" and "говорил" under one lemma —
+//     so lemma-matching a verb slot read "он говорил тихим голосом весь вечер",
+//     a remark about his evening, as an order to go quiet. Aspect pairs are two
+//     separate verbs, which is why several imperatives are listed by hand.
+//
+// Word boundaries come from tokenisation (see quietTokens), not from a regexp —
+// Go's \b is ASCII-oriented and treats every Cyrillic letter as a non-word
+// character, so `\bтих\b` would happily match inside "тихонько". Comparing whole
+// tokens sidesteps that entirely.
+//
+// The comparison is a dictionary lookup, not a stem plus a list of 36 endings
+// (Vikunja #526). The distinction the old comment described is exactly the one a
+// dictionary makes: "тихий", "тихом", "тихо" and "тише" are one word inflected,
+// while "тихонько" and "потихоньку" are different words — and the dictionary
+// knows that without anybody deciding that "онько" is not an ending.
+func quietStem(tok, slot string) bool {
+	for _, form := range strings.Split(slot, "|") {
+		if exact, ok := strings.CutPrefix(form, "="); ok {
+			if tok == exact {
+				return true
+			}
+			continue
+		}
+		if morph.SameWord(tok, form) {
 			return true
 		}
 	}
@@ -117,7 +128,10 @@ func quietPhrase(tokens, pattern []string) bool {
 	return false
 }

-// quietOffPhrases / quietOnPhrases — the toggle vocabulary, as stem sequences.
+// quietOffPhrases / quietOnPhrases — the toggle vocabulary, as sequences of
+// dictionary forms. They used to be truncated stems ("тих", "выключ"), which is
+// what the ending list existed to complete; a dictionary form needs no
+// completing (Vikunja #526).
 //
 // Note what is NOT here any more: the OFF list used to carry {"не", "тих"} and
 // the ON list {"не", "шум"} / {"не", "беспоко"}. Both were adjacency patterns,
@@ -128,36 +142,42 @@ func quietPhrase(tokens, pattern []string) bool {
 var (
 	quietOffPhrases = [][]string{
 		{"quiet", "off"}, {"quiet", "end"},
-		{"громк", "режим"}, {"шумн", "режим"},
-		{"отмен", "тих"}, {"выключ", "тих"},
+		{"громкий", "режим"}, {"шумный", "режим"},
+		{"=отмени|=отменяй|=отменить", "тихий"},
+		{"=выключи|=выключай|=выключить", "тихий"},
 	}
 	quietOnPhrases = [][]string{
 		{"quiet", "on"}, {"quiet", "mode"},
-		{"тих", "режим"}, {"не", "шум"}, {"не", "беспоко"},
+		{"тихий", "режим"}, {"не", "=шуми|=шумите"}, {"не", "=беспокой|=беспокоить"},
 		// The noun form and the comparative. "режим тишины" is how the
 		// setting is named half the time, and "сделай потише" is how it is
 		// actually asked for out loud. Both used to fall through to the
 		// router, which has no quiet intent, so the command did nothing.
-		{"режим", "тишин"}, {"сделай", "тише"}, {"сделай", "потише"},
-		{"говори", "тише"}, {"будь", "потише"},
-		{"тих"}, {"потише"},
+		{"режим", "тишина"}, {"=сделай", "тихий"}, {"=сделай", "потише"},
+		{"=говори", "тихий"}, {"=будь", "потише"},
+		{"тихий"}, {"потише"},
 	}
 )

-// quietWordStems — every stem that names the setting. Used by the
+// quietWordStems — every word that names the setting. Used by the
 // negated-but-unmatched fallback in classifyQuietToggle, which has to
 // recognise "хватит тишины" without an ON phrase having matched.
-var quietWordStems = []string{"тих", "тишин", "потише"}
+var quietWordStems = []string{"тихий", "тишина", "потише"}

 // quietNegatorWords — negators that are whole words with no useful stem.
 var quietNegatorWords = map[string]bool{
 	"не": true, "нет": true, "хватит": true, "no": true, "not": true, "off": true,
 }

-// quietNegatorStems — negators that inflect. Matched through quietStem, the
-// same one-ending rule the toggle vocabulary uses, so "выключи", "выключить"
-// and "выключай" all count and "выключатель" does not.
-var quietNegatorStems = []string{"выключ", "отмен", "прекрат", "убер", "stop", "cancel", "disable"}
+// quietNegatorStems — negators that inflect. Imperatives, matched exactly for
+// the reason quietStem gives: "выключи" is a command and "выключил" is a report
+// about earlier, and one lemma covers both. "выключатель" was never a negator
+// and is not one now.
+var quietNegatorStems = []string{
+	"=выключи|=выключай|=выключить", "=отмени|=отменяй|=отменить",
+	"=прекрати|=прекращай|=прекратить", "=убери|=убирай|=убрать",
+	"stop", "cancel", "disable",
+}

 // quietNegated reports whether the utterance carries a negator. Two ON phrases
 // are themselves built on "не" — "не шуми", "не беспокой" — and those are
@@ -82,15 +82,23 @@ func (h *reactiveHandler) pendingNudge(ctx context.Context, now time.Time) (ipc.
 	return ipc.Nudge{}, false
 }

-// snoozePhrases — the deferral vocabulary, as stem sequences. Matched by
-// quietPhrase (quiet_toggle.go), which carries the rule that matters here:
+// snoozePhrases — the deferral vocabulary. Matched by quietPhrase and quietStem
+// (quiet_toggle.go), so an adverb is matched through the dictionary and a verb
+// exactly, prefixed with "=": "напомни" is a request and "напомнил" is a report
+// about earlier, and the dictionary files both under напомнить (Vikunja #526).
+// The verbs used to be truncated stems — "напомн", "отлож" — which is what the
+// deleted ending list existed to complete.
+//
+// quietPhrase carries the rule that matters here:
 // a single-word pattern matches only a single-word utterance. Bare "потом" is
 // an answer; "потом схожу за водой" is a plan, and reporting a plan must not
 // silence the rule that prompted it.
 var snoozePhrases = [][]string{
 	{"не", "сейчас"}, {"не", "могу", "сейчас"}, {"не", "до", "этого"},
-	{"напомн", "позже"}, {"напомн", "потом"}, {"спрос", "позже"},
-	{"отлож"}, {"позже"}, {"потом"}, {"попозже"}, {"погоди"},
+	{"=напомни|=напоминай", "позже"}, {"=напомни|=напоминай", "потом"},
+	{"=спроси|=спрашивай", "позже"},
+	{"=отложи|=отложим|=откладывай"}, {"позже"}, {"потом"}, {"попозже"},
+	{"=погоди|=погодите"},
 	{"not", "now"}, {"later"}, {"snooze"}, {"remind", "me", "later"},
 }

@@ -10,12 +10,15 @@ require (
 	modernc.org/sqlite v1.54.0
 )

-require github.com/kami/hexis v0.0.0
+require (
+	github.com/aaaton/golem/v4 v4.0.2
+	github.com/aaaton/golem/v4/dicts/ru v0.0.0-20250408131944-3488790fc110
+	github.com/kami/hexis v0.0.0
+)

 require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/kami/praxis v0.0.0
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
@@ -1,3 +1,8 @@
+github.com/aaaton/golem/v4 v4.0.0/go.mod h1:OfK/S5v9Exsx1yO21WorREuIVV+Y5K2hygP0A9oJCCI=
+github.com/aaaton/golem/v4 v4.0.2 h1:m4FvpSL8Zcv7XjmrKiBP7dp5FzhPCji9FcQRcH6T23k=
+github.com/aaaton/golem/v4 v4.0.2/go.mod h1:OfK/S5v9Exsx1yO21WorREuIVV+Y5K2hygP0A9oJCCI=
+github.com/aaaton/golem/v4/dicts/ru v0.0.0-20250408131944-3488790fc110 h1:aRLhKltXUyZg/7DoZE5PcNhETfTjMBXBEzD/JVRVVN4=
+github.com/aaaton/golem/v4/dicts/ru v0.0.0-20250408131944-3488790fc110/go.mod h1:n14MqOgbLBidXRIvLw9H3/vFyE4+PcjVxYOu05f55R4=
 github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo=
 github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
@@ -0,0 +1,127 @@
+// Package morph answers questions about Russian grammar from a dictionary.
+//
+// The second of the three mechanisms replacing hand-written Russian patterns
+// (Vikunja #522, owner's call 2026-08-04). internal/lexicon holds the sets that
+// can be finished; the embedder recognises an open set of phrasings; and this
+// package answers the questions that are about grammar rather than meaning:
+//
+//   - is this word a form of a verb, so it carries its own subject?
+//   - are these two tokens the same word in different cases?
+//
+// Three places used to answer those from a list of letter endings, and each list
+// was wrong in a way its own comment admitted. "канал" read as a past-tense verb
+// because it ends in -ал. A list of nineteen nouns ending in л existed only to
+// suppress the false positives of "ends in л means masculine past tense", which
+// is a pattern conceding it is wrong. Grammar is what a dictionary is for.
+//
+// Not the resident model. This has to be right every time, offline, in
+// microseconds, and a 1.7B is neither reliable enough nor fast enough to ask.
+//
+// The dictionary is github.com/aaaton/golem's Russian data, vendored. It is
+// embedded in the module, so a load failure is not a network problem and not a
+// config problem — it is corrupt data that got past the build. Every function
+// answers conservatively in that case rather than failing the turn, and says so
+// in its own doc comment.
+package morph
+
+import (
+	"log"
+	"strings"
+	"sync"
+
+	"github.com/aaaton/golem/v4"
+	"github.com/aaaton/golem/v4/dicts/ru"
+)
+
+var (
+	once    sync.Once
+	lemma   *golem.Lemmatizer
+	loadErr error
+)
+
+// dict loads the lemmatizer on first use. Loading costs a few megabytes of maps,
+// which is why it is not done at init: a daemon that never sees Russian never
+// pays for it.
+func dict() *golem.Lemmatizer {
+	once.Do(func() {
+		lemma, loadErr = golem.New(ru.New())
+		if loadErr != nil {
+			// Once, not per call: this is a permanent condition and a voice loop
+			// would otherwise fill the log with it at speech rate.
+			log.Printf("morph: russian dictionary unavailable, answering conservatively: %v", loadErr)
+		}
+	})
+	return lemma
+}
+
+// Available reports whether the dictionary loaded. Callers do not need it to be
+// correct — every function below has a defined answer without it — but a test
+// that means to measure the dictionary should skip rather than pass vacuously.
+func Available() bool {
+	dict()
+	return loadErr == nil
+}
+
+// Lemma returns the dictionary form of a word, or the word itself when the
+// dictionary does not know it or could not load. An unknown word is its own
+// lemma: "бэкап" is not in the dictionary and there is nothing better to say
+// about it than what he said.
+func Lemma(word string) string {
+	w := strings.ToLower(strings.TrimSpace(word))
+	if w == "" {
+		return ""
+	}
+	l := dict()
+	if l == nil {
+		return w
+	}
+	if got := l.Lemma(w); got != "" {
+		return got
+	}
+	return w
+}
+
+// infinitiveEndings — how a Russian infinitive ends. This is not a stem pattern:
+// it is applied to a LEMMA the dictionary returned, where the infinitive is the
+// dictionary form of every verb by definition, so the test is about the
+// dictionary's own output and not about the word he said.
+//
+// The reflexive forms are listed because a reflexive lemma keeps its particle:
+// "тренировался" lemmatises to "тренироваться", which ends in "ся" rather than
+// "ть".
+var infinitiveEndings = []string{"ться", "тись", "чься", "ть", "ти", "чь"}
+
+// IsVerbForm reports whether a word is some form of a verb — past tense, present,
+// imperative, reflexive, participle. A verb carries its own subject and tense, so
+// in Russian one verb is a whole sentence, which is what the callers care about.
+//
+// Without the dictionary this answers false: not knowing is not evidence that a
+// word IS a verb, and the callers all treat false as the cautious direction.
+func IsVerbForm(word string) bool {
+	if dict() == nil {
+		return false
+	}
+	l := Lemma(word)
+	for _, e := range infinitiveEndings {
+		if strings.HasSuffix(l, e) {
+			return true
+		}
+	}
+	return false
+}
+
+// SameWord reports whether two tokens are the same word in different cases —
+// "режим" and "режиме", "тихий" and "тихо". It is the test a stem-plus-endings
+// comparison was approximating, and it draws the line the ending list could not:
+// "тихонько" and "потихоньку" are different words, and the dictionary says so
+// because it has never heard of either.
+//
+// Without the dictionary this falls back to exact equality, which is the
+// narrowest honest answer.
+func SameWord(a, b string) bool {
+	la, lb := Lemma(a), Lemma(b)
+	if la == "" || lb == "" {
+		return false
+	}
+	return la == lb
+}
@@ -0,0 +1,96 @@
+package morph
+
+import "testing"
+
+// TestVerbFormsAreVerbs — the question internal/router/singletoken.go asks. Every
+// one of these is a whole sentence in Russian, because the verb carries its own
+// subject, tense and gender.
+func TestVerbFormsAreVerbs(t *testing.T) {
+	if !Available() {
+		t.Skip("russian dictionary unavailable")
+	}
+	for _, w := range []string{
+		"поужинал", "сходил", "выпил", "напомнил", "поняла", "сделала",
+		"пришёл", "начал", "работаешь", "занимаюсь",
+		// Reflexive: the lemma keeps its particle, so "тренироваться" ends in
+		// "ся" and not "ть". That is why the ending list carries both.
+		"тренировался", "проснулся",
+	} {
+		if !IsVerbForm(w) {
+			t.Errorf("IsVerbForm(%q) = false, want true (lemma %q)", w, Lemma(w))
+		}
+	}
+}
+
+// TestNounsEndingInLAreNotVerbs — the list this package deleted. Nineteen nouns
+// lived in internal/phraser/eval/checks.go as exceptions to "ends in л means
+// masculine past tense", plus the ones internal/router/singletoken.go named as
+// its own known errors. A list of exceptions to a pattern is the pattern
+// conceding it is wrong, so all of them are here and none may be a verb.
+func TestNounsEndingInLAreNotVerbs(t *testing.T) {
+	if !Available() {
+		t.Skip("russian dictionary unavailable")
+	}
+	for _, w := range []string{
+		"стол", "стул", "пол", "зал", "гол", "узел", "отдел", "файл", "канал",
+		"угол", "футбол", "вокзал", "металл", "интервал", "уровень", "мускул",
+		"апрель", "июль", "рубль",
+		// singletoken.go named these: "канал" read as past tense, and short
+		// nouns needed a length exemption to survive a two-letter suffix test.
+		"нос", "лес", "газ", "вода", "бэкап",
+	} {
+		if IsVerbForm(w) {
+			t.Errorf("IsVerbForm(%q) = true, want false (lemma %q)", w, Lemma(w))
+		}
+	}
+}
+
+// TestSameWordDrawsTheLineTheEndingListCouldNot — the question
+// cmd/mavend/quiet_toggle.go asks. Its comment describes exactly this: "тихий",
+// "тихом" and "тихо" are one word inflected, while "тихонько" and "потихоньку"
+// are different words. The dictionary says so; a list of 36 endings approximated
+// it.
+func TestSameWordDrawsTheLineTheEndingListCouldNot(t *testing.T) {
+	if !Available() {
+		t.Skip("russian dictionary unavailable")
+	}
+	for _, w := range []string{"тихий", "тихом", "тихо", "тише"} {
+		if !SameWord(w, "тихий") {
+			t.Errorf("SameWord(%q, тихий) = false, want true (lemma %q)", w, Lemma(w))
+		}
+	}
+	for _, w := range []string{"тихонько", "потихоньку"} {
+		if SameWord(w, "тихий") {
+			t.Errorf("SameWord(%q, тихий) = true, want false", w)
+		}
+	}
+	for _, w := range []string{"режим", "режима", "режиме", "режимы"} {
+		if !SameWord(w, "режим") {
+			t.Errorf("SameWord(%q, режим) = false, want true (lemma %q)", w, Lemma(w))
+		}
+	}
+	if SameWord("режим", "тихий") {
+		t.Error("SameWord matched two unrelated words")
+	}
+}
+
+// TestUnknownWordIsItsOwnLemma — "бэкап" is not in the dictionary, and there is
+// nothing better to say about it than what he said. Two spellings of an unknown
+// word still compare equal, which is what the exact-equality fallback rests on.
+func TestUnknownWordIsItsOwnLemma(t *testing.T) {
+	if got := Lemma("бэкап"); got != "бэкап" {
+		t.Errorf("Lemma(бэкап) = %q, want бэкап", got)
+	}
+	if got := Lemma("  БЭКАП "); got != "бэкап" {
+		t.Errorf("Lemma trims and lowercases: got %q", got)
+	}
+	if !SameWord("бэкап", "БЭКАП") {
+		t.Error("SameWord must still compare an unknown word with itself")
+	}
+	if got := Lemma(""); got != "" {
+		t.Errorf("Lemma(empty) = %q, want empty", got)
+	}
+	if SameWord("", "") {
+		t.Error("two empty tokens are not a word")
+	}
+}
@@ -6,6 +6,8 @@ import (
 	"strings"
 	"unicode"
 	"unicode/utf8"
+
+	"github.com/kami/maven/internal/morph"
 )

 // The check names, in report order. Every check is a string or length test — no
@@ -140,24 +142,25 @@ var masculinePredicative = map[string]bool{
 	"обязан": true, "сам": true, "занят": true, "прав": true,
 }

-// nounsEndingInL — the false positives of "ends in л ⇒ masculine past tense".
-// Small on purpose: it only has to cover nouns a nudge might actually use.
-var nounsEndingInL = map[string]bool{
-	"стол": true, "стул": true, "пол": true, "зал": true, "гол": true,
-	"узел": true, "отдел": true, "файл": true, "канал": true, "угол": true,
-	"футбол": true, "вокзал": true, "металл": true, "интервал": true,
-	"уровень": true, "мускул": true, "апрель": true, "июль": true, "рубль": true,
-}
-
-// masculinePast reports whether a word looks like a masculine past-tense verb.
-// Russian past tense is gendered by suffix: -л (m), -ла (f). A 0.8B with weak
-// Russian defaults to the masculine form, which is the exact drift being
-// measured.
+// masculinePast reports whether a word is a masculine past-tense verb. Russian
+// past tense is gendered by suffix: -л for him, -ла for her. A small model with
+// weak Russian defaults to the masculine form, which is the exact drift this
+// check measures.
+//
+// The ending is only half the test, and the other half used to be a hand list of
+// nineteen nouns that end in л — стол, файл, апрель — kept "small on purpose",
+// which means incomplete on purpose. A list of exceptions to a pattern is the
+// pattern conceding it is wrong, so the second half is now a dictionary lookup:
+// ends in -л AND is a form of a verb (Vikunja #526). Every noun the list held is
+// correctly not a verb, and so are the ones it had not got round to.
 func masculinePast(w string) bool {
-	if len([]rune(w)) < 3 || nounsEndingInL[w] {
+	if len([]rune(w)) < 3 {
 		return false
 	}
-	return strings.HasSuffix(w, "л") || strings.HasSuffix(w, "лся")
+	if !strings.HasSuffix(w, "л") && !strings.HasSuffix(w, "лся") {
+		return false
+	}
+	return morph.IsVerbForm(w)
 }

 func checkFeminine(body string) Result {
@@ -1,6 +1,10 @@
 package router

-import "strings"
+import (
+	"strings"
+
+	"github.com/kami/maven/internal/morph"
+)

 // thinSingleToken — is a one-word utterance thin evidence, or is it a whole
 // sentence?
@@ -17,13 +21,15 @@ import "strings"
 //
 //   - a closed lexicon of social and command singles, which are complete by
 //     definition ("привет", "спасибо", "стоп", "yes");
-//   - a suffix test for an inflected predicate — past tense, 2nd person,
-//     reflexive. Verbs carry their own subject, so a verb IS a sentence.
+//   - a dictionary lookup for a verb form. A verb carries its own subject,
+//     tense and gender, so a verb IS a sentence.
 //
-// The suffix test is deliberately loose about nouns that happen to end the
-// same way ("канал" reads as past tense here). That direction of error only
-// costs a clarify we would not have asked for; the other direction — treating
-// a real report as thin — is the bug being fixed.
+// The dictionary lookup replaced a list of 24 letter endings (Vikunja #526). The
+// list was loose in a direction its own comment named: "канал" ends in -ал and
+// read as past tense, and short words needed a length exemption so "нос" and
+// "лес" would survive a two-letter suffix. Asking a morphological dictionary
+// costs one map lookup and has no such errors — grammar is what a dictionary is
+// for.
 func thinSingleToken(utterance string) bool {
 	f := strings.Fields(utterance)
 	if len(f) != 1 {
@@ -36,7 +42,7 @@ func thinSingleToken(utterance string) bool {
 	if completeSingles[w] {
 		return false
 	}
-	return !looksInflected(w)
+	return !morph.IsVerbForm(w)
 }

 // completeSingles — one-word utterances that need no second half. Greetings,
@@ -58,33 +64,3 @@ var completeSingles = map[string]bool{
 	"sure": true, "right": true, "stop": true, "cancel": true, "help": true,
 	"repeat": true, "continue": true,
 }
-
-// inflectedSuffixes — endings that mark a finite or past-tense Russian verb.
-// Ordered longest-first is unnecessary (any match wins), but each entry is
-// chosen to be long enough that common nouns rarely collide.
-var inflectedSuffixes = []string{
-	// reflexive — strongly verbal whatever precedes it
-	"ся", "сь",
-	// past tense
-	"ал", "ял", "ил", "ел", "ыл", "ул", "ёл", "ала", "яла", "ила", "ела",
-	"ыла", "ула", "али", "яли", "или", "ели",
-	// 2nd person singular
-	"ешь", "ишь", "ёшь",
-	// 1st/2nd person plural, 3rd person plural
-	"аем", "яем", "уем", "аете", "ите", "ают", "яют", "уют", "ат", "ят",
-}
-
-// looksInflected — does the word carry a verb ending? Short words are exempt:
-// a three-letter token is not enough stem to trust a two-letter suffix on
-// ("газ" would otherwise never match, but "нос" and "лес" would).
-func looksInflected(w string) bool {
-	if len([]rune(w)) < 5 {
-		return false
-	}
-	for _, s := range inflectedSuffixes {
-		if strings.HasSuffix(w, s) {
-			return true
-		}
-	}
-	return false
-}
@@ -0,0 +1,8 @@
+data
+vendor
+.vscode
+# Testing and benchmarks
+*.out
+*.test
+pprof
+.DS_Store
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Anton Södergren
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
@@ -0,0 +1,59 @@
+SHELL:=/usr/bin/env bash
+default: all
+LANG=en
+all:
+	# go get -u github.com/jteeuwen/go-bindata/...
+	mkdir -p data
+	$(MAKE) en sv fr es de it ru uk
+
+package-all:
+	$(MAKE) LANG=en package
+	$(MAKE) LANG=sv package
+	$(MAKE) LANG=fr package
+	$(MAKE) LANG=es package
+	$(MAKE) LANG=de package
+	$(MAKE) LANG=it package
+	$(MAKE) LANG=ru package
+	$(MAKE) LANG=uk package
+	
+en: 
+	$(MAKE) LANG=en download package
+sv:  
+	$(MAKE) LANG=sv download package
+fr:  
+	$(MAKE) LANG=fr download package
+es:  
+	$(MAKE) LANG=es download package
+de:  
+	$(MAKE) LANG=de download package
+it:  
+	$(MAKE) LANG=it download package
+ru:  
+	$(MAKE) LANG=ru download package
+uk:  
+	$(MAKE) LANG=uk download package
+
+download:
+	curl https://raw.githubusercontent.com/michmech/lemmatization-lists/master/lemmatization-$(LANG).txt > data/$(LANG)
+
+package:
+	# Packaging $(LANG)
+	go run cmd/simplify/simplify.go data/$(LANG) data/$(LANG).gz
+	go run cmd/genpack/genpack.go -locale $(LANG) -path data/$(LANG).gz > v4/dicts/$(LANG)/pack.go
+	# ----------------
+
+benchcmp:
+	# ensure no govenor weirdness
+	# sudo cpufreq-set -g performance
+	go test -test.benchmem=true -run=NONE -bench=. ./... > bench_current.test
+	git stash save "stashing for benchcmp"
+	@go test -test.benchmem=true -run=NONE -bench=. ./... > bench_head.test
+	git stash pop
+	benchcmp bench_head.test bench_current.test
+
+profile:
+	@mkdir -p pprof/
+	go test -run=NONE -cpuprofile pprof/cpu.prof -memprofile pprof/mem.prof -bench .
+	go tool pprof -pdf pprof/cpu.prof > pprof/cpu.pdf
+	xdg-open pprof/cpu.pdf
+	go tool pprof -weblist=.* pprof/cpu.prof
@@ -0,0 +1,87 @@
+# GoLem
+
+This project is a dictionary based lemmatizer written in go. 
+
+Since v4 all dictionaries need to be gotten individually.
+
+```
+go get github.com/aaaton/golem/v4
+```
+
+
+### What?
+
+A [lemmatizer](https://en.wikipedia.org/wiki/Lemmatisation) is a tool that finds the base form of words.
+
+| Lang    | Input      | Output  |
+| ------- | ---------- | ------- |
+| English | aligning   | align   |
+| Swedish | sprungit   | springa |
+| French  | abattaient | abattre |
+
+It's based on the dictionaries found on [michmech/lemmatization-lists](https://github.com/michmech/lemmatization-lists), which are available under the [Open Database License](https://opendatacommons.org/licenses/odbl/summary/). This project would not be feasible without them.
+
+### Languages
+
+At the moment golem supports English, Swedish, French, Spanish, Italian & German, but adding another language should be no more trouble than getting the dictionary for that language. Some of which are already available on lexiconista. Please let me know if there is something you would like to see in here, or fork the project and create a pull request.
+
+English
+```
+go get github.com/aaaton/golem/v4/dicts/en
+```
+
+Swedish
+```
+go get github.com/aaaton/golem/v4/dicts/sv
+```
+
+French
+```
+go get github.com/aaaton/golem/v4/dicts/fr
+```
+
+German
+```
+go get github.com/aaaton/golem/v4/dicts/de
+```
+
+Spanish
+```
+go get github.com/aaaton/golem/v4/dicts/es
+```
+
+Italian
+```
+go get github.com/aaaton/golem/v4/dicts/it
+```
+
+### Basic usage
+
+```golang
+package main
+
+import (
+	"github.com/aaaton/golem/v4"
+	"github.com/aaaton/golem/v4/dicts/en"
+)
+
+func main() {
+	// the language packages are available under golem/dicts
+	// "en" is for english
+	lemmatizer, err := golem.New(en.New())
+	if err != nil {
+		panic(err)
+	}
+	word := lemmatizer.Lemma("Abducting")
+	if word != "abduct" {
+		panic("The output is not what is expected!")
+	}
+}
+```
+
+### Contributors
+
+- axamon
+- charlesgiroux
+- glaslos
+- ptdewey
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Anton Södergren
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
@@ -0,0 +1,101 @@
+package golem
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// LanguagePack is what each language should implement
+type LanguagePack interface {
+	GetResource() ([]byte, error)
+	GetLocale() string
+}
+
+// Lemmatizer is the key to lemmatizing a word in a language
+type Lemmatizer struct {
+	m map[string]int
+	v [][]string
+}
+
+func newLemmatizerFromBytes(b []byte) (Lemmatizer, error) {
+	lines := strings.Split(string(b), "\n")
+	s := Lemmatizer{
+		m: make(map[string]int),
+		v: [][]string{},
+	}
+	// TODO: Would it be better to do with a reader
+	// instead of loading the full thing into an array?
+
+	// br := bufio.NewReader(bytes.NewReader(b))
+	// line, err := br.ReadString('\n')
+	// for err == nil {
+	// wordIndex := make(map[string])
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+		words := strings.Split(line, "\t")
+		if len(words) < 2 {
+			return s, fmt.Errorf("expected more than 1 form per word")
+		}
+		base := words[0]
+		for _, word := range words {
+			if index, ok := s.m[word]; ok {
+				s.v[index] = append(s.v[index], word)
+			} else {
+				index := len(s.v)
+				s.v = append(s.v, []string{base})
+				s.m[word] = index
+			}
+		}
+	}
+	return s, nil
+}
+
+// New produces a new Lemmatizer
+func New(pack LanguagePack) (*Lemmatizer, error) {
+	resource, err := pack.GetResource()
+	if err != nil {
+		return nil, fmt.Errorf(`Could not open resource file for "%s"`, pack.GetLocale())
+	}
+	l, err := newLemmatizerFromBytes(resource)
+	if err != nil {
+		return nil, fmt.Errorf(`language %s is not valid: %s`, pack.GetLocale(), err)
+	}
+	return &l, nil
+}
+
+// InDict checks if a certain word is in the dictionary
+func (l *Lemmatizer) InDict(word string) bool {
+	_, ok := l.m[strings.ToLower(word)]
+	return ok
+}
+
+// Lemma gets one of the base forms of a word
+func (l *Lemmatizer) Lemma(word string) string {
+	if out, ok := l.m[strings.ToLower(word)]; ok {
+		return l.v[out][0]
+	}
+	return word
+}
+
+// LemmaLower gets one of the base forms of a lower case word
+// expects `word` to be lowercased
+func (l *Lemmatizer) LemmaLower(word string) string {
+	if out, ok := l.m[word]; ok {
+		return l.v[out][0]
+	}
+	return word
+}
+
+// Lemmas gets all the base forms of a word, if multiple exist
+func (l *Lemmatizer) Lemmas(word string) (out []string) {
+	if index, ok := l.m[strings.ToLower(word)]; ok {
+		out := l.v[index]
+		// to get rid of the randomness, we sort the output
+		sort.Strings(out)
+		return out
+	}
+	return []string{word}
+}
@@ -1,3 +1,9 @@
+# github.com/aaaton/golem/v4 v4.0.2
+## explicit; go 1.13
+github.com/aaaton/golem/v4
+# github.com/aaaton/golem/v4/dicts/ru v0.0.0-20250408131944-3488790fc110
+## explicit; go 1.13
+github.com/aaaton/golem/v4/dicts/ru
 # github.com/coder/websocket v1.8.12
 ## explicit; go 1.19
 github.com/coder/websocket
@@ -15,8 +21,6 @@ github.com/google/uuid
 # github.com/kami/hexis v0.0.0 => /home/kami/apps/hexis
 ## explicit; go 1.25.5
 github.com/kami/hexis/pkg/client
-# github.com/kami/praxis v0.0.0 => /home/kami/apps/praxis
-## explicit; go 1.23
 # github.com/mattn/go-isatty v0.0.20
 ## explicit; go 1.15
 github.com/mattn/go-isatty
@@ -79,4 +83,5 @@ modernc.org/memory
 modernc.org/sqlite
 modernc.org/sqlite/lib
 modernc.org/sqlite/vtab
+# github.com/kami/praxis v0.0.0 => /home/kami/apps/praxis
 # github.com/kami/nexus v0.0.0 => /home/kami/apps/nexus