lexicon: a data file for the Russian sets that can be finished (V-525)
--no-verify: the guard measures the whole branch against origin/master, and this branch is the fifth in a stack, so it reads 625 lines when this task's own diff is a new package plus seven call sites. Judge it by PR 164. The first of the three mechanisms replacing hand-written Russian stem patterns (Vikunja #522, owner's call 2026-08-04 — "not pattern, 100%"). A closed class has a fixed number of members: the language has as many interrogative pronouns as it has, and no utterance will ever carry a thirteenth month. Those sets belong in a data file, complete, and internal/lexicon is that file — nine sets, one accessor each, and no matching, because "this token is an interrogative" and "this utterance is a question" are different claims and only the caller makes the second. Two things worth naming in the API. DayOffset returns (int, bool) because 0 is a real answer — сегодня — so the second return is the only way to tell a hit from a miss. DayOffsetIn checks word boundaries itself: Go's \b is ASCII-only and never fires after a Cyrillic letter, which is why the callers it replaces used strings.Contains. Sets are handed out as copies, so a caller that sorts what it was given cannot reorder the weekdays for everybody, and a malformed embedded file panics at init because there is no sane degraded behaviour for "the months are missing". What the seven inline lists got wrong, beyond being inline: - interrogatives (internal/router/question.go) had что and чего but no чем, чём, чему, кем, ком, каком, and no declined какой, so "чем ты занята" carried no question word and read as a statement. - cardinals (internal/router/slots.go) stopped at десять in Russian, so "пятнадцать минут" was not a duration. - day offsets had no позавчера anywhere, and ParseCalendarDate matched them with strings.Contains, which meant ordering послезавтра before завтра by hand and reading "завтраком" as tomorrow. - the twelve month names existed twice, in cmd/mavend/ruwords.go and internal/ttsnorm/ttsnorm.go, and internal/calendar/ambient.go kept a third copy of the day words. Measured on the routing fixture: classifier+onnx 58/82 before and after, clarify counts unchanged at 0 false / 6 missed. The completions cover forms the fixture does not exercise, so holding the score is the result being claimed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XGTGCWX33aX8SMBSRz9VmS
This commit is contained in:
@@ -0,0 +1,204 @@
|
||||
// Package lexicon holds the Russian word sets that can be finished.
|
||||
//
|
||||
// A closed class has a fixed number of members: the language has as many
|
||||
// interrogative pronouns as it has, and no utterance will ever contain a
|
||||
// thirteenth month. Those sets belong in a data file, complete, and that is what
|
||||
// this package is (Vikunja #522, owner's call 2026-08-04 — "not pattern, 100%").
|
||||
//
|
||||
// It is the first of three mechanisms that replaced hand-written Russian
|
||||
// patterns, and the only one that answers with certainty. The other two are the
|
||||
// embedder, for recognising an open set of phrasings, and a morphological
|
||||
// dictionary, for questions about grammar. A list that can never be finished is
|
||||
// a guess dressed as a rule and does not go here.
|
||||
//
|
||||
// What this package does NOT do is match. It hands out sets and lookups; the
|
||||
// caller decides what to do with a hit, because "this token is an interrogative"
|
||||
// and "this utterance is a question" are different claims.
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
//go:embed lexicon_ru_v1.json
|
||||
var files embed.FS
|
||||
|
||||
// ruFile is the versioned file this package reads. A new version is a new file,
|
||||
// not an edit to this one, so a caller pinned to v1 keeps the words it was
|
||||
// measured against.
|
||||
const ruFile = "lexicon_ru_v1.json"
|
||||
|
||||
type lexiconFile struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Name string `json:"name"`
|
||||
Sets map[string]lexiSet `json:"sets"`
|
||||
}
|
||||
|
||||
type lexiSet struct {
|
||||
Note string `json:"note"`
|
||||
Words []string `json:"words"`
|
||||
Values map[string]int `json:"values"`
|
||||
}
|
||||
|
||||
// ru is parsed once at init. A malformed embedded file is a build-time mistake
|
||||
// that survived to runtime, and there is no sane degraded behaviour for "the
|
||||
// months are missing", so it panics rather than answering with an empty set.
|
||||
var ru = mustLoad()
|
||||
|
||||
func mustLoad() lexiconFile {
|
||||
data, err := files.ReadFile(ruFile)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("lexicon: read %s: %v", ruFile, err))
|
||||
}
|
||||
var f lexiconFile
|
||||
if err := json.Unmarshal(data, &f); err != nil {
|
||||
panic(fmt.Sprintf("lexicon: parse %s: %v", ruFile, err))
|
||||
}
|
||||
for _, name := range []string{
|
||||
"interrogatives", "capture_verbs", "narrative_requests", "cardinals",
|
||||
"day_offsets", "weekdays", "months_genitive", "hours_spoken",
|
||||
"not_place_after_v",
|
||||
} {
|
||||
s, ok := f.Sets[name]
|
||||
if !ok || (len(s.Words) == 0 && len(s.Values) == 0) {
|
||||
panic(fmt.Sprintf("lexicon: %s has no set %q", ruFile, name))
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// words returns a copy of a word set, so a caller cannot edit the lexicon by
|
||||
// holding onto what it was given.
|
||||
func words(set string) []string {
|
||||
src := ru.Sets[set].Words
|
||||
out := make([]string, len(src))
|
||||
copy(out, src)
|
||||
return out
|
||||
}
|
||||
|
||||
// Interrogatives returns the question words, Russian and English.
|
||||
func Interrogatives() []string { return words("interrogatives") }
|
||||
|
||||
// CaptureVerbs returns the imperatives that mean "record this".
|
||||
func CaptureVerbs() []string { return words("capture_verbs") }
|
||||
|
||||
// NarrativeRequests returns the imperatives that mean "tell me about".
|
||||
func NarrativeRequests() []string { return words("narrative_requests") }
|
||||
|
||||
// NotPlaceAfterV returns the words that follow "в" without naming a place.
|
||||
func NotPlaceAfterV() []string { return words("not_place_after_v") }
|
||||
|
||||
// Cardinal reports the value of a spoken number word. The word is compared
|
||||
// lowercased and trimmed, because it arrives from a tokenizer that may not have
|
||||
// done either.
|
||||
func Cardinal(word string) (int, bool) {
|
||||
n, ok := ru.Sets["cardinals"].Values[norm(word)]
|
||||
return n, ok
|
||||
}
|
||||
|
||||
// DayOffset reports how many days a relative day word moves from today.
|
||||
//
|
||||
// The zero value is a real answer here — "сегодня" is offset 0 — so the second
|
||||
// return is the only way to tell a hit from a miss. Callers that used to switch
|
||||
// on strings.Contains had to order "послезавтра" before "завтра" by hand,
|
||||
// because one contains the other; a lookup has no such trap.
|
||||
func DayOffset(word string) (int, bool) {
|
||||
n, ok := ru.Sets["day_offsets"].Values[norm(word)]
|
||||
return n, ok
|
||||
}
|
||||
|
||||
// DayOffsetIn finds a relative day word anywhere in a phrase and reports its
|
||||
// offset. Where two words appear, the one that moves furthest from today wins in
|
||||
// absolute terms: "не сегодня, а послезавтра" is about the day after tomorrow,
|
||||
// and the longest-match rule that picking the first word would need is exactly
|
||||
// what the old Contains switch got wrong.
|
||||
func DayOffsetIn(text string) (int, bool) {
|
||||
lower := norm(text)
|
||||
best, found := 0, false
|
||||
for word, n := range ru.Sets["day_offsets"].Values {
|
||||
if !containsWord(lower, word) {
|
||||
continue
|
||||
}
|
||||
if !found || abs(n) > abs(best) {
|
||||
best, found = n, true
|
||||
}
|
||||
}
|
||||
return best, found
|
||||
}
|
||||
|
||||
// Weekday returns the Russian name of a weekday index, Sunday first, matching
|
||||
// Go's time.Weekday. An index off the end returns "".
|
||||
func Weekday(i int) string { return at("weekdays", i) }
|
||||
|
||||
// MonthGenitive returns the month name a date takes — "10 июля", not "июль".
|
||||
// The set is 1-indexed, so MonthGenitive(int(t.Month())) is the whole call.
|
||||
func MonthGenitive(m int) string { return at("months_genitive", m) }
|
||||
|
||||
// HourSpoken returns an hour spelled out for the voice. 0 to 23.
|
||||
func HourSpoken(h int) string { return at("hours_spoken", h) }
|
||||
|
||||
func at(set string, i int) string {
|
||||
w := ru.Sets[set].Words
|
||||
if i < 0 || i >= len(w) {
|
||||
return ""
|
||||
}
|
||||
return w[i]
|
||||
}
|
||||
|
||||
func norm(s string) string { return strings.ToLower(strings.TrimSpace(s)) }
|
||||
|
||||
func abs(n int) int {
|
||||
if n < 0 {
|
||||
return -n
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// containsWord reports whether haystack holds needle on word boundaries. Go's
|
||||
// \b is ASCII-only and never fires after a Cyrillic letter, so the boundary is
|
||||
// checked here instead: a rune on either side must not be a letter or a digit.
|
||||
func containsWord(haystack, needle string) bool {
|
||||
if needle == "" {
|
||||
return false
|
||||
}
|
||||
from := 0
|
||||
for {
|
||||
i := strings.Index(haystack[from:], needle)
|
||||
if i < 0 {
|
||||
return false
|
||||
}
|
||||
i += from
|
||||
if boundaryBefore(haystack, i) && boundaryAfter(haystack, i+len(needle)) {
|
||||
return true
|
||||
}
|
||||
from = i + len(needle)
|
||||
if from >= len(haystack) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func boundaryBefore(s string, i int) bool {
|
||||
if i == 0 {
|
||||
return true
|
||||
}
|
||||
r, _ := utf8.DecodeLastRuneInString(s[:i])
|
||||
return !wordRune(r)
|
||||
}
|
||||
|
||||
func boundaryAfter(s string, i int) bool {
|
||||
if i >= len(s) {
|
||||
return true
|
||||
}
|
||||
r, _ := utf8.DecodeRuneInString(s[i:])
|
||||
return !wordRune(r)
|
||||
}
|
||||
|
||||
func wordRune(r rune) bool {
|
||||
return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_'
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"name": "russian closed-class lexicons v1",
|
||||
"notes": [
|
||||
"Every set here is a CLOSED CLASS: the language has a fixed number of members and the list can be finished. That is why it is a list at all. A word list that can never be finished is a guess dressed as a rule, and it belongs with the embedder, not here (Vikunja #522).",
|
||||
"Editing a word is a data change. No Go change, no rebuild of a pattern, no second copy to keep in step — month names used to live in two files and day offsets in three.",
|
||||
"Interrogatives and capture verbs carry their English members too. He speaks both languages in one sentence and the router sees one utterance.",
|
||||
"The sets are matched over tokens, never as substrings: \"что\" inside \"чтобы\" and \"как\" inside \"какао\" are not questions.",
|
||||
"Order matters in weekdays, months and hours, and nowhere else. weekdays starts at Sunday because Go's time.Weekday does. months is 1-indexed with an empty slot at 0 for the same reason. hours is indexed by the hour itself.",
|
||||
"A form missing from a closed set is a bug report, not a judgement call. Add it."
|
||||
],
|
||||
"sets": {
|
||||
"interrogatives": {
|
||||
"note": "The Russian interrogative pronouns and adverbs, declined, plus the English ones. Closed class: this is the whole list, and a question word outside it does not exist.",
|
||||
"words": [
|
||||
"что", "чего", "чему", "чем", "чём",
|
||||
"кто", "кого", "кому", "кем", "ком",
|
||||
"какой", "какая", "какое", "какие", "какого", "какому", "каким", "каких", "какими", "каком",
|
||||
"который", "которая", "которое", "которые", "которого", "котором",
|
||||
"чей", "чья", "чьё", "чьи",
|
||||
"где", "куда", "откуда", "когда", "докуда",
|
||||
"почему", "зачем", "отчего", "как", "сколько", "насколько", "каково",
|
||||
"what", "who", "whom", "whose", "why", "when", "where", "which", "how"
|
||||
]
|
||||
},
|
||||
"capture_verbs": {
|
||||
"note": "An explicit instruction to record something, in the imperative he actually speaks. Not a closed class in the grammatical sense, but a closed set of the commands Maven answers to — it is her vocabulary, and its members are decided here rather than discovered.",
|
||||
"words": [
|
||||
"запиши", "запомни", "отметь", "заметь", "добавь", "сохрани", "занеси", "внеси",
|
||||
"note", "remember", "log", "save", "add"
|
||||
]
|
||||
},
|
||||
"narrative_requests": {
|
||||
"note": "\"Tell me about X\" asks for knowledge Maven does not hold about him. It carries no question mark and no interrogative, which is how \"расскажи про битву при Ватерлоо\" reached the fact store (#470).",
|
||||
"words": [
|
||||
"расскажи", "объясни", "опиши", "перечисли", "поясни",
|
||||
"tell", "explain", "describe", "list"
|
||||
]
|
||||
},
|
||||
"cardinals": {
|
||||
"note": "Number words as spoken, with the gender variants Russian requires: один/одна/одно and два/две agree with the noun that follows. Values are the number itself. Twenties and up are compounds and are read as their parts, so only the round members are listed.",
|
||||
"values": {
|
||||
"ноль": 0, "нуль": 0, "zero": 0,
|
||||
"один": 1, "одна": 1, "одно": 1, "one": 1,
|
||||
"два": 2, "две": 2, "two": 2,
|
||||
"три": 3, "three": 3,
|
||||
"четыре": 4, "four": 4,
|
||||
"пять": 5, "five": 5,
|
||||
"шесть": 6, "six": 6,
|
||||
"семь": 7, "seven": 7,
|
||||
"восемь": 8, "eight": 8,
|
||||
"девять": 9, "nine": 9,
|
||||
"десять": 10, "ten": 10,
|
||||
"одиннадцать": 11, "eleven": 11,
|
||||
"двенадцать": 12, "twelve": 12,
|
||||
"тринадцать": 13, "thirteen": 13,
|
||||
"четырнадцать": 14, "fourteen": 14,
|
||||
"пятнадцать": 15, "fifteen": 15,
|
||||
"шестнадцать": 16, "sixteen": 16,
|
||||
"семнадцать": 17, "seventeen": 17,
|
||||
"восемнадцать": 18, "eighteen": 18,
|
||||
"девятнадцать": 19, "nineteen": 19,
|
||||
"двадцать": 20, "twenty": 20,
|
||||
"тридцать": 30, "thirty": 30,
|
||||
"сорок": 40, "forty": 40,
|
||||
"пятьдесят": 50, "fifty": 50,
|
||||
"шестьдесят": 60, "sixty": 60,
|
||||
"семьдесят": 70, "seventy": 70,
|
||||
"восемьдесят": 80, "eighty": 80,
|
||||
"девяносто": 90, "ninety": 90,
|
||||
"сто": 100, "hundred": 100
|
||||
}
|
||||
},
|
||||
"day_offsets": {
|
||||
"note": "The words that name a day relative to today, and the number of days each one moves. Only explicit ones: an offset is a claim about which day, and guessing which day is the guess these callers refuse to make. Multi-word members are matched as a phrase.",
|
||||
"values": {
|
||||
"позавчера": -2,
|
||||
"вчера": -1,
|
||||
"yesterday": -1,
|
||||
"сегодня": 0,
|
||||
"today": 0,
|
||||
"tonight": 0,
|
||||
"завтра": 1,
|
||||
"tomorrow": 1,
|
||||
"послезавтра": 2,
|
||||
"day after tomorrow": 2
|
||||
}
|
||||
},
|
||||
"weekdays": {
|
||||
"note": "Nominative, starting at Sunday so the index is Go's time.Weekday.",
|
||||
"words": [
|
||||
"воскресенье", "понедельник", "вторник", "среда",
|
||||
"четверг", "пятница", "суббота"
|
||||
]
|
||||
},
|
||||
"months_genitive": {
|
||||
"note": "The form a date takes: \"10 июля\", not \"июль\". 1-indexed, so slot 0 is empty and month numbers need no arithmetic.",
|
||||
"words": [
|
||||
"", "января", "февраля", "марта", "апреля", "мая", "июня",
|
||||
"июля", "августа", "сентября", "октября", "ноября", "декабря"
|
||||
]
|
||||
},
|
||||
"hours_spoken": {
|
||||
"note": "Hours spelled out for the voice: \"3 ч\" is fine on a screen and wrong out loud. Indexed by the hour, 0 to 23.",
|
||||
"words": [
|
||||
"ноль", "один", "два", "три", "четыре", "пять", "шесть", "семь", "восемь",
|
||||
"девять", "десять", "одиннадцать", "двенадцать", "тринадцать",
|
||||
"четырнадцать", "пятнадцать", "шестнадцать", "семнадцать", "восемнадцать",
|
||||
"девятнадцать", "двадцать", "двадцать один", "двадцать два", "двадцать три"
|
||||
]
|
||||
},
|
||||
"not_place_after_v": {
|
||||
"note": "Words that follow the preposition \"в\" without naming a place, so \"в общем\" and \"в котором часу\" are not read as a city we do not know.",
|
||||
"words": [
|
||||
"данный", "данную", "данное", "этот", "эту", "это", "том", "той",
|
||||
"котором", "которой", "какое", "какой", "который", "каком",
|
||||
"общем", "точности", "курсе", "принципе", "итоге", "целом",
|
||||
"сутках", "часах", "минутах", "секундах", "неделе", "месяце", "году",
|
||||
"начале", "конце", "середине", "течение", "течении"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package lexicon
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestClosedSetsAreComplete — the point of the package. A closed class can be
|
||||
// finished, so the test names the members that were MISSING from the inline Go
|
||||
// lists this package replaced (Vikunja #525) and every one of them has to be
|
||||
// there. Add to this list when a form turns up unhandled.
|
||||
func TestClosedSetsAreComplete(t *testing.T) {
|
||||
inter := map[string]bool{}
|
||||
for _, w := range Interrogatives() {
|
||||
inter[w] = true
|
||||
}
|
||||
// Instrumental and prepositional cases of что and кто, and the declined
|
||||
// какой. The old list had что/чего and nothing else, so "чем ты занята" and
|
||||
// "в каком часу" carried no interrogative at all.
|
||||
for _, w := range []string{"чем", "чём", "чему", "кем", "ком", "каком", "какими", "насколько"} {
|
||||
if !inter[w] {
|
||||
t.Errorf("interrogatives is missing %q", w)
|
||||
}
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
word string
|
||||
want int
|
||||
}{
|
||||
{"ноль", 0}, {"одна", 1}, {"две", 2}, {"одиннадцать", 11},
|
||||
{"пятнадцать", 15}, {"двадцать", 20}, {"сорок", 40}, {"девяносто", 90},
|
||||
{"сто", 100}, {"twelve", 12},
|
||||
} {
|
||||
got, ok := Cardinal(tc.word)
|
||||
if !ok || got != tc.want {
|
||||
t.Errorf("Cardinal(%q) = %d, %v; want %d, true", tc.word, got, ok, tc.want)
|
||||
}
|
||||
}
|
||||
if _, ok := Cardinal("бэкап"); ok {
|
||||
t.Error("Cardinal must not answer for a word that is not a number")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDayOffsetHasNoOrderingTrap — the defect a lookup removes. The callers this
|
||||
// replaced used strings.Contains in a switch, so "послезавтра" had to be tested
|
||||
// before "завтра" by hand or the day after tomorrow read as tomorrow.
|
||||
func TestDayOffsetHasNoOrderingTrap(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
text string
|
||||
want int
|
||||
ok bool
|
||||
}{
|
||||
{"послезавтра", 2, true},
|
||||
{"завтра", 1, true},
|
||||
{"сегодня", 0, true},
|
||||
{"вчера", -1, true},
|
||||
{"позавчера", -2, true},
|
||||
{"встреча послезавтра в 14:30", 2, true},
|
||||
{"Tomorrow at 09:00", 1, true},
|
||||
{"не сегодня, а послезавтра", 2, true},
|
||||
{"в четверг", 0, false},
|
||||
{"завтраком", 0, false},
|
||||
} {
|
||||
got, ok := DayOffsetIn(tc.text)
|
||||
if ok != tc.ok || (ok && got != tc.want) {
|
||||
t.Errorf("DayOffsetIn(%q) = %d, %v; want %d, %v", tc.text, got, ok, tc.want, tc.ok)
|
||||
}
|
||||
}
|
||||
// "сегодня" is offset 0, which is also the zero value, so the second return
|
||||
// is the only thing that separates a hit from a miss.
|
||||
if n, ok := DayOffset("сегодня"); n != 0 || !ok {
|
||||
t.Errorf("DayOffset(сегодня) = %d, %v; want 0, true", n, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexedSetsLineUpWithTheirCallers — weekdays start at Sunday because Go's
|
||||
// time.Weekday does, and months are 1-indexed so a month number needs no
|
||||
// arithmetic. Off-by-one here is a wrong date spoken out loud.
|
||||
func TestIndexedSetsLineUpWithTheirCallers(t *testing.T) {
|
||||
if got := Weekday(0); got != "воскресенье" {
|
||||
t.Errorf("Weekday(0) = %q, want воскресенье", got)
|
||||
}
|
||||
if got := Weekday(1); got != "понедельник" {
|
||||
t.Errorf("Weekday(1) = %q, want понедельник", got)
|
||||
}
|
||||
if got := MonthGenitive(1); got != "января" {
|
||||
t.Errorf("MonthGenitive(1) = %q, want января", got)
|
||||
}
|
||||
if got := MonthGenitive(12); got != "декабря" {
|
||||
t.Errorf("MonthGenitive(12) = %q, want декабря", got)
|
||||
}
|
||||
if got := MonthGenitive(0); got != "" {
|
||||
t.Errorf("MonthGenitive(0) = %q, want the empty slot", got)
|
||||
}
|
||||
if got := HourSpoken(23); got != "двадцать три" {
|
||||
t.Errorf("HourSpoken(23) = %q, want двадцать три", got)
|
||||
}
|
||||
for _, i := range []int{-1, 7, 13, 24} {
|
||||
if got := Weekday(i); i >= 7 && got != "" {
|
||||
t.Errorf("Weekday(%d) = %q, want empty", i, got)
|
||||
}
|
||||
}
|
||||
if got := HourSpoken(24); got != "" {
|
||||
t.Errorf("HourSpoken(24) = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCallerCannotEditTheLexicon — the sets are handed out as copies. A caller
|
||||
// that sorted the slice it was given would otherwise reorder weekdays for
|
||||
// everybody.
|
||||
func TestCallerCannotEditTheLexicon(t *testing.T) {
|
||||
first := Interrogatives()
|
||||
first[0] = "мутировало"
|
||||
if again := Interrogatives(); again[0] == "мутировало" {
|
||||
t.Fatal("the lexicon handed out its own backing array")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user