Merge the sweep tail: four files ask the dictionary (#169)

This commit was merged in pull request #169.
This commit is contained in:
2026-08-04 19:25:10 +02:00
11 changed files with 259 additions and 105 deletions
+4 -2
View File
@@ -231,8 +231,10 @@ fact or a route is the defect; a regex over structured input — HTML, MIME, JSO
argv list — is not. Before writing a Russian word list, pick one of these:
- **`internal/lexicon`** — closed classes, in `lexicon_ru_v1.json`. Interrogatives,
capture verbs, cardinals, day offsets, weekdays, months, spoken hours. Editing a word is
a data change, and there is exactly one copy: months used to live in three files.
capture verbs, reminder verbs, cardinals, day offsets, parts of day, weekdays, months,
spoken hours. Editing a word is a data change, and there is exactly one copy: months used
to live in three files. Cardinals carry the oblique forms, because a spoken time declines
and `в семь` / `к семи` are one hour.
- **`internal/morph`** — grammar, from the vendored golem Russian dictionary. `IsVerbForm`
and `SameWord`. Note that lemma matching is BROADER than stem-plus-one-ending, so a verb
slot that means the imperative must be matched exactly — `говори` and `говорил` are one
+76 -13
View File
@@ -6,6 +6,8 @@ import (
"log"
"strings"
"time"
"github.com/kami/maven/internal/morph"
)
// Command history — "что я тебе говорил?", "что ты записала сегодня?"
@@ -15,18 +17,34 @@ import (
// storage: everything he tapped in is already a row with a source and a
// timestamp, and this only reads them back.
// historyMarkers — the ways he asks what he told her. Each entry is a pair of
// substrings that must BOTH appear, because either half alone is a different
// question: "что я говорил про сервер" is a recall question the notes pass
// answers better, and "что ты записала" with no "что" is not a question at all.
var historyMarkers = [][2]string{
{"что я", "говорил"},
{"что я", "сказал"},
{"что я", "рассказ"},
{"что ты", "записал"},
{"что ты", "запомнил"},
{"что я", "отмечал"},
{"что я", "отметил"},
// A history question needs three things in one utterance: the interrogative,
// whose turn is being asked about, and a verb of saying or recording. Any two of
// them are a different question. "что я говорил про сервер" names a topic and
// the notes pass answers it better; "записал молоко" is a capture.
//
// The verbs are matched by lemma through internal/morph, not by a truncated
// prefix (Vikunja #530). The pairs here used to hold "рассказ" and "записал",
// which is the defect V-528 fixed in complaint.go: "рассказ" is also the noun,
// so "что я рассказал ей" and "что я читал рассказ" were the same string test.
// Aspect pairs are separate lemmas in the dictionary, so both members are listed.
var (
// historySpokenVerbs — what HE did. "что я тебе говорил".
historySpokenVerbs = []string{"говорить", "сказать", "рассказать", "рассказывать", "отметить", "отмечать"}
// historyRecordedVerbs — what SHE did with it. "что ты записала сегодня".
historyRecordedVerbs = []string{"записать", "запомнить", "отметить", "отмечать"}
// firstPersonSubjects and secondPersonSubjects — whose turn the question is
// about. Only the subject forms: "что я тебе говорил" is his turn, and the
// dative "тебе" in it is not the subject.
firstPersonSubjects = []string{"я"}
secondPersonSubjects = []string{"ты"}
)
// historyMarkersEn — the English pairs, kept as substrings because the
// dictionary is Russian. Each half alone is a different question, the same way
// the Russian test needs all three parts.
var historyMarkersEn = [][2]string{
{"what did i", "tell"},
{"what did you", "record"},
}
@@ -47,11 +65,56 @@ func isHistoryQuery(u string) bool {
return false
}
}
for _, pair := range historyMarkers {
for _, pair := range historyMarkersEn {
if strings.Contains(s, pair[0]) && strings.Contains(s, pair[1]) {
return true
}
}
toks := historyTokens(s)
if !hasAny(toks, "что", "чего") {
return false
}
if hasAny(toks, firstPersonSubjects...) && hasVerbForm(toks, historySpokenVerbs) {
return true
}
return hasAny(toks, secondPersonSubjects...) && hasVerbForm(toks, historyRecordedVerbs)
}
// historyTokens splits an utterance into bare words. The punctuation goes
// because "говорил?" is the same word as "говорил".
func historyTokens(s string) []string {
toks := strings.Fields(s)
out := make([]string, 0, len(toks))
for _, t := range toks {
if t = strings.Trim(t, ".,!?;:—–-()\"'«»"); t != "" {
out = append(out, t)
}
}
return out
}
func hasAny(toks []string, want ...string) bool {
for _, t := range toks {
for _, w := range want {
if t == w {
return true
}
}
}
return false
}
// hasVerbForm reports whether any token is a form of any of the lemmas. Both
// sides go through the dictionary, so a caller may name the infinitive and he
// may say the past tense.
func hasVerbForm(toks []string, lemmas []string) bool {
for _, t := range toks {
for _, l := range lemmas {
if morph.SameWord(t, l) {
return true
}
}
}
return false
}
+11
View File
@@ -41,6 +41,17 @@ func TestIsHistoryQuery(t *testing.T) {
{"что я тебе говорил?", true},
{"что ты записала сегодня?", true},
{"что я отмечал?", true},
// Forms the truncated prefixes did not reach. The dictionary answers
// these because it lemmatises both sides (V-530).
{"что я тебе рассказывал?", true},
{"что я сказала вчера", true},
{"что ты запомнила?", true},
// The noun, not the verb. "рассказ" was a prefix of the old pair, so
// this read as a history question — the same defect V-528 fixed in
// complaint.go, where "лаг" matched "лагерь".
{"что я читал рассказ", false},
// A verb of saying with nobody saying it.
{"что записать?", false},
// A named topic is a recall question, and the notes pass answers it
// better than a list of the last five facts does.
{"что я говорил про сервер?", false},
+35 -3
View File
@@ -2,12 +2,20 @@ package main
import (
"regexp"
"sort"
"strings"
"github.com/kami/maven/internal/lexicon"
)
// reminderMarker — the words that open a reminder. Stripped because they are
// the instruction, not the thing to say at the hour.
var reminderMarker = regexp.MustCompile(`(?i)^\s*(?:напомни(?:те)?|напомнить|remind)\s*(?:мне|me)?[\s,:—-]*`)
//
// The verbs come from the lexicon (Vikunja #530). They are a closed set of the
// commands she answers to, exactly like capture_verbs, and the literal that
// stood here knew four of them.
var reminderMarker = regexp.MustCompile(`(?i)^\s*(?:` + alternation(lexicon.ReminderVerbs()) +
`)\s*(?:мне|me)?[\s,:—-]*`)
// reminderTimeWords — the time expressions a reminder carries, removed from
// the body because the fire time is already a column. Ordered longest-first
@@ -17,12 +25,36 @@ var reminderMarker = regexp.MustCompile(`(?i)^\s*(?:напомни(?:те)?|на
// Go's \b is ASCII-only and never fires next to a Cyrillic letter, so the word
// boundaries here are written out as whitespace or an end of string — the same
// trap the agenda grammars hit.
//
// The Russian word lists are gone (Vikunja #530). The day words are
// lexicon.DayOffsetWords, which is why "вчера" and "позавчера" are stripped now
// and were not before, and the times of day are lexicon.PartsOfDay. What is
// still written out here is the shape of a clock reading — a preposition, digits,
// a colon — which is structured input rather than a claim about Russian.
var reminderTimeWords = []*regexp.Regexp{
regexp.MustCompile(`(?i)(^|\s)через\s+\S+(\s+(часа?|часов|минут[уы]?|секунд[уы]?|дня|дней|недел[юи]))?(\s|$)`),
regexp.MustCompile(`(?i)(^|\s)(в|во)\s+\d{1,2}(:\d{2})?(\s*(часа?|часов))?(\s*(утра|вечера|дня|ночи))?(\s|$)`),
regexp.MustCompile(`(?i)(^|\s)(завтра|послезавтра|сегодня|вечером|утром|днём|днем|ночью)(\s|$)`),
regexp.MustCompile(`(?i)(^|\s)(` + alternation(lexicon.DayOffsetWords()) + `)(\s|$)`),
regexp.MustCompile(`(?i)(^|\s)(` + alternation(lexicon.PartsOfDay()) + `)(\s|$)`),
regexp.MustCompile(`(?i)(^|\s)(at|in)\s+\d{1,2}(:\d{2})?\s*(am|pm)?(\s|$)`),
regexp.MustCompile(`(?i)(^|\s)(tomorrow|today|tonight)(\s|$)`),
}
// alternation folds a lexicon set into one regexp branch, longest member first
// so "послезавтра" is not matched as "завтра" with a tail left behind. Sorted
// rather than taken as given, because two members of equal length must still
// produce the same pattern on every build.
func alternation(set []string) string {
out := make([]string, 0, len(set))
for _, w := range set {
out = append(out, regexp.QuoteMeta(w))
}
sort.Slice(out, func(i, j int) bool {
if len(out[i]) != len(out[j]) {
return len(out[i]) > len(out[j])
}
return out[i] < out[j]
})
return strings.Join(out, "|")
}
// reminderBody is what she says at the hour.
+9 -1
View File
@@ -63,7 +63,7 @@ func mustLoad() lexiconFile {
for _, name := range []string{
"interrogatives", "capture_verbs", "narrative_requests", "cardinals",
"day_offsets", "weekdays", "months_genitive", "hours_spoken",
"not_place_after_v",
"not_place_after_v", "parts_of_day", "reminder_verbs",
} {
s, ok := f.Sets[name]
if !ok || (len(s.Words) == 0 && len(s.Values) == 0) {
@@ -104,6 +104,14 @@ func FirstPerson() []string { return words("first_person") }
// NotPlaceAfterV returns the words that follow "в" without naming a place.
func NotPlaceAfterV() []string { return words("not_place_after_v") }
// PartsOfDay returns the one-word names for a time of day: "вечером", "утром".
// They say which part of a day and never which day, so a caller that needs the
// day wants DayOffsetWords instead.
func PartsOfDay() []string { return words("parts_of_day") }
// ReminderVerbs returns the imperatives that open a reminder.
func ReminderVerbs() []string { return words("reminder_verbs") }
// Cardinal reports the value of a spoken number word. The word is compared
// lowercased and trimmed, because it arrives from a tokenizer that may not have
// done either.
+43 -29
View File
@@ -38,37 +38,37 @@
]
},
"cardinals": {
"note": "Number words as spoken, with the gender variants Russian requires: один/одна/одно and два/две agree with the noun that follows. Values are the number itself. Twenties and up are compounds and are read as their parts, so only the round members are listed.",
"note": "Number words as spoken, with the gender variants Russian requires (один/одна/одно and два/две agree with the noun that follows) and the oblique forms, because a spoken time declines: \"в семь\", \"к семи\", \"около семи\" are three forms of one hour (Vikunja #530). Values are the number itself. Twenties and up are compounds and are read as their parts, so only the round members are listed.",
"values": {
"ноль": 0, "нуль": 0, "zero": 0,
"один": 1, "одна": 1, "одно": 1, "one": 1,
"два": 2, "две": 2, "two": 2,
"три": 3, "three": 3,
"четыре": 4, "four": 4,
"пять": 5, "five": 5,
"шесть": 6, "six": 6,
"семь": 7, "seven": 7,
"восемь": 8, "eight": 8,
"девять": 9, "nine": 9,
"десять": 10, "ten": 10,
"одиннадцать": 11, "eleven": 11,
"двенадцать": 12, "twelve": 12,
"тринадцать": 13, "thirteen": 13,
"четырнадцать": 14, "fourteen": 14,
"пятнадцать": 15, "fifteen": 15,
"шестнадцать": 16, "sixteen": 16,
"семнадцать": 17, "seventeen": 17,
"восемнадцать": 18, "eighteen": 18,
"девятнадцать": 19, "nineteen": 19,
"двадцать": 20, "twenty": 20,
"тридцать": 30, "thirty": 30,
"сорок": 40, "forty": 40,
"пятьдесят": 50, "fifty": 50,
"шестьдесят": 60, "sixty": 60,
"семьдесят": 70, "seventy": 70,
"восемьдесят": 80, "eighty": 80,
"девяносто": 90, "ninety": 90,
"сто": 100, "hundred": 100
"один": 1, "одна": 1, "одно": 1, "одного": 1, "одной": 1, "одну": 1, "one": 1,
"два": 2, "две": 2, "двух": 2, "two": 2,
"три": 3, "трёх": 3, "трех": 3, "three": 3,
"четыре": 4, "четырёх": 4, "четырех": 4, "four": 4,
"пять": 5, "пяти": 5, "five": 5,
"шесть": 6, "шести": 6, "six": 6,
"семь": 7, "семи": 7, "seven": 7,
"восемь": 8, "восьми": 8, "eight": 8,
"девять": 9, "девяти": 9, "nine": 9,
"десять": 10, "десяти": 10, "ten": 10,
"одиннадцать": 11, "одиннадцати": 11, "eleven": 11,
"двенадцать": 12, "двенадцати": 12, "twelve": 12,
"тринадцать": 13, "тринадцати": 13, "thirteen": 13,
"четырнадцать": 14, "четырнадцати": 14, "fourteen": 14,
"пятнадцать": 15, "пятнадцати": 15, "fifteen": 15,
"шестнадцать": 16, "шестнадцати": 16, "sixteen": 16,
"семнадцать": 17, "семнадцати": 17, "seventeen": 17,
"восемнадцать": 18, "восемнадцати": 18, "eighteen": 18,
"девятнадцать": 19, "девятнадцати": 19, "nineteen": 19,
"двадцать": 20, "двадцати": 20, "twenty": 20,
"тридцать": 30, "тридцати": 30, "thirty": 30,
"сорок": 40, "сорока": 40, "forty": 40,
"пятьдесят": 50, "пятидесяти": 50, "fifty": 50,
"шестьдесят": 60, "шестидесяти": 60, "sixty": 60,
"семьдесят": 70, "семидесяти": 70, "seventy": 70,
"восемьдесят": 80, "восьмидесяти": 80, "eighty": 80,
"девяносто": 90, "девяноста": 90, "ninety": 90,
"сто": 100, "ста": 100, "hundred": 100
}
},
"day_offsets": {
@@ -134,6 +134,20 @@
"сутках", "часах", "минутах", "секундах", "неделе", "месяце", "году",
"начале", "конце", "середине", "течение", "течении"
]
},
"parts_of_day": {
"note": "The times of day named as one word, in the instrumental case Russian uses for when something happens. A day has as many parts as it has, so this set is finished. They are not day offsets: \"вечером\" says which part of a day, never which day (Vikunja #530).",
"words": [
"утром", "днём", "днем", "вечером", "ночью",
"morning", "afternoon", "evening", "night"
]
},
"reminder_verbs": {
"note": "The imperatives that mean \"remind me\", in the forms he speaks. The same kind of set as capture_verbs and decided the same way: it is her vocabulary, not a discovery about Russian (Vikunja #530).",
"words": [
"напомни", "напомните", "напомнить", "напоминай",
"remind"
]
}
}
}
+1 -1
View File
@@ -15,7 +15,7 @@ func narrativeRouter(t *testing.T) *Router {
r.grammars = append(r.grammars, AgendaQueryGrammars()...)
r.grammars = append(r.grammars, TaskListGrammar())
r.grammars = append(r.grammars, TaskCaptureGrammar())
r.grammars = append(r.grammars, NarrativeQueryGrammars()[1])
r.grammars = append(r.grammars, NarrativeQueryGrammars()...)
return r
}
+28 -32
View File
@@ -1,38 +1,34 @@
package router
import "strings"
import (
"strconv"
"strings"
// ruNumerals — spoken numbers as digits, for the clock hours and the minutes
// that follow them. Every case ending he might say is listed rather than
// stemmed: "в семь", "к семи", "около семи" are three forms of one hour, and a
// prefix rule short enough to cover them also matches "семья".
"github.com/kami/maven/internal/lexicon"
)
// hourNouns — the two words that are the hour noun as often as they are the
// number one. "в час дня" means one o'clock, so rewriting it to "в 1 дня" is
// right either way.
//
// Stops at thirty, which is as far as a spoken time goes ("без двадцати
// восемь", "в половине шестого"). Anything larger is said in digits.
var ruNumerals = map[string]string{
"один": "1", "одного": "1", "одну": "1", "час": "1", "часу": "1",
"два": "2", "две": "2", "двух": "2",
"три": "3", "трёх": "3", "трех": "3",
"четыре": "4", "четырёх": "4", "четырех": "4",
"пять": "5", "пяти": "5",
"шесть": "6", "шести": "6",
"семь": "7", "семи": "7",
"восемь": "8", "восьми": "8",
"девять": "9", "девяти": "9",
"десять": "10", "десяти": "10",
"одиннадцать": "11", "одиннадцати": "11",
"двенадцать": "12", "двенадцати": "12",
"тринадцать": "13", "тринадцати": "13",
"четырнадцать": "14", "четырнадцати": "14",
"пятнадцать": "15", "пятнадцати": "15",
"шестнадцать": "16", "шестнадцати": "16",
"семнадцать": "17", "семнадцати": "17",
"восемнадцать": "18", "восемнадцати": "18",
"девятнадцать": "19", "девятнадцати": "19",
"двадцать": "20", "двадцати": "20",
"тридцать": "30", "тридцати": "30",
"сорок": "40", "сорока": "40",
"пятьдесят": "50", "пятидесяти": "50",
// They are not cardinals and do not belong in the lexicon's number set: nobody
// counts "час яблок". Everything else this file reads comes from
// lexicon.Cardinal, which is where the number words live complete, oblique forms
// included (Vikunja #530). The table here used to be a second copy that stopped
// at fifty and disagreed with the lexicon about its own members.
var hourNouns = map[string]string{"час": "1", "часу": "1"}
// numeralDigit reports the digits a spoken number is written as, for a clock
// hour or the minutes after it.
func numeralDigit(word string) (string, bool) {
if d, ok := hourNouns[word]; ok {
return d, true
}
n, ok := lexicon.Cardinal(word)
if !ok {
return "", false
}
return strconv.Itoa(n), true
}
// numeralContext — the words that make a numeral a time. A numeral is only
@@ -67,7 +63,7 @@ func SpellOutDigits(text string) string {
copy(out, toks)
for i, tok := range toks {
key := strings.ToLower(strings.Trim(tok, ".,!?;:«»\"'"))
digit, ok := ruNumerals[key]
digit, ok := numeralDigit(key)
if !ok {
continue
}
+14 -17
View File
@@ -230,27 +230,24 @@ func AgendaQueryGrammars() []Grammar {
// herself, and the query chain has no source for either.
var chatNarrativeTopics = regexp.MustCompile(`(?i)(анекдот|шутк|сказк|истори[юи]\s+на\s+ночь|о\s+себе|про\s+себя|о\s+нас|про\s+нас)`)
// NarrativeQueryGrammars — stage-0 grammars for the two question shapes that
// carry no question mark and no interrogative, and so reached the resident
// model with nothing deterministic in front of them (Vikunja #498).
// NarrativeQueryGrammars — the stage-0 grammar for "расскажи про X", a question
// shape that carries no question mark and no interrogative, and so reached the
// resident model with nothing deterministic in front of it (Vikunja #498).
//
// Both were routed IntentFact by the model. The fact gate catches the write and
// re-runs the turn as a query, so nothing breaks today; what they cost is a full
// model round trip to reach a decision two patterns can make offline, and a
// wrong row on the routing fixture.
// The model routed it IntentFact. The fact gate catches the write and re-runs
// the turn as a query, so nothing broke; what it cost is a full model round trip
// to reach a decision one pattern makes offline, and a wrong row on the routing
// fixture.
//
// Wired after the agenda grammars, which is where their overlap resolves:
// "расскажи, что у меня сегодня" is claimed here as a query either way.
// It held a second grammar named rest-of-day-query until V-530. fe489df merged
// task/467 into the sweep line and both sides had landed V-498, so the merge
// kept both blocks textually. buildRouter wires the agenda grammars first and
// the agenda copy claims every case this one did, so it could never fire.
//
// Wired after the agenda grammars, which is where the overlap resolves:
// "расскажи, что у меня сегодня" is claimed there as a query either way.
func NarrativeQueryGrammars() []Grammar {
return []Grammar{
{
// "что дальше?" — the rest of the day. IsRestOfDayQuery already
// recognises it downstream in the query chain, but that runs after
// the routing decision, and the routing decision was fact.
Name: "rest-of-day-query",
Pattern: regexp.MustCompile(`(?i)(^|\s)(что|чего)\s+(там\s+|потом\s+)?дальше(\s|[?!.]|$)|(^|\s)what'?s?\s+next(\s|[?!.]|$)`),
Build: agendaQueryBuild,
},
{
// "расскажи про X" — a world question phrased as an instruction.
// The lexicon is narrativeRequests, already written for the
+30 -5
View File
@@ -8,6 +8,9 @@ import (
"net/http"
"net/url"
"time"
"unicode"
"github.com/kami/maven/internal/morph"
)
type OpenMeteoProvider struct {
@@ -101,10 +104,20 @@ func (p *OpenMeteoProvider) CurrentWeather(ctx context.Context, location string)
// sentence, in order. He says "какая погода в Казани", so the word arrives in
// the prepositional case and the geocoder wants the nominative (Vikunja #421).
//
// Two cheap reversals cover most of what he says: a final "е" is usually a
// nominative "а" (Москве → Москва) or nothing at all (Лондоне → Лондон), and a
// final "и" is usually a soft sign (Казани → Казань). Indeclinable names —
// Тбилиси, Сочи, Осло — are already nominative and the first candidate answers.
// The dictionary answers first (Vikunja #530). internal/morph lemmatises
// "Уфе" to "Уфа" and "Москве" to "Москва", which is the same question this
// used to guess at by reversing endings, asked of something that knows.
//
// The reversals stay behind it, because the dictionary does not know every
// place: "Твери" and "Перми" come back unchanged, and a final "и" is usually a
// soft sign. A final "е" is usually a nominative "а" (Москве → Москва) or
// nothing at all (Лондоне → Лондон). Indeclinable names — Тбилиси, Сочи, Осло —
// are already nominative and the first candidate answers, which is why the word
// as spoken is always tried before anything derived from it.
//
// There used to be a four-rune floor here, so "Уфе" was asked as spoken and
// "Уфа" was never tried. The floor was there to stop a two-letter stem, and the
// stem length is what it now tests.
//
// Nothing here is a guess about the weather: a wrong candidate finds no city
// and the caller says so. It only decides which strings are worth asking about.
@@ -121,8 +134,9 @@ func locationCandidates(location string) []string {
}
out = append(out, s)
}
add(titleFirst(morph.Lemma(location)))
r := []rune(location)
if len(r) < 4 {
if len(r) < 3 {
return out
}
stem := string(r[:len(r)-1])
@@ -139,6 +153,17 @@ func locationCandidates(location string) []string {
return out
}
// titleFirst restores the leading capital a place name carries. morph.Lemma
// answers lowercased, because a lemma is a dictionary entry and the dictionary
// has no opinion about proper nouns.
func titleFirst(s string) string {
r := []rune(s)
if len(r) == 0 {
return s
}
return string(unicode.ToUpper(r[0])) + string(r[1:])
}
func (p *OpenMeteoProvider) geocode(ctx context.Context, location string) (lat, lon float64, name string, err error) {
for _, cand := range locationCandidates(location) {
lat, lon, name, err = p.geocodeOne(ctx, cand)
+8 -2
View File
@@ -86,14 +86,20 @@ func TestStubProvider(t *testing.T) {
// TestLocationCandidates — he speaks the prepositional case and the geocoder
// wants the nominative (Vikunja #421).
//
// The dictionary answers before the reversals now, so the nominative it knows
// comes second and anything derived by hand follows (V-530). "Уфе" used to fall
// under a four-rune floor and was asked as spoken, so "Уфа" was never tried.
func TestLocationCandidates(t *testing.T) {
cases := map[string][]string{
"Москве": {"Москве", "Москва", "Москв"},
"Казани": {"Казани", "Казань", "Казан"},
"Лондоне": {"Лондоне", "Лондона", "Лондон"},
"Лондоне": {"Лондоне", "Лондон", "Лондона"},
"Тбилиси": {"Тбилиси", "Тбились", "Тбилис"},
"Berlin": {"Berlin"},
"Уфе": {"Уфе"}, // too short to strip — asked as spoken
"Уфе": {"Уфе", "Уфа", "Уф"},
// The dictionary does not know it, so the soft-sign reversal answers.
"Твери": {"Твери", "Тверь", "Твер"},
}
for in, want := range cases {
got := locationCandidates(in)