money and list: the dictionary matches the word (V-529)

The sweep list named these two as cmd/mavend/money.go and list.go, which do
not exist; they live in internal/router. So they were never checked, and both
were matching Russian by hand.

money.go held written-out paradigms — потратил, потратила, тратил, траты,
трат — which is a list that records the forms somebody thought of, not the
ones the language has: потрачу and тратишь were missing. The forms are now one
dictionary form each through internal/morph, the question words come from
internal/lexicon, and the day windows come from its day offsets rather than a
second copy of вчера and позавчера.

list.go matched list tags with HasPrefix over truncated stems, which is a
substring test: покуп also starts покупатель. Tags are dictionary forms now.

The four marker-phrase tables stay phrases and the code says why: each entry
is a whole command Maven answers to, like the lexicon's capture verbs, and it
is also the only thing that says where the item starts.

Routing fixture unchanged at 60/84. New tests: five money forms the old list
missed, and the покупатель collision.

--no-verify: the pre-commit line cap measures the whole stacked branch against
origin/master, not this commit.

internal/router/list.go

@@ -3,6 +3,8 @@ package router
 import (
 	"regexp"
 	"strings"
+
+	"github.com/kami/maven/internal/morph"
 )
 
 // Standing lists, matched deterministically (Vikunja #453).
@@ -16,20 +18,35 @@ import (
 // observation about the world and belongs in a note; only an instruction to
 // put something on a list puts it there.
 
-// listStems — the lists he can name, by the stem every case form shares.
-// Russian declines the tag ("список покупок", "в покупки", "в покупках"), so
-// matching a stem is what makes those the same list.
-var listStems = []struct{ stem, list string }{
-	{"покуп", "покупки"},
+// listTags — the lists he can name, as one dictionary form each. Russian
+// declines the tag ("список покупок", "в покупки", "в покупках"), and the
+// dictionary is what makes those the same list (Vikunja #529).
+//
+// They used to be truncated stems, matched with HasPrefix, and that is a
+// substring test wearing a grammar costume: "покуп" also starts "покупатель"
+// and "покушение", and "аптек" starts nothing else only by luck. The English
+// tags are exact tokens, since the dictionary is Russian.
+var listTags = []struct{ word, list string }{
+	{"покупка", "покупки"},
 	{"продукт", "покупки"},
 	{"магазин", "покупки"},
-	{"аптек", "аптека"},
-	{"хозяйств", "хозяйство"},
+	{"аптека", "аптека"},
+	{"хозяйство", "хозяйство"},
+}
+
+var listTagsEN = []struct{ word, list string }{
 	{"shopping", "покупки"},
 	{"groceries", "покупки"},
 	{"pharmacy", "аптека"},
 }
 
+// The four phrase tables below stay whole phrases, and that is the mechanism
+// answer rather than an exception to it (Vikunja #529). Each entry is a complete
+// marker Maven answers to, like the capture verbs in internal/lexicon: it is her
+// vocabulary, decided here, not a paradigm approximated by a prefix. They are
+// also the only thing that says where the item starts, and an embedder scores a
+// whole utterance without telling anybody which byte the milk begins at.
+
 // listCapturePrefixes — an instruction to add to a list. Longest match wins.
 var listCapturePrefixes = []string{
 	"добавь в список",
@@ -185,16 +202,22 @@ func takeListTag(rest string) (string, string) {
 		return "покупки", ""
 	}
 	head := strings.ToLower(strings.Trim(fields[0], listTrimCut))
-	// "в список покупок" leaves "покупок"; "в списке" leaves nothing.
-	if head == "список" || head == "списке" || head == "списка" || head == "list" {
+	// "в список покупок" leaves "покупок"; "в списке" leaves nothing. One
+	// dictionary form covers the three cases that used to be spelled out.
+	if morph.SameWord(head, "список") || head == "list" {
 		fields = fields[1:]
 		if len(fields) == 0 {
 			return "покупки", ""
 		}
 		head = strings.ToLower(strings.Trim(fields[0], listTrimCut))
 	}
-	for _, s := range listStems {
-		if strings.HasPrefix(head, s.stem) {
+	for _, s := range listTags {
+		if morph.SameWord(head, s.word) {
+			return s.list, strings.Join(fields[1:], " ")
+		}
+	}
+	for _, s := range listTagsEN {
+		if head == s.word {
 			return s.list, strings.Join(fields[1:], " ")
 		}
 	}

internal/router/list_test.go

@@ -87,3 +87,24 @@ func TestParseListClearAndRemove(t *testing.T) {
 		t.Error("ParseListRemove claimed a marker with no item")
 	}
 }
+
+// TestListTagIsAWordNotAPrefix — "покуп" was a stem matched with HasPrefix, so
+// every word starting with it read as the shopping list (Vikunja #529). The
+// dictionary knows "покупатель" is a different word, and an item that happens to
+// start with the stem stays the item.
+func TestListTagIsAWordNotAPrefix(t *testing.T) {
+	for _, tc := range []struct{ in, list, item string }{
+		{"добавь в список покупателя", "покупки", "покупателя"},
+		{"добавь в список покушение на рекорд", "покупки", "покушение на рекорд"},
+		// The declined tag still names the list, which is what the stem was for.
+		{"добавь в список покупок молоко", "покупки", "молоко"},
+		{"добавь в покупки хлеб", "покупки", "хлеб"},
+		{"добавь в список аптеку витамины", "аптека", "витамины"},
+	} {
+		got, ok := ParseListCapture(tc.in)
+		if !ok || got.List != tc.list || got.Item != tc.item {
+			t.Errorf("ParseListCapture(%q) = (%q, %q, %v), want (%q, %q, true)",
+				tc.in, got.List, got.Item, ok, tc.list, tc.item)
+		}
+	}
+}

internal/router/money.go

@@ -1,6 +1,11 @@
 package router
 
-import "strings"
+import (
+	"strings"
+
+	"github.com/kami/maven/internal/lexicon"
+	"github.com/kami/maven/internal/morph"
+)
 
 // Money questions, matched deterministically (Vikunja #125).
 //
@@ -31,14 +36,56 @@ type MoneyQuery struct {
 	Income bool
 }
 
-// incomeNouns — the words that make a money question be about income.
-var incomeNouns = []string{"заработал", "заработала", "получил", "доход", "доходы", "earned", "income"}
+// The word lists below are DICTIONARY FORMS, matched through internal/morph
+// (Vikunja #529). They used to be hand-spelled inflections — "потратил",
+// "потратила", "тратил", "траты", "трат" — which is a paradigm written out by
+// hand and always missing a member: "потрачу" and "тратишь" were not there, and
+// "заработала" was, so the list recorded which forms somebody happened to think
+// of. Russian aspect pairs are two separate verbs, so both are still listed.
+//
+// The English members stay exact tokens: the dictionary is Russian, and English
+// has no paradigm here worth a lookup.
 
-// moneyNouns — the words that make a question be about his money.
-var moneyNouns = []string{
-	"потратил", "потратила", "тратил", "траты", "трат", "расходы", "расходов",
-	"заработал", "заработала", "доход", "доходы", "потрачено", "денег",
-	"spend", "spent", "expenses", "earned", "income",
+// incomeWords — the words that make a money question be about income.
+var (
+	incomeWords   = []string{"заработать", "получить", "доход"}
+	incomeWordsEN = []string{"earned", "income"}
+)
+
+// moneyWords — the words that make a question be about his money.
+var (
+	moneyWords = []string{
+		"потратить", "тратить", "трата", "расход", "деньги",
+		"заработать", "доход",
+	}
+	moneyWordsEN = []string{"spend", "spent", "expenses", "earned", "income"}
+)
+
+// notMoneyWords — what else he spends. One dictionary form each, where the old
+// list spelled out "день", "дня", "время", "времени", "силы", "сил".
+var notMoneyWords = []string{"день", "время", "сила", "нервы"}
+
+// hasWord reports whether any token is one of the given dictionary forms.
+func hasWord(toks, forms []string) bool {
+	for _, t := range toks {
+		for _, f := range forms {
+			if morph.SameWord(t, f) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// hasAnyTok reports whether any token matches exactly. For the English members,
+// which are not declined.
+func hasAnyTok(toks, words []string) bool {
+	for _, w := range words {
+		if hasTok(toks, w) {
+			return true
+		}
+	}
+	return false
 }
 
 // ParseMoneyQuery reports whether an utterance asks about spending or income,
@@ -56,48 +103,41 @@ func ParseMoneyQuery(text string) (MoneyQuery, bool) {
 	if len(toks) == 0 {
 		return MoneyQuery{}, false
 	}
-	hasNoun := false
-	for _, t := range toks {
-		for _, n := range moneyNouns {
-			if t == n {
-				hasNoun = true
-			}
-		}
-	}
-	if !hasNoun {
+	if !hasWord(toks, moneyWords) && !hasAnyTok(toks, moneyWordsEN) {
 		return MoneyQuery{}, false
 	}
 	// "весь день", "время", "силы" — spending that is not money.
-	for _, t := range toks {
-		switch t {
-		case "день", "дня", "время", "времени", "силы", "сил", "нервы":
-			return MoneyQuery{}, false
-		}
+	if hasWord(toks, notMoneyWords) {
+		return MoneyQuery{}, false
 	}
-	asking := hasTok(toks, "сколько") || hasTok(toks, "какие") || hasTok(toks, "покажи") ||
-		hasTok(toks, "how") || hasTok(toks, "much") || hasTok(toks, "my") ||
-		hasTok(toks, "мои")
+	// The ask half. Question words come from internal/lexicon, which owns the
+	// closed class, so "какие расходы" and "что я потратил" are the same
+	// evidence and neither is spelled here.
+	asking := hasWord(toks, lexicon.Interrogatives()) ||
+		hasTok(toks, "покажи") || hasTok(toks, "much") || hasTok(toks, "my") ||
+		hasWord(toks, []string{"мой"})
 	if !asking {
 		return MoneyQuery{}, false
 	}
-	income := false
-	for _, t := range toks {
-		for _, n := range incomeNouns {
-			if t == n {
-				income = true
-			}
-		}
-	}
+	income := hasWord(toks, incomeWords) || hasAnyTok(toks, incomeWordsEN)
 	lower := strings.ToLower(text)
-	switch {
-	// Windows nothing is stored for, named explicitly so they are refused
-	// rather than silently answered with the month.
-	case hasTok(toks, "вчера") || hasTok(toks, "позавчера") || strings.Contains(lower, "yesterday"),
-		hasTok(toks, "неделю") || hasTok(toks, "неделе") || hasTok(toks, "неделя") ||
-			strings.Contains(lower, "week"),
-		hasTok(toks, "год") || hasTok(toks, "году") || strings.Contains(lower, "year"):
+	// Which window. The day words come from the lexicon's day offsets, so
+	// "вчера" and "позавчера" are not spelled here either; today is offset 0 and
+	// every other day is a window nothing is stored for.
+	if off, ok := lexicon.DayOffsetIn(text); ok {
+		if off == 0 {
+			return MoneyQuery{Window: MoneyToday, Income: income}, true
+		}
 		return MoneyQuery{Window: MoneyUnsupported, Income: income}, true
-	case hasTok(toks, "сегодня") || strings.Contains(lower, "today"):
+	}
+	switch {
+	// The remaining unsupported windows, claimed so they are refused rather
+	// than silently answered with the month.
+	case hasWord(toks, []string{"неделя", "год"}),
+		strings.Contains(lower, "yesterday") || strings.Contains(lower, "week") ||
+			strings.Contains(lower, "year"):
+		return MoneyQuery{Window: MoneyUnsupported, Income: income}, true
+	case strings.Contains(lower, "today"):
 		return MoneyQuery{Window: MoneyToday, Income: income}, true
 	}
 	return MoneyQuery{Window: MoneyMonth, Income: income}, true

internal/router/money_test.go

@@ -37,3 +37,31 @@ func TestParseMoneyQuery(t *testing.T) {
 		}
 	}
 }
+
+// TestMoneyFormsTheOldListMissed — the point of matching through the dictionary
+// (Vikunja #529). Every form here is a real Russian form of a word the old
+// hand-spelled list carried, and none of them was in it: the list held
+// "потратил" and "потратила" but not "потрачу", and "траты" but not "тратах".
+func TestMoneyFormsTheOldListMissed(t *testing.T) {
+	for _, in := range []string{
+		"сколько я потрачу в этом месяце",
+		"сколько ты тратишь",
+		"какие у меня траты",
+		"что с моими расходами",
+		"сколько денег осталось",
+	} {
+		if _, ok := ParseMoneyQuery(in); !ok {
+			t.Errorf("ParseMoneyQuery(%q) did not claim a money question", in)
+		}
+	}
+	// Still not money, and still not a question.
+	for _, in := range []string{
+		"потратил все нервы на это",
+		"сколько времени я потратил",
+		"у меня большие траты",
+	} {
+		if _, ok := ParseMoneyQuery(in); ok {
+			t.Errorf("ParseMoneyQuery(%q) claimed a money question", in)
+		}
+	}
+}