Merge branch 'fix/g05' into fix/integrated

internal/router/money.go

@@ -16,12 +16,29 @@ const (
 	MoneyNone MoneyWindow = iota
 	MoneyToday
 	MoneyMonth
+	// MoneyUnsupported — a money question over a window nothing is stored for
+	// ("вчера", "на прошлой неделе"). Claimed, not answered: the poller keeps
+	// today and the month, and answering a question about yesterday with the
+	// month-to-date total is worse than saying she does not keep it.
+	MoneyUnsupported
 )
 
+// MoneyQuery — a parsed money question. Income is set when he asked what he
+// EARNED rather than what he spent; the two read the same fact and differ only
+// in which half of it leads the answer.
+type MoneyQuery struct {
+	Window MoneyWindow
+	Income bool
+}
+
+// incomeNouns — the words that make a money question be about income.
+var incomeNouns = []string{"заработал", "заработала", "получил", "доход", "доходы", "earned", "income"}
+
 // moneyNouns — the words that make a question be about his money.
 var moneyNouns = []string{
 	"потратил", "потратила", "тратил", "траты", "трат", "расходы", "расходов",
-	"заработал", "потрачено", "денег", "spend", "spent", "expenses",
+	"заработал", "заработала", "доход", "доходы", "потрачено", "денег",
+	"spend", "spent", "expenses", "earned", "income",
 }
 
 // ParseMoneyQuery reports whether an utterance asks about spending or income,
@@ -30,11 +47,14 @@ var moneyNouns = []string{
 //
 // Narrow on purpose. A money noun alone is not enough — "я потратил весь день
 // на это" is him talking about his day, so an amount word or an explicit
-// question word has to be there too.
-func ParseMoneyQuery(text string) (MoneyWindow, bool) {
+// question word has to be there too. The two evidence halves are INDEPENDENT:
+// "траты" and "расходы" used to sit in both lists, so either word alone
+// satisfied the whole gate and "у меня в этом месяце большие траты", a
+// statement, came back with a figure.
+func ParseMoneyQuery(text string) (MoneyQuery, bool) {
 	toks := planTokens(text)
 	if len(toks) == 0 {
-		return MoneyNone, false
+		return MoneyQuery{}, false
 	}
 	hasNoun := false
 	for _, t := range toks {
@@ -45,27 +65,40 @@ func ParseMoneyQuery(text string) (MoneyWindow, bool) {
 		}
 	}
 	if !hasNoun {
-		return MoneyNone, false
+		return MoneyQuery{}, false
 	}
 	// "весь день", "время", "силы" — spending that is not money.
 	for _, t := range toks {
 		switch t {
 		case "день", "дня", "время", "времени", "силы", "сил", "нервы":
-			return MoneyNone, false
+			return MoneyQuery{}, false
 		}
 	}
 	asking := hasTok(toks, "сколько") || hasTok(toks, "какие") || hasTok(toks, "покажи") ||
 		hasTok(toks, "how") || hasTok(toks, "much") || hasTok(toks, "my") ||
-		hasTok(toks, "мои") || hasTok(toks, "траты") || hasTok(toks, "расходы")
+		hasTok(toks, "мои")
 	if !asking {
-		return MoneyNone, false
+		return MoneyQuery{}, false
+	}
+	income := false
+	for _, t := range toks {
+		for _, n := range incomeNouns {
+			if t == n {
+				income = true
+			}
+		}
 	}
 	lower := strings.ToLower(text)
 	switch {
+	// Windows nothing is stored for, named explicitly so they are refused
+	// rather than silently answered with the month.
+	case hasTok(toks, "вчера") || hasTok(toks, "позавчера") || strings.Contains(lower, "yesterday"),
+		hasTok(toks, "неделю") || hasTok(toks, "неделе") || hasTok(toks, "неделя") ||
+			strings.Contains(lower, "week"),
+		hasTok(toks, "год") || hasTok(toks, "году") || strings.Contains(lower, "year"):
+		return MoneyQuery{Window: MoneyUnsupported, Income: income}, true
 	case hasTok(toks, "сегодня") || strings.Contains(lower, "today"):
-		return MoneyToday, true
-	case hasTok(toks, "месяц") || hasTok(toks, "месяце") || strings.Contains(lower, "month"):
-		return MoneyMonth, true
+		return MoneyQuery{Window: MoneyToday, Income: income}, true
 	}
-	return MoneyMonth, true
+	return MoneyQuery{Window: MoneyMonth, Income: income}, true
 }

internal/router/money_test.go

@@ -15,17 +15,25 @@ func TestParseMoneyQuery(t *testing.T) {
 		{"какие у меня расходы за месяц", MoneyMonth, true},
 		{"how much did I spend today", MoneyToday, true},
 		{"сколько я заработал в этом месяце", MoneyMonth, true},
+		// Windows nothing is stored for are claimed and refused, never answered
+		// with the month-to-date figure.
+		{"сколько я потратил вчера?", MoneyUnsupported, true},
+		{"сколько я потратил на прошлой неделе?", MoneyUnsupported, true},
+		{"how much did I spend yesterday", MoneyUnsupported, true},
 		// Not about money.
 		{"я потратил весь день на это", MoneyNone, false},
+		// A statement, not a question: the noun and the ask must be independent
+		// evidence, and "траты" used to satisfy both halves on its own.
+		{"у меня в этом месяце большие траты", MoneyNone, false},
 		{"потратил много сил", MoneyNone, false},
 		{"какая погода?", MoneyNone, false},
 		{"я купил молоко", MoneyNone, false},
 		{"", MoneyNone, false},
 	}
 	for _, c := range cases {
-		w, ok := ParseMoneyQuery(c.in)
-		if ok != c.ok || w != c.window {
-			t.Errorf("ParseMoneyQuery(%q) = (%v, %v), want (%v, %v)", c.in, w, ok, c.window, c.ok)
+		q, ok := ParseMoneyQuery(c.in)
+		if ok != c.ok || q.Window != c.window {
+			t.Errorf("ParseMoneyQuery(%q) = (%v, %v), want (%v, %v)", c.in, q.Window, ok, c.window, c.ok)
 		}
 	}
 }

internal/router/task.go

@@ -47,7 +47,10 @@ func ParseTaskCapture(text string) (TaskCapture, bool) {
 	rest := strings.TrimSpace(trimmed[len(best):])
 	rest = strings.TrimLeft(rest, ":—- ")
 	rest = strings.TrimSpace(rest)
-	rest = strings.TrimRight(rest, ".!")
+	// The question mark goes too. Whisper punctuates dictated Russian, and
+	// "добавь в задачи позвонить в банк?" must not store the mark or carry it
+	// into the dedupe key.
+	rest = strings.TrimRight(rest, ".!?")
 	rest, weight := stripUrgency(rest)
 	if rest == "" {
 		return TaskCapture{}, false
@@ -55,30 +58,86 @@ func ParseTaskCapture(text string) (TaskCapture, bool) {
 	return TaskCapture{Text: rest, Weight: weight}, true
 }
 
+// urgencyIntensifiers — words that may sit between the edge and the marker.
+// "очень срочно оплатить интернет" is the marker at the edge with one word in
+// front of it, and it means exactly what "срочно оплатить интернет" means.
+var urgencyIntensifiers = []string{"очень", "прям", "прямо", "really", "very", "super"}
+
+// urgencyEdgeTrim — punctuation to ignore around an edge token and to clean off
+// the remainder afterwards.
+const urgencyEdgeTrim = " .,;:!?—-"
+
 // stripUrgency pulls a leading or trailing urgency word out of the task text
 // and returns the weight it implies. Only at the edges: "срочно оплатить
 // интернет" and "оплатить интернет срочно" are the same instruction, while
 // "позвонить в срочную помощь" is a task whose text happens to contain the
 // stem, and cutting a word out of the middle of it would mangle the task.
 //
+// Matched as a TOKEN, not as a fixed prefix or suffix string. The old shape
+// required exactly one space before a trailing marker, so "оплатить интернет,
+// срочно" — which is what whisper produces from dictated Russian — kept weight
+// 0 and stored the comma and the word as part of the task, polluting the dedupe
+// key with the very flag he was trying to set.
+//
 // The word is removed from the text, because the list should read "оплатить
 // интернет (важно)" and not "важно оплатить интернет (важно)".
 func stripUrgency(text string) (string, int) {
+	fields := strings.Fields(text)
+	if len(fields) == 0 {
+		return text, 0
+	}
 	for _, m := range urgencyMarkers {
-		lower := strings.ToLower(text)
-		switch {
-		case strings.HasPrefix(lower, m.Word+" "):
-			return strings.TrimSpace(text[len(m.Word):]), m.Weight
-		case strings.HasSuffix(lower, " "+m.Word):
-			return strings.TrimSpace(text[:len(text)-len(m.Word)]), m.Weight
-		case lower == m.Word:
-			// Nothing but the marker — no task in it.
-			return "", 0
+		// Strongest marker first (task_phrases.go sorts them), leading edge
+		// before trailing, so a text carrying both keeps the stronger one.
+		if lo, hi, ok := urgencySpan(fields, m.Word); ok {
+			rest := strings.Join(append(append([]string{}, fields[:lo]...), fields[hi+1:]...), " ")
+			rest = strings.Trim(rest, urgencyEdgeTrim)
+			if rest == "" {
+				// Nothing but the marker — no task in it.
+				return "", 0
+			}
+			return rest, m.Weight
 		}
 	}
 	return text, 0
 }
 
+// urgencySpan finds the marker at either edge, allowing intensifiers between
+// the edge and the marker, and returns the inclusive token range to cut.
+func urgencySpan(fields []string, word string) (lo, hi int, ok bool) {
+	for i := 0; i < len(fields); i++ {
+		if isUrgencyToken(fields[i], word) {
+			return 0, i, true
+		}
+		if !isIntensifier(fields[i]) {
+			break
+		}
+	}
+	for i := len(fields) - 1; i >= 0; i-- {
+		if isUrgencyToken(fields[i], word) {
+			return i, len(fields) - 1, true
+		}
+		if !isIntensifier(fields[i]) {
+			break
+		}
+	}
+	return 0, 0, false
+}
+
+func isUrgencyToken(tok, word string) bool {
+	return strings.Trim(strings.ToLower(tok), urgencyEdgeTrim) == word
+}
+
+func isIntensifier(tok string) bool {
+	t := strings.Trim(strings.ToLower(tok), urgencyEdgeTrim)
+	for _, w := range urgencyIntensifiers {
+		if t == w {
+			return true
+		}
+	}
+	return false
+}
+
 // IsTaskListQuery reports whether an utterance asks for the outstanding task
 // list — "какие у меня задачи?", "что мне нужно сделать?", "список дел".
 //
@@ -94,13 +153,23 @@ func IsTaskListQuery(text string) bool {
 	if hasTok(toks, "как") && (hasTok(toks, "дела") || hasTok(toks, "делишки")) {
 		return false
 	}
-	// "что мне нужно сделать" / "что мне делать" — no task noun at all.
-	if (hasTok(toks, "что") || hasTok(toks, "чем")) &&
-		(hasTok(toks, "сделать") || hasTok(toks, "заняться")) {
-		return true
-	}
-	if hasTok(toks, "what") && hasTok(toks, "do") {
-		return true
+	// "что мне нужно сделать" / "чем мне заняться" — no task noun at all, so
+	// the pronoun is what carries the meaning. Without it these rules claimed
+	// every question with a verb in them: "что нужно сделать чтобы перезапустить
+	// сервер?" and "what does docker do?" both answered "задач нет." from ahead
+	// of recall and the model, which is the failure the source ordering exists
+	// to avoid, pointed the other way.
+	//
+	// A "с"/"со" object excludes them too: "что мне сделать с этим файлом" has
+	// the pronoun and is still a question about a file.
+	if !hasTok(toks, "с") && !hasTok(toks, "со") {
+		if hasTok(toks, "мне") && (hasTok(toks, "что") || hasTok(toks, "чем")) &&
+			(hasTok(toks, "сделать") || hasTok(toks, "делать") || hasTok(toks, "заняться")) {
+			return true
+		}
+		if hasTok(toks, "what") && hasTok(toks, "do") && hasTok(toks, "i") && !hasTok(toks, "you") {
+			return true
+		}
 	}
 	hasNoun := false
 	for _, t := range toks {

internal/router/task_phrases.json

@@ -21,6 +21,7 @@
   ],
   "capture_prefixes": [
     "добавь в задачи",
+    "добавь в тудушки",
     "добавь в список задач",
     "добавь в список дел",
     "добавь в список",
@@ -28,6 +29,7 @@
     "запиши в задачи",
     "запиши задачу",
     "новая задача",
+    "поставь задачу",
     "в задачи",
     "add a task",
     "add task",

internal/router/task_test.go

@@ -20,6 +20,17 @@ func TestParseTaskCapture(t *testing.T) {
 		{"новая задача важно позвонить маме", "позвонить маме", 2, true},
 		// The stem inside the task text is part of the task, not a marker.
 		{"добавь в задачи позвонить в срочную помощь", "позвонить в срочную помощь", 0, true},
+		// Whisper punctuates dictated Russian. The marker used to be missed as
+		// soon as anything sat next to it, and then it stayed in the task text
+		// and in the dedupe key — the exact task he was trying to flag.
+		{"добавь в задачи оплатить интернет, срочно", "оплатить интернет", 3, true},
+		{"добавь в задачи очень срочно оплатить интернет", "оплатить интернет", 3, true},
+		{"добавь в задачи оплатить интернет — важно", "оплатить интернет", 2, true},
+		// A dictated question mark is not part of the task.
+		{"добавь в задачи позвонить в банк?", "позвонить в банк", 0, true},
+		// The phrasings he uses that the prefix list did not have.
+		{"поставь задачу вынести мусор", "вынести мусор", 0, true},
+		{"добавь в тудушки купить лампочки", "купить лампочки", 0, true},
 		// A marker with nothing after it files nothing.
 		{"добавь в задачи", "", 0, false},
 		{"новая задача", "", 0, false},
@@ -47,6 +58,7 @@ func TestIsTaskListQuery(t *testing.T) {
 		"задачи",
 		"мои задачи",
 		"what should I do",
+		"что мне делать?",
 	}
 	for _, s := range yes {
 		if !IsTaskListQuery(s) {
@@ -55,6 +67,13 @@ func TestIsTaskListQuery(t *testing.T) {
 	}
 	no := []string{
 		"как дела?",
+		// No task noun and no pronoun: these fired ahead of recall and the
+		// model, and answered a question about a file or a server with
+		// "задач нет."
+		"что нужно сделать чтобы перезапустить сервер?",
+		"что мне сделать с этим файлом?",
+		"what does docker do?",
+		"what do you do?",
 		"какая погода?",
 		"напомни мне позвонить маме в шесть",
 		"я сделал зарядку",