Maven/internal/router/factparser_corpus_test.go

package router

import (
	"strconv"
	"strings"
	"testing"
)

// The fact-parser corpus (V-586). DefaultFactParser is the only thing standing
// between a spoken sentence and a written self-care fact, and until this file
// it was covered by a handful of examples chosen by whoever last edited it. The
// RU routing fixture does not cover it either: it holds three fact cases and
// all three miss on intent, so the parser is never reached and a parser change
// scores as "unchanged".
//
// So the corpus is here, and it scores BOTH implementations: the closed-class
// parser that ships, and legacyFactParse below, a faithful copy of the
// hand-written substring version 22edc3c replaced. The table is the measurement
// — see docs/evals/2026-08-06-fact-parser.md for the numbers as of that day.
//
// Three case classes, and the third is the point:
//
//	true positive  — a sentence he would say, with the key it must write.
//	misfire        — a sentence the substring parser wrote a fact for and
//	                 should not have. want is "".
//	false-negative — a sentence he would plausibly say whose word is in NO
//	                 lexicon set. want is the key a human would assign, and
//	                 the ship parser is EXPECTED to miss it. These measure the
//	                 cost of the word-list design, not a bug in it.
//
// Nothing here asserts a score. A closed class is a decision about vocabulary
// and the bar belongs in a dated eval, not in an assertion that turns every
// vocabulary edit into a red test. What it does fail on is a regression in the
// two classes that are not judgement calls: a true positive that stops matching
// and a misfire that starts.

type factCase struct {
	utterance string
	want      string // "" — no fact
	class     string // "tp", "misfire", "fn"
	broken    bool   // ship parser is known to get this wrong; see the 06-08 eval
	note      string
}

var factCorpus = []factCase{
	// ---- water, true positives -------------------------------------------
	{"выпил стакан воды", "water", "tp", false, ""},
	{"попил воды", "water", "tp", false, ""},
	{"я попил водички", "water", "tp", false, ""},
	{"пью воду", "water", "tp", false, ""},
	{"воду пил уже", "water", "tp", false, ""},
	{"допил воду", "water", "tp", true, "REGRESSION: the dictionary lemmatises допил to допилить, to finish sawing — the same saw collision drink_verbs already carries пил for, unfixed for the prefixed form"},
	{"запил таблетку водой", "water", "tp", false, ""},
	{"drank water", "water", "tp", false, ""},
	{"i drank some water", "water", "tp", false, ""},

	// ---- meal, true positives --------------------------------------------
	{"поужинал", "meal", "tp", false, ""},
	{"я пообедал", "meal", "tp", false, ""},
	{"позавтракал кашей", "meal", "tp", false, ""},
	{"перекусил бутербродом", "meal", "tp", false, ""},
	{"покушал", "meal", "tp", false, ""},
	{"поел супа", "meal", "tp", false, ""},
	{"обед был в час", "meal", "tp", false, ""},
	{"ужинать буду позже", "meal", "tp", false, ""},
	{"i ate", "meal", "tp", false, ""},
	{"had lunch", "meal", "tp", false, ""},
	{"dinner done", "meal", "tp", false, ""},

	// ---- shower, true positives ------------------------------------------
	{"принял душ", "shower", "tp", false, ""},
	{"сходил в душ", "shower", "tp", false, ""},
	{"душ принят", "shower", "tp", false, ""},
	{"ополоснулся душем", "shower", "tp", false, ""},
	{"took a shower", "shower", "tp", false, ""},
	{"i showered", "shower", "tp", false, ""},

	// ---- break, true positives -------------------------------------------
	{"сделал перерыв", "break", "tp", false, ""},
	{"отдохнул полчаса", "break", "tp", false, ""},
	{"передохнул немного", "break", "tp", false, ""},
	{"отдыхаю", "break", "tp", false, ""},
	{"был перерыв на обед", "meal", "tp", false, "meal wins the switch; both keys are true of the sentence"},
	{"took a break", "break", "tp", false, ""},

	// ---- sleep, true positives -------------------------------------------
	{"спал восемь часов", "sleep", "tp", false, ""},
	{"спала плохо", "sleep", "tp", false, "she may report her own; the parser is speaker-agnostic"},
	{"поспал днём", "sleep", "tp", false, ""},
	{"выспался наконец", "sleep", "tp", false, ""},
	{"проспал будильник", "sleep", "tp", false, ""},
	{"пойду спать", "sleep", "tp", false, ""},
	{"slept 8 hours", "sleep", "tp", false, ""},
	{"i slept badly", "sleep", "tp", false, ""},

	// ---- misfires 22edc3c set out to reject -------------------------------
	{"пилот сказал что вылет через час", "", "misfire", false, "substring пил"},
	{"водитель уже подъехал", "", "misfire", false, "substring вод"},
	{"надо заводить машину", "", "misfire", false, "substring вод"},
	{"душа болит", "", "misfire", false, "substring душ"},
	{"в комнате душно", "", "misfire", false, "substring душ"},
	{"это была беда", "", "misfire", false, "substring еда"},
	{"наша победа", "", "misfire", false, "substring еда"},
	{"пила лежит в гараже", "", "misfire", false, "substring пил"},
	{"водитель пилота ждёт", "", "misfire", false, "both stems in one sentence — the old water arm fires"},
	{"обеденный перерыв отменили", "", "misfire", true, "neither parser gets this: the old one writes meal off the adjective, the new one writes break off перерыв. A cancelled break is not a break taken."},

	// ---- hard negatives that decide the design ----------------------------
	{"есть новости по бэкапу базы", "", "misfire", false, "есть is the existential, not a meal"},
	{"напоминания на завтра есть", "", "misfire", false, "завтра carries завтрак as a substring"},
	{"на душе легко", "", "misfire", false, "душе is the soul, and also the prepositional of душ"},
	{"пилил доску весь вечер", "", "misfire", false, ""},
	{"поставь будильник на завтра", "", "misfire", false, "завтра again, no meal"},

	// ---- false negatives: words in no lexicon set -------------------------
	// Water.
	{"выпил чаю", "water", "fn", false, "hydration by any liquid; чай is in no set"},
	{"глотнул воды", "water", "fn", false, "глотнуть is not a drink verb"},
	{"хлебнул воды", "water", "fn", false, "хлебнуть is not a drink verb"},
	{"воды хлебнул из бутылки", "water", "fn", false, ""},
	{"выпил стакан", "water", "fn", false, "the noun is elided; he says this"},
	{"i hydrated", "water", "fn", false, "hydrate is in no set"},
	{"finished my bottle of water", "water", "fn", false, "no drink verb in the English set"},

	// Meal.
	{"ем суп", "meal", "fn", false, "есть is deliberately absent, and this is the cost"},
	{"съел бутерброд", "meal", "fn", false, "съесть is in no set"},
	{"наелся", "meal", "fn", false, "наесться is in no set"},
	{"пожрал", "meal", "fn", false, "coarse but spoken"},
	{"полдник был", "meal", "fn", false, "полдник is in no set"},
	{"i had a snack", "meal", "fn", false, "snack is in no set"},
	{"having supper", "meal", "fn", false, "supper is in no set"},
	{"brunch was good", "meal", "fn", false, "brunch is in no set"},
	{"i eat now", "meal", "fn", false, "eat is in no set — only ate is"},

	// Shower.
	{"был в душе", "shower", "fn", false, "prepositional; anyExact cannot take it without taking the soul"},
	{"после душа полегчало", "shower", "fn", false, "genitive, same collision"},
	{"помылся", "shower", "fn", false, "помыться is in no set"},
	{"сходил в ванную", "shower", "fn", false, "ванная is in no set"},
	{"искупался", "shower", "fn", false, "искупаться is in no set"},
	{"i am showering", "shower", "fn", false, "showering is not a member and the set is exact-matched"},

	// Break.
	{"сделал передышку", "break", "fn", false, "передышка is in no set"},
	{"перекур", "break", "fn", false, "перекур is in no set"},
	{"полежал немного", "break", "fn", false, "полежать is in no set"},
	{"сделал паузу", "break", "fn", false, "пауза is in no set"},
	{"i took five", "break", "fn", false, "idiomatic, in no set"},
	{"resting now", "break", "fn", false, "resting is not a member and rest is matched by lemma only"},

	// Sleep.
	{"вздремнул", "sleep", "fn", false, "вздремнуть is in no set"},
	{"прикорнул на диване", "sleep", "fn", false, "прикорнуть is in no set"},
	{"дрых до обеда", "sleep", "fn", false, "дрыхнуть is in no set — and обед makes this a MEAL for both parsers"},
	{"недоспал", "sleep", "fn", false, "недоспать is in no set"},
	{"лёг в двенадцать", "sleep", "fn", false, "лечь is in no set"},
	{"сон был короткий", "sleep", "fn", false, "сон deliberately absent"},
	{"i napped", "sleep", "fn", false, "nap is in no set"},
	{"took a nap", "sleep", "fn", false, "break matches nothing here either"},
}

// legacyFactParse — DefaultFactParser exactly as it stood at 22edc3c's parent
// (0445693), copied here so the corpus scores the trade rather than describing
// it. Do not fix it. It is a frozen baseline, and the day it stops being worth
// comparing against, delete it and the two-column table with it.
func legacyFactParse(utterance string) (string, string, bool) {
	s := strings.ToLower(strings.TrimSpace(utterance))
	hasRoot := func(root string) bool { return strings.Contains(s, root) }
	containsWord := func(w string) bool {
		for _, tok := range strings.Fields(s) {
			if tok == w {
				return true
			}
		}
		return false
	}
	switch {
	case (containsWord("water") && containsWord("drank")) ||
		(hasRoot("вод") && (hasRoot("пил") || hasRoot("пью") || hasRoot("пей"))):
		return "water", `"drank"`, true
	case containsWord("meal") || (containsWord("ate") && !containsWord("backup")) || containsWord("lunch") || containsWord("dinner") ||
		hasRoot("поел") || hasRoot("поесть") || hasRoot("куша") || hasRoot("обед") || hasRoot("ужин") || hasRoot("завтрак") || hasRoot("еда"):
		return "meal", `"ate"`, true
	case containsWord("shower") || hasRoot("душ"):
		return "shower", `"took"`, true
	case containsWord("break") || hasRoot("перерыв") || hasRoot("отдох"):
		return "break", `"took"`, true
	case containsWord("slept") || containsWord("sleep") ||
		hasRoot("спал") || hasRoot("выспал"):
		if v, ok := parseDurationValue(afterWord(s, "slept")); ok {
			return "sleep", strconv.Quote(v), true
		}
		return "sleep", `"slept"`, true
	}
	return "", "", false
}

// TestFactParserCorpus scores both parsers over the corpus and prints the
// side-by-side. Run it with -v; the table is the output.
func TestFactParserCorpus(t *testing.T) {
	type tally struct{ tpHit, tpMiss, misfire, misfireOK, fnHit, fnMiss int }
	var now, old tally

	var rows []string
	rows = append(rows, "| utterance | class | want | old (substring) | new (closed class) |")
	rows = append(rows, "|---|---|---|---|---|")

	score := func(key string, ok bool, c factCase, tl *tally) string {
		got := ""
		if ok {
			got = key
		}
		switch c.class {
		case "tp":
			if got == c.want {
				tl.tpHit++
			} else {
				tl.tpMiss++
			}
		case "misfire":
			if got == c.want {
				tl.misfireOK++
			} else {
				tl.misfire++
			}
		case "fn":
			if got == c.want {
				tl.fnHit++
			} else {
				tl.fnMiss++
			}
		}
		if got == "" {
			return "—"
		}
		return got
	}

	for _, c := range factCorpus {
		nk, _, nok := DefaultFactParser{}.Parse(c.utterance)
		ok, _, ook := legacyFactParse(c.utterance)
		gotNew := score(nk, nok, c, &now)
		gotOld := score(ok, ook, c, &old)
		want := c.want
		if want == "" {
			want = "—"
		}
		mark := ""
		if gotOld != gotNew {
			mark = " **≠**"
		}
		rows = append(rows, "| `"+c.utterance+"` | "+c.class+" | "+want+" | "+gotOld+" | "+gotNew+mark+" |")
	}

	line := func(name string, tl tally) string {
		return name +
			": true positives " + strconv.Itoa(tl.tpHit) + "/" + strconv.Itoa(tl.tpHit+tl.tpMiss) +
			", misfires rejected " + strconv.Itoa(tl.misfireOK) + "/" + strconv.Itoa(tl.misfireOK+tl.misfire) +
			", false-negative cases recovered " + strconv.Itoa(tl.fnHit) + "/" + strconv.Itoa(tl.fnHit+tl.fnMiss)
	}
	t.Log("\n" + strings.Join(rows, "\n") + "\n\n" + line("old (substring)", old) + "\n" + line("new (closed class)", now))

	// The two regressions that are not judgement calls.
	for _, c := range factCorpus {
		k, _, ok := DefaultFactParser{}.Parse(c.utterance)
		got := ""
		if ok {
			got = k
		}
		if c.class == "fn" {
			continue
		}
		if c.broken {
			// Recorded as wrong on 06-08. If it starts passing, someone fixed
			// it and the flag is now a lie — say so rather than staying green.
			if got == c.want {
				t.Errorf("%q now returns %q as wanted — drop its broken flag", c.utterance, got)
			}
			continue
		}
		if got != c.want {
			t.Errorf("%s %q: got %q, want %q (%s)", c.class, c.utterance, got, c.want, c.note)
		}
	}
}