package router import ( "strconv" "strings" "testing" ) // The fact-parser corpus (V-586). DefaultFactParser is the only thing standing // between a spoken sentence and a written self-care fact, and until this file // it was covered by a handful of examples chosen by whoever last edited it. The // RU routing fixture does not cover it either: it holds three fact cases and // all three miss on intent, so the parser is never reached and a parser change // scores as "unchanged". // // So the corpus is here, and it scores BOTH implementations: the closed-class // parser that ships, and legacyFactParse below, a faithful copy of the // hand-written substring version 22edc3c replaced. The table is the measurement // — see docs/evals/2026-08-06-fact-parser.md for the numbers as of that day. // // Three case classes, and the third is the point: // // true positive — a sentence he would say, with the key it must write. // misfire — a sentence the substring parser wrote a fact for and // should not have. want is "". // false-negative — a sentence he would plausibly say whose word is in NO // lexicon set. want is the key a human would assign, and // the ship parser is EXPECTED to miss it. These measure the // cost of the word-list design, not a bug in it. // // Nothing here asserts a score. A closed class is a decision about vocabulary // and the bar belongs in a dated eval, not in an assertion that turns every // vocabulary edit into a red test. What it does fail on is a regression in the // two classes that are not judgement calls: a true positive that stops matching // and a misfire that starts. type factCase struct { utterance string want string // "" — no fact class string // "tp", "misfire", "fn" broken bool // ship parser is known to get this wrong; see the 06-08 eval note string } var factCorpus = []factCase{ // ---- water, true positives ------------------------------------------- {"выпил стакан воды", "water", "tp", false, ""}, {"попил воды", "water", "tp", false, ""}, {"я попил водички", "water", "tp", false, ""}, {"пью воду", "water", "tp", false, ""}, {"воду пил уже", "water", "tp", false, ""}, {"допил воду", "water", "tp", true, "REGRESSION: the dictionary lemmatises допил to допилить, to finish sawing — the same saw collision drink_verbs already carries пил for, unfixed for the prefixed form"}, {"запил таблетку водой", "water", "tp", false, ""}, {"drank water", "water", "tp", false, ""}, {"i drank some water", "water", "tp", false, ""}, // ---- meal, true positives -------------------------------------------- {"поужинал", "meal", "tp", false, ""}, {"я пообедал", "meal", "tp", false, ""}, {"позавтракал кашей", "meal", "tp", false, ""}, {"перекусил бутербродом", "meal", "tp", false, ""}, {"покушал", "meal", "tp", false, ""}, {"поел супа", "meal", "tp", false, ""}, {"обед был в час", "meal", "tp", false, ""}, {"ужинать буду позже", "meal", "tp", false, ""}, {"i ate", "meal", "tp", false, ""}, {"had lunch", "meal", "tp", false, ""}, {"dinner done", "meal", "tp", false, ""}, // ---- shower, true positives ------------------------------------------ {"принял душ", "shower", "tp", false, ""}, {"сходил в душ", "shower", "tp", false, ""}, {"душ принят", "shower", "tp", false, ""}, {"ополоснулся душем", "shower", "tp", false, ""}, {"took a shower", "shower", "tp", false, ""}, {"i showered", "shower", "tp", false, ""}, // ---- break, true positives ------------------------------------------- {"сделал перерыв", "break", "tp", false, ""}, {"отдохнул полчаса", "break", "tp", false, ""}, {"передохнул немного", "break", "tp", false, ""}, {"отдыхаю", "break", "tp", false, ""}, {"был перерыв на обед", "meal", "tp", false, "meal wins the switch; both keys are true of the sentence"}, {"took a break", "break", "tp", false, ""}, // ---- sleep, true positives ------------------------------------------- {"спал восемь часов", "sleep", "tp", false, ""}, {"спала плохо", "sleep", "tp", false, "she may report her own; the parser is speaker-agnostic"}, {"поспал днём", "sleep", "tp", false, ""}, {"выспался наконец", "sleep", "tp", false, ""}, {"проспал будильник", "sleep", "tp", false, ""}, {"пойду спать", "sleep", "tp", false, ""}, {"slept 8 hours", "sleep", "tp", false, ""}, {"i slept badly", "sleep", "tp", false, ""}, // ---- misfires 22edc3c set out to reject ------------------------------- {"пилот сказал что вылет через час", "", "misfire", false, "substring пил"}, {"водитель уже подъехал", "", "misfire", false, "substring вод"}, {"надо заводить машину", "", "misfire", false, "substring вод"}, {"душа болит", "", "misfire", false, "substring душ"}, {"в комнате душно", "", "misfire", false, "substring душ"}, {"это была беда", "", "misfire", false, "substring еда"}, {"наша победа", "", "misfire", false, "substring еда"}, {"пила лежит в гараже", "", "misfire", false, "substring пил"}, {"водитель пилота ждёт", "", "misfire", false, "both stems in one sentence — the old water arm fires"}, {"обеденный перерыв отменили", "", "misfire", true, "neither parser gets this: the old one writes meal off the adjective, the new one writes break off перерыв. A cancelled break is not a break taken."}, // ---- hard negatives that decide the design ---------------------------- {"есть новости по бэкапу базы", "", "misfire", false, "есть is the existential, not a meal"}, {"напоминания на завтра есть", "", "misfire", false, "завтра carries завтрак as a substring"}, {"на душе легко", "", "misfire", false, "душе is the soul, and also the prepositional of душ"}, {"пилил доску весь вечер", "", "misfire", false, ""}, {"поставь будильник на завтра", "", "misfire", false, "завтра again, no meal"}, // ---- false negatives: words in no lexicon set ------------------------- // Water. {"выпил чаю", "water", "fn", false, "hydration by any liquid; чай is in no set"}, {"глотнул воды", "water", "fn", false, "глотнуть is not a drink verb"}, {"хлебнул воды", "water", "fn", false, "хлебнуть is not a drink verb"}, {"воды хлебнул из бутылки", "water", "fn", false, ""}, {"выпил стакан", "water", "fn", false, "the noun is elided; he says this"}, {"i hydrated", "water", "fn", false, "hydrate is in no set"}, {"finished my bottle of water", "water", "fn", false, "no drink verb in the English set"}, // Meal. {"ем суп", "meal", "fn", false, "есть is deliberately absent, and this is the cost"}, {"съел бутерброд", "meal", "fn", false, "съесть is in no set"}, {"наелся", "meal", "fn", false, "наесться is in no set"}, {"пожрал", "meal", "fn", false, "coarse but spoken"}, {"полдник был", "meal", "fn", false, "полдник is in no set"}, {"i had a snack", "meal", "fn", false, "snack is in no set"}, {"having supper", "meal", "fn", false, "supper is in no set"}, {"brunch was good", "meal", "fn", false, "brunch is in no set"}, {"i eat now", "meal", "fn", false, "eat is in no set — only ate is"}, // Shower. {"был в душе", "shower", "fn", false, "prepositional; anyExact cannot take it without taking the soul"}, {"после душа полегчало", "shower", "fn", false, "genitive, same collision"}, {"помылся", "shower", "fn", false, "помыться is in no set"}, {"сходил в ванную", "shower", "fn", false, "ванная is in no set"}, {"искупался", "shower", "fn", false, "искупаться is in no set"}, {"i am showering", "shower", "fn", false, "showering is not a member and the set is exact-matched"}, // Break. {"сделал передышку", "break", "fn", false, "передышка is in no set"}, {"перекур", "break", "fn", false, "перекур is in no set"}, {"полежал немного", "break", "fn", false, "полежать is in no set"}, {"сделал паузу", "break", "fn", false, "пауза is in no set"}, {"i took five", "break", "fn", false, "idiomatic, in no set"}, {"resting now", "break", "fn", false, "resting is not a member and rest is matched by lemma only"}, // Sleep. {"вздремнул", "sleep", "fn", false, "вздремнуть is in no set"}, {"прикорнул на диване", "sleep", "fn", false, "прикорнуть is in no set"}, {"дрых до обеда", "sleep", "fn", false, "дрыхнуть is in no set — and обед makes this a MEAL for both parsers"}, {"недоспал", "sleep", "fn", false, "недоспать is in no set"}, {"лёг в двенадцать", "sleep", "fn", false, "лечь is in no set"}, {"сон был короткий", "sleep", "fn", false, "сон deliberately absent"}, {"i napped", "sleep", "fn", false, "nap is in no set"}, {"took a nap", "sleep", "fn", false, "break matches nothing here either"}, } // legacyFactParse — DefaultFactParser exactly as it stood at 22edc3c's parent // (0445693), copied here so the corpus scores the trade rather than describing // it. Do not fix it. It is a frozen baseline, and the day it stops being worth // comparing against, delete it and the two-column table with it. func legacyFactParse(utterance string) (string, string, bool) { s := strings.ToLower(strings.TrimSpace(utterance)) hasRoot := func(root string) bool { return strings.Contains(s, root) } containsWord := func(w string) bool { for _, tok := range strings.Fields(s) { if tok == w { return true } } return false } switch { case (containsWord("water") && containsWord("drank")) || (hasRoot("вод") && (hasRoot("пил") || hasRoot("пью") || hasRoot("пей"))): return "water", `"drank"`, true case containsWord("meal") || (containsWord("ate") && !containsWord("backup")) || containsWord("lunch") || containsWord("dinner") || hasRoot("поел") || hasRoot("поесть") || hasRoot("куша") || hasRoot("обед") || hasRoot("ужин") || hasRoot("завтрак") || hasRoot("еда"): return "meal", `"ate"`, true case containsWord("shower") || hasRoot("душ"): return "shower", `"took"`, true case containsWord("break") || hasRoot("перерыв") || hasRoot("отдох"): return "break", `"took"`, true case containsWord("slept") || containsWord("sleep") || hasRoot("спал") || hasRoot("выспал"): if v, ok := parseDurationValue(afterWord(s, "slept")); ok { return "sleep", strconv.Quote(v), true } return "sleep", `"slept"`, true } return "", "", false } // TestFactParserCorpus scores both parsers over the corpus and prints the // side-by-side. Run it with -v; the table is the output. func TestFactParserCorpus(t *testing.T) { type tally struct{ tpHit, tpMiss, misfire, misfireOK, fnHit, fnMiss int } var now, old tally var rows []string rows = append(rows, "| utterance | class | want | old (substring) | new (closed class) |") rows = append(rows, "|---|---|---|---|---|") score := func(key string, ok bool, c factCase, tl *tally) string { got := "" if ok { got = key } switch c.class { case "tp": if got == c.want { tl.tpHit++ } else { tl.tpMiss++ } case "misfire": if got == c.want { tl.misfireOK++ } else { tl.misfire++ } case "fn": if got == c.want { tl.fnHit++ } else { tl.fnMiss++ } } if got == "" { return "—" } return got } for _, c := range factCorpus { nk, _, nok := DefaultFactParser{}.Parse(c.utterance) ok, _, ook := legacyFactParse(c.utterance) gotNew := score(nk, nok, c, &now) gotOld := score(ok, ook, c, &old) want := c.want if want == "" { want = "—" } mark := "" if gotOld != gotNew { mark = " **≠**" } rows = append(rows, "| `"+c.utterance+"` | "+c.class+" | "+want+" | "+gotOld+" | "+gotNew+mark+" |") } line := func(name string, tl tally) string { return name + ": true positives " + strconv.Itoa(tl.tpHit) + "/" + strconv.Itoa(tl.tpHit+tl.tpMiss) + ", misfires rejected " + strconv.Itoa(tl.misfireOK) + "/" + strconv.Itoa(tl.misfireOK+tl.misfire) + ", false-negative cases recovered " + strconv.Itoa(tl.fnHit) + "/" + strconv.Itoa(tl.fnHit+tl.fnMiss) } t.Log("\n" + strings.Join(rows, "\n") + "\n\n" + line("old (substring)", old) + "\n" + line("new (closed class)", now)) // The two regressions that are not judgement calls. for _, c := range factCorpus { k, _, ok := DefaultFactParser{}.Parse(c.utterance) got := "" if ok { got = k } if c.class == "fn" { continue } if c.broken { // Recorded as wrong on 06-08. If it starts passing, someone fixed // it and the flag is now a lie — say so rather than staying green. if got == c.want { t.Errorf("%q now returns %q as wanted — drop its broken flag", c.utterance, got) } continue } if got != c.want { t.Errorf("%s %q: got %q, want %q (%s)", c.class, c.utterance, got, c.want, c.note) } } }