Merge pull request 'Measure the fact parser: land the corpus on master (V-586)' (#181) from task/586-defaultfactparser-uses-hand-written-russ into master
This commit was merged in pull request #181.
This commit is contained in:
@@ -0,0 +1,286 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The fact-parser corpus (V-586). DefaultFactParser is the only thing standing
|
||||
// between a spoken sentence and a written self-care fact, and until this file
|
||||
// it was covered by a handful of examples chosen by whoever last edited it. The
|
||||
// RU routing fixture does not cover it either: it holds three fact cases and
|
||||
// all three miss on intent, so the parser is never reached and a parser change
|
||||
// scores as "unchanged".
|
||||
//
|
||||
// So the corpus is here, and it scores BOTH implementations: the closed-class
|
||||
// parser that ships, and legacyFactParse below, a faithful copy of the
|
||||
// hand-written substring version 22edc3c replaced. The table is the measurement
|
||||
// — see docs/evals/2026-08-06-fact-parser.md for the numbers as of that day.
|
||||
//
|
||||
// Three case classes, and the third is the point:
|
||||
//
|
||||
// true positive — a sentence he would say, with the key it must write.
|
||||
// misfire — a sentence the substring parser wrote a fact for and
|
||||
// should not have. want is "".
|
||||
// false-negative — a sentence he would plausibly say whose word is in NO
|
||||
// lexicon set. want is the key a human would assign, and
|
||||
// the ship parser is EXPECTED to miss it. These measure the
|
||||
// cost of the word-list design, not a bug in it.
|
||||
//
|
||||
// Nothing here asserts a score. A closed class is a decision about vocabulary
|
||||
// and the bar belongs in a dated eval, not in an assertion that turns every
|
||||
// vocabulary edit into a red test. What it does fail on is a regression in the
|
||||
// two classes that are not judgement calls: a true positive that stops matching
|
||||
// and a misfire that starts.
|
||||
|
||||
type factCase struct {
|
||||
utterance string
|
||||
want string // "" — no fact
|
||||
class string // "tp", "misfire", "fn"
|
||||
broken bool // ship parser is known to get this wrong; see the 06-08 eval
|
||||
note string
|
||||
}
|
||||
|
||||
var factCorpus = []factCase{
|
||||
// ---- water, true positives -------------------------------------------
|
||||
{"выпил стакан воды", "water", "tp", false, ""},
|
||||
{"попил воды", "water", "tp", false, ""},
|
||||
{"я попил водички", "water", "tp", false, ""},
|
||||
{"пью воду", "water", "tp", false, ""},
|
||||
{"воду пил уже", "water", "tp", false, ""},
|
||||
{"допил воду", "water", "tp", true, "REGRESSION: the dictionary lemmatises допил to допилить, to finish sawing — the same saw collision drink_verbs already carries пил for, unfixed for the prefixed form"},
|
||||
{"запил таблетку водой", "water", "tp", false, ""},
|
||||
{"drank water", "water", "tp", false, ""},
|
||||
{"i drank some water", "water", "tp", false, ""},
|
||||
|
||||
// ---- meal, true positives --------------------------------------------
|
||||
{"поужинал", "meal", "tp", false, ""},
|
||||
{"я пообедал", "meal", "tp", false, ""},
|
||||
{"позавтракал кашей", "meal", "tp", false, ""},
|
||||
{"перекусил бутербродом", "meal", "tp", false, ""},
|
||||
{"покушал", "meal", "tp", false, ""},
|
||||
{"поел супа", "meal", "tp", false, ""},
|
||||
{"обед был в час", "meal", "tp", false, ""},
|
||||
{"ужинать буду позже", "meal", "tp", false, ""},
|
||||
{"i ate", "meal", "tp", false, ""},
|
||||
{"had lunch", "meal", "tp", false, ""},
|
||||
{"dinner done", "meal", "tp", false, ""},
|
||||
|
||||
// ---- shower, true positives ------------------------------------------
|
||||
{"принял душ", "shower", "tp", false, ""},
|
||||
{"сходил в душ", "shower", "tp", false, ""},
|
||||
{"душ принят", "shower", "tp", false, ""},
|
||||
{"ополоснулся душем", "shower", "tp", false, ""},
|
||||
{"took a shower", "shower", "tp", false, ""},
|
||||
{"i showered", "shower", "tp", false, ""},
|
||||
|
||||
// ---- break, true positives -------------------------------------------
|
||||
{"сделал перерыв", "break", "tp", false, ""},
|
||||
{"отдохнул полчаса", "break", "tp", false, ""},
|
||||
{"передохнул немного", "break", "tp", false, ""},
|
||||
{"отдыхаю", "break", "tp", false, ""},
|
||||
{"был перерыв на обед", "meal", "tp", false, "meal wins the switch; both keys are true of the sentence"},
|
||||
{"took a break", "break", "tp", false, ""},
|
||||
|
||||
// ---- sleep, true positives -------------------------------------------
|
||||
{"спал восемь часов", "sleep", "tp", false, ""},
|
||||
{"спала плохо", "sleep", "tp", false, "she may report her own; the parser is speaker-agnostic"},
|
||||
{"поспал днём", "sleep", "tp", false, ""},
|
||||
{"выспался наконец", "sleep", "tp", false, ""},
|
||||
{"проспал будильник", "sleep", "tp", false, ""},
|
||||
{"пойду спать", "sleep", "tp", false, ""},
|
||||
{"slept 8 hours", "sleep", "tp", false, ""},
|
||||
{"i slept badly", "sleep", "tp", false, ""},
|
||||
|
||||
// ---- misfires 22edc3c set out to reject -------------------------------
|
||||
{"пилот сказал что вылет через час", "", "misfire", false, "substring пил"},
|
||||
{"водитель уже подъехал", "", "misfire", false, "substring вод"},
|
||||
{"надо заводить машину", "", "misfire", false, "substring вод"},
|
||||
{"душа болит", "", "misfire", false, "substring душ"},
|
||||
{"в комнате душно", "", "misfire", false, "substring душ"},
|
||||
{"это была беда", "", "misfire", false, "substring еда"},
|
||||
{"наша победа", "", "misfire", false, "substring еда"},
|
||||
{"пила лежит в гараже", "", "misfire", false, "substring пил"},
|
||||
{"водитель пилота ждёт", "", "misfire", false, "both stems in one sentence — the old water arm fires"},
|
||||
{"обеденный перерыв отменили", "", "misfire", true, "neither parser gets this: the old one writes meal off the adjective, the new one writes break off перерыв. A cancelled break is not a break taken."},
|
||||
|
||||
// ---- hard negatives that decide the design ----------------------------
|
||||
{"есть новости по бэкапу базы", "", "misfire", false, "есть is the existential, not a meal"},
|
||||
{"напоминания на завтра есть", "", "misfire", false, "завтра carries завтрак as a substring"},
|
||||
{"на душе легко", "", "misfire", false, "душе is the soul, and also the prepositional of душ"},
|
||||
{"пилил доску весь вечер", "", "misfire", false, ""},
|
||||
{"поставь будильник на завтра", "", "misfire", false, "завтра again, no meal"},
|
||||
|
||||
// ---- false negatives: words in no lexicon set -------------------------
|
||||
// Water.
|
||||
{"выпил чаю", "water", "fn", false, "hydration by any liquid; чай is in no set"},
|
||||
{"глотнул воды", "water", "fn", false, "глотнуть is not a drink verb"},
|
||||
{"хлебнул воды", "water", "fn", false, "хлебнуть is not a drink verb"},
|
||||
{"воды хлебнул из бутылки", "water", "fn", false, ""},
|
||||
{"выпил стакан", "water", "fn", false, "the noun is elided; he says this"},
|
||||
{"i hydrated", "water", "fn", false, "hydrate is in no set"},
|
||||
{"finished my bottle of water", "water", "fn", false, "no drink verb in the English set"},
|
||||
|
||||
// Meal.
|
||||
{"ем суп", "meal", "fn", false, "есть is deliberately absent, and this is the cost"},
|
||||
{"съел бутерброд", "meal", "fn", false, "съесть is in no set"},
|
||||
{"наелся", "meal", "fn", false, "наесться is in no set"},
|
||||
{"пожрал", "meal", "fn", false, "coarse but spoken"},
|
||||
{"полдник был", "meal", "fn", false, "полдник is in no set"},
|
||||
{"i had a snack", "meal", "fn", false, "snack is in no set"},
|
||||
{"having supper", "meal", "fn", false, "supper is in no set"},
|
||||
{"brunch was good", "meal", "fn", false, "brunch is in no set"},
|
||||
{"i eat now", "meal", "fn", false, "eat is in no set — only ate is"},
|
||||
|
||||
// Shower.
|
||||
{"был в душе", "shower", "fn", false, "prepositional; anyExact cannot take it without taking the soul"},
|
||||
{"после душа полегчало", "shower", "fn", false, "genitive, same collision"},
|
||||
{"помылся", "shower", "fn", false, "помыться is in no set"},
|
||||
{"сходил в ванную", "shower", "fn", false, "ванная is in no set"},
|
||||
{"искупался", "shower", "fn", false, "искупаться is in no set"},
|
||||
{"i am showering", "shower", "fn", false, "showering is not a member and the set is exact-matched"},
|
||||
|
||||
// Break.
|
||||
{"сделал передышку", "break", "fn", false, "передышка is in no set"},
|
||||
{"перекур", "break", "fn", false, "перекур is in no set"},
|
||||
{"полежал немного", "break", "fn", false, "полежать is in no set"},
|
||||
{"сделал паузу", "break", "fn", false, "пауза is in no set"},
|
||||
{"i took five", "break", "fn", false, "idiomatic, in no set"},
|
||||
{"resting now", "break", "fn", false, "resting is not a member and rest is matched by lemma only"},
|
||||
|
||||
// Sleep.
|
||||
{"вздремнул", "sleep", "fn", false, "вздремнуть is in no set"},
|
||||
{"прикорнул на диване", "sleep", "fn", false, "прикорнуть is in no set"},
|
||||
{"дрых до обеда", "sleep", "fn", false, "дрыхнуть is in no set — and обед makes this a MEAL for both parsers"},
|
||||
{"недоспал", "sleep", "fn", false, "недоспать is in no set"},
|
||||
{"лёг в двенадцать", "sleep", "fn", false, "лечь is in no set"},
|
||||
{"сон был короткий", "sleep", "fn", false, "сон deliberately absent"},
|
||||
{"i napped", "sleep", "fn", false, "nap is in no set"},
|
||||
{"took a nap", "sleep", "fn", false, "break matches nothing here either"},
|
||||
}
|
||||
|
||||
// legacyFactParse — DefaultFactParser exactly as it stood at 22edc3c's parent
|
||||
// (0445693), copied here so the corpus scores the trade rather than describing
|
||||
// it. Do not fix it. It is a frozen baseline, and the day it stops being worth
|
||||
// comparing against, delete it and the two-column table with it.
|
||||
func legacyFactParse(utterance string) (string, string, bool) {
|
||||
s := strings.ToLower(strings.TrimSpace(utterance))
|
||||
hasRoot := func(root string) bool { return strings.Contains(s, root) }
|
||||
containsWord := func(w string) bool {
|
||||
for _, tok := range strings.Fields(s) {
|
||||
if tok == w {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
switch {
|
||||
case (containsWord("water") && containsWord("drank")) ||
|
||||
(hasRoot("вод") && (hasRoot("пил") || hasRoot("пью") || hasRoot("пей"))):
|
||||
return "water", `"drank"`, true
|
||||
case containsWord("meal") || (containsWord("ate") && !containsWord("backup")) || containsWord("lunch") || containsWord("dinner") ||
|
||||
hasRoot("поел") || hasRoot("поесть") || hasRoot("куша") || hasRoot("обед") || hasRoot("ужин") || hasRoot("завтрак") || hasRoot("еда"):
|
||||
return "meal", `"ate"`, true
|
||||
case containsWord("shower") || hasRoot("душ"):
|
||||
return "shower", `"took"`, true
|
||||
case containsWord("break") || hasRoot("перерыв") || hasRoot("отдох"):
|
||||
return "break", `"took"`, true
|
||||
case containsWord("slept") || containsWord("sleep") ||
|
||||
hasRoot("спал") || hasRoot("выспал"):
|
||||
if v, ok := parseDurationValue(afterWord(s, "slept")); ok {
|
||||
return "sleep", strconv.Quote(v), true
|
||||
}
|
||||
return "sleep", `"slept"`, true
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// TestFactParserCorpus scores both parsers over the corpus and prints the
|
||||
// side-by-side. Run it with -v; the table is the output.
|
||||
func TestFactParserCorpus(t *testing.T) {
|
||||
type tally struct{ tpHit, tpMiss, misfire, misfireOK, fnHit, fnMiss int }
|
||||
var now, old tally
|
||||
|
||||
var rows []string
|
||||
rows = append(rows, "| utterance | class | want | old (substring) | new (closed class) |")
|
||||
rows = append(rows, "|---|---|---|---|---|")
|
||||
|
||||
score := func(key string, ok bool, c factCase, tl *tally) string {
|
||||
got := ""
|
||||
if ok {
|
||||
got = key
|
||||
}
|
||||
switch c.class {
|
||||
case "tp":
|
||||
if got == c.want {
|
||||
tl.tpHit++
|
||||
} else {
|
||||
tl.tpMiss++
|
||||
}
|
||||
case "misfire":
|
||||
if got == c.want {
|
||||
tl.misfireOK++
|
||||
} else {
|
||||
tl.misfire++
|
||||
}
|
||||
case "fn":
|
||||
if got == c.want {
|
||||
tl.fnHit++
|
||||
} else {
|
||||
tl.fnMiss++
|
||||
}
|
||||
}
|
||||
if got == "" {
|
||||
return "—"
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
for _, c := range factCorpus {
|
||||
nk, _, nok := DefaultFactParser{}.Parse(c.utterance)
|
||||
ok, _, ook := legacyFactParse(c.utterance)
|
||||
gotNew := score(nk, nok, c, &now)
|
||||
gotOld := score(ok, ook, c, &old)
|
||||
want := c.want
|
||||
if want == "" {
|
||||
want = "—"
|
||||
}
|
||||
mark := ""
|
||||
if gotOld != gotNew {
|
||||
mark = " **≠**"
|
||||
}
|
||||
rows = append(rows, "| `"+c.utterance+"` | "+c.class+" | "+want+" | "+gotOld+" | "+gotNew+mark+" |")
|
||||
}
|
||||
|
||||
line := func(name string, tl tally) string {
|
||||
return name +
|
||||
": true positives " + strconv.Itoa(tl.tpHit) + "/" + strconv.Itoa(tl.tpHit+tl.tpMiss) +
|
||||
", misfires rejected " + strconv.Itoa(tl.misfireOK) + "/" + strconv.Itoa(tl.misfireOK+tl.misfire) +
|
||||
", false-negative cases recovered " + strconv.Itoa(tl.fnHit) + "/" + strconv.Itoa(tl.fnHit+tl.fnMiss)
|
||||
}
|
||||
t.Log("\n" + strings.Join(rows, "\n") + "\n\n" + line("old (substring)", old) + "\n" + line("new (closed class)", now))
|
||||
|
||||
// The two regressions that are not judgement calls.
|
||||
for _, c := range factCorpus {
|
||||
k, _, ok := DefaultFactParser{}.Parse(c.utterance)
|
||||
got := ""
|
||||
if ok {
|
||||
got = k
|
||||
}
|
||||
if c.class == "fn" {
|
||||
continue
|
||||
}
|
||||
if c.broken {
|
||||
// Recorded as wrong on 06-08. If it starts passing, someone fixed
|
||||
// it and the flag is now a lie — say so rather than staying green.
|
||||
if got == c.want {
|
||||
t.Errorf("%q now returns %q as wanted — drop its broken flag", c.utterance, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("%s %q: got %q, want %q (%s)", c.class, c.utterance, got, c.want, c.note)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user