package eval import ( "context" "strings" "testing" "github.com/kami/maven/internal/loop" "github.com/kami/maven/internal/phraser" ) func TestLoadFixture(t *testing.T) { f, err := Load() if err != nil { t.Fatalf("Load: %v", err) } if _, err := f.Now(); err != nil { t.Fatalf("Now: %v", err) } seen := map[string]bool{} for _, c := range f.Cases { if seen[c.ID] { t.Errorf("duplicate case id %q", c.ID) } seen[c.ID] = true if c.Rule == "" || c.Severity < 1 || c.Severity > 4 { t.Errorf("%s: rule %q severity %d", c.ID, c.Rule, c.Severity) } if len(c.WantAny) == 0 { t.Errorf("%s: no want_any, the on-topic check would always pass", c.ID) } } // Coverage floor: all five loop rules plus both minted families, or the // fixture measures a subset and the score does not mean what it says. for _, rule := range []string{"water", "meal", "break", "service_down", "netdata_critical", "routine", "morning"} { found := false for _, c := range f.Cases { if ruleFamily(c.Rule) == rule { found = true } } if !found { t.Errorf("no case for rule family %q", rule) } } } // TestStubBaseline is the CI ratchet: the deterministic Stub, no model, no // network. The floor is low on purpose — the Stub is English template phrasing, // so it fails `lang` on every case by construction. The point of the ratchet is // that the checks keep running and the Stub does not get worse, not that the // Stub is good. func TestStubBaseline(t *testing.T) { f, err := Load() if err != nil { t.Fatalf("Load: %v", err) } rep, err := Score(context.Background(), "stub (deterministic floor)", phraser.NewStub(), f) if err != nil { t.Fatalf("Score: %v", err) } t.Log("\n" + rep.String() + rep.Messages()) if rep.Errors != 0 { t.Errorf("stub returned %d errors — the deterministic path must never fail", rep.Errors) } // Per-check ratchets rather than one composite: the Stub's composite is 0 // (it never passes `lang`), so a composite floor would catch nothing. floors := map[string]int{ CheckMood: 15, // 12, not 15: the Stub's `break` template genuinely runs past the // ceiling ("you've been at your desk for 4 hours without a break — step // away for a bit." is 76 chars but 16 words). Left failing rather than // raising the ceiling to hide it. CheckLength: 12, CheckFeminine: 15, CheckHisGender: 15, CheckAddress: 15, CheckCringe: 15, CheckOnTopic: 12, } for name, floor := range floors { if rep.ByCheck[name] < floor { t.Errorf("check %s: %d/%d, below ratchet %d — phrasing regressed", name, rep.ByCheck[name], rep.Total, floor) } } } // TestChecksCatchWhatTheyClaim — the checks are the measurement, so they get // their own tests. Without these, a bad regexp would silently make every // phrasing run look clean. func TestChecksCatchWhatTheyClaim(t *testing.T) { water := Case{Rule: "water", WantAny: []string{"вод"}} cases := []struct { name string body string want string // the check that must fail, "" for a clean message }{ {"clean", "уже четыре часа без воды — попей.", ""}, {"long", "уже четыре часа без воды, а это довольно много, и вообще пить надо регулярно, иначе будет плохо совсем", CheckLength}, {"english", "you haven't had water in 4 hours, drink something", CheckLang}, {"masculine self", "я напомнил про воду.", CheckFeminine}, {"masculine dropped pronoun", "напомнил тебе про воду.", CheckFeminine}, {"masculine predicative", "я должен сказать: попей воды.", CheckFeminine}, // The other direction: HE is male, so second-person masculine is right. {"second person masculine ok", "ты не пил воду четыре часа.", ""}, // The real observed failure: she addressed him as a woman. {"feminine second person", "ты давно не отдыхала — попей воды.", CheckHisGender}, {"feminine second person no dash", "ты пила воду четыре часа назад.", CheckHisGender}, // Her own feminine verb next to "ты" is correct and must not be flagged. {"her feminine verb near ты", "я заметила, что ты не пил воду.", ""}, {"her feminine verb about him", "напомнила тебе про воду.", ""}, // A feminine noun subject in the same sentence is not him. {"feminine noun subject ok", "зарядка была утром, ты её пропустил, попей воды.", ""}, {"feminine self ok", "я заметила: воды не было четыре часа.", ""}, {"pet name", "милый, попей воды.", CheckCringe}, {"emoji", "попей воды 💧", CheckCringe}, {"exclamations", "попей воды!!", CheckCringe}, {"fake concern", "я беспокоюсь: воды не было четыре часа.", CheckCringe}, {"apology", "извини, что отвлекаю — попей воды.", CheckCringe}, {"emotional support", "я рядом, ты не один. попей воды.", CheckCringe}, {"asks how he feels", "как ты себя чувствуешь? попей воды.", CheckCringe}, {"praise", "молодец! теперь попей воды.", CheckCringe}, {"off topic", "пора бы уже что-то сделать.", CheckOnTopic}, // The two recorded persona breaks from the phrasing eval run. Pinned as // unit tests because an eval run is sampled and may not reproduce them. {"formal plural", "Приходите… Жду вас", CheckAddress}, {"third person about him", "Он не ел 11 дней", CheckAddress}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var failed []string for _, r := range RunChecks(water, tc.body, "neutral") { if !r.Pass { failed = append(failed, r.Name+"("+r.Detail+")") } } joined := strings.Join(failed, " ") switch { case tc.want == "" && len(failed) > 0: t.Errorf("clean message flagged: %s", joined) case tc.want != "" && !strings.Contains(joined, tc.want+"("): t.Errorf("want %s to fail, got %q", tc.want, joined) } }) } } // TestAddressCheck — the address check on its own, so the messages that must NOT // trip it can be written without also having to satisfy the on-topic check. func TestAddressCheck(t *testing.T) { bad := []string{ "Приходите… Жду вас", // the recorded formal-plural break "Он не ел 11 дней", // the recorded third-person break "Выпейте воды, пожалуйста.", // plural imperative on its own "Ваш обед был давно.", // formal possessive } for _, body := range bad { if r := checkAddress(body); r.Pass { t.Errorf("persona break not caught: %q", body) } else { t.Logf("%q -> %s", body, r.Detail) } } good := []string{ "ты не пил воду четыре часа — попей.", // correct informal address "сервис netdata упал, он не отвечает.", // legitimately about a third party "я заметила, что зарядка была утром.", // no address at all "в интернете опять тихо, всё работает.", // "интернете" is a noun, not an imperative } for _, body := range good { if r := checkAddress(body); !r.Pass { t.Errorf("clean message flagged: %q -> %s", body, r.Detail) } } } func TestMoodCheckUsesTheEnum(t *testing.T) { if r := checkMood("cheerful"); r.Pass { t.Error("mood outside the enum passed") } if r := checkMood(""); r.Pass { t.Error("empty mood passed") } for m := range Moods { if r := checkMood(m); !r.Pass { t.Errorf("enum mood %q failed", m) } } } // TestCandidateCarriesTheContext — the whole harness is worthless if the // Candidate it builds does not carry the duration the prompt is supposed to // name. func TestCandidateCarriesTheContext(t *testing.T) { f, err := Load() if err != nil { t.Fatalf("Load: %v", err) } now, _ := f.Now() for _, c := range f.Cases { cand := c.Candidate(now) if cand.Rule.Name != c.Rule || cand.Severity != loop.Severity(c.Severity) { t.Errorf("%s: candidate lost rule or severity", c.ID) } if c.SinceMinutes > 0 { d, ok := cand.State.Since(c.Rule) if !ok || int(d.Minutes()) != c.SinceMinutes { t.Errorf("%s: since %v ok=%v, want %d minutes", c.ID, d, ok, c.SinceMinutes) } } if c.FactKey != "" { fact, ok := cand.State.Fact(c.Rule) if !ok || fact.Key != c.FactKey { t.Errorf("%s: fact key %q, want %q", c.ID, fact.Key, c.FactKey) } } } }