diff --git a/PHRASING-EVAL-31-07-2026.md b/PHRASING-EVAL-31-07-2026.md index cc971e5..b998b48 100644 --- a/PHRASING-EVAL-31-07-2026.md +++ b/PHRASING-EVAL-31-07-2026.md @@ -105,11 +105,14 @@ that never says food. ## Broken, found, not fixed -1. **`checkFeminine` only catches half the constraint.** It scans for masculine - self-reference and passed 15/15 both runs — but three messages address the *owner* in - the feminine: "ты давно не отдыхал**а**", "он не ел". The owner is a man. The check has - no second-person gender test, so this scores clean while being exactly the persona - failure the constraint exists to prevent. This is the most important gap in the harness. +1. ~~**`checkFeminine` only catches half the constraint.**~~ **Fixed** (#381). It scanned for + masculine self-reference only, so three messages that addressed the *owner* in the feminine + ("ты давно не отдыхал**а**") scored clean. There is now a second check, `hisgender`: a + feminine past-tense verb (-ла/-лась) in a sentence addressed to him ("ты", "тебе", "твой") + fails, unless the verb is hers ("я заметила", "напомнила тебе"). It is a suffix rule, not a + parser — see the comment in `checks.go` for what it misses. A fresh 15-case run after adding + it scored **12/15** with `hisgender` 15/15; the model did not repeat the feminine address in + that sample, and the check is pinned by unit tests on the recorded bad strings instead. 2. **Grammar is not checked at all, and it is bad.** `"Он не ел 11 дней"` (it was 11 hours), `"Сонуждились 7 дней"` (not a word), `"Они забыли воду"` (wrong person entirely). Every one of these passes all six checks. The fixture measures properties, not fluency, and at @@ -125,8 +128,7 @@ that never says food. ## Next steps -1. **Add a second-person gender check** to `checks.go`. Finding 1 above. Until it exists the - feminine column means less than it looks like. +1. ~~**Add a second-person gender check**~~ — done, `hisgender` in `checks.go` (#381). 2. **Decide whether the fallback should count as a pass.** Right now `Score` cannot tell a model answer from a fallback. Either mark fallback bodies in `PhrasedNudge` or count them in their own column. Without that, any future prompt change can score well by failing diff --git a/internal/phraser/eval/checks.go b/internal/phraser/eval/checks.go index 16d2f8f..2c8f8c3 100644 --- a/internal/phraser/eval/checks.go +++ b/internal/phraser/eval/checks.go @@ -17,10 +17,14 @@ const ( CheckFeminine = "feminine" // her self-reference is feminine (hard constraint) CheckCringe = "cringe" // DESIGN.md § Non-goals, "not a relationship" CheckOnTopic = "ontopic" // says the thing the rule is about + + // CheckHisGender — the other half of the persona rule: SHE is feminine, HE + // is male. "ты давно не отдыхала" addresses the operator as a woman. + CheckHisGender = "hisgender" ) // CheckNames — report order. -var CheckNames = []string{CheckMood, CheckLang, CheckLength, CheckFeminine, CheckCringe, CheckOnTopic} +var CheckNames = []string{CheckMood, CheckLang, CheckLength, CheckFeminine, CheckHisGender, CheckCringe, CheckOnTopic} // Result — one check on one message. type Result struct { @@ -52,6 +56,7 @@ func RunChecks(c Case, body, mood string) []Result { checkLang(body), checkLength(body), checkFeminine(body), + checkHisGender(body), checkCringe(body), checkOnTopic(c, body), } @@ -176,6 +181,122 @@ func checkFeminine(body string) Result { return Result{CheckFeminine, true, ""} } +// --- he is male ---------------------------------------------------------- +// +// The mirror of checkFeminine, and the failure it was written for: the model +// wrote "ты давно не отдыхала", which addresses the operator as a woman. That +// scored clean, because checkFeminine only ever looks at how SHE speaks about +// herself. +// +// How it works: Russian past tense is gendered by suffix, -л (m) / -ла (f). So +// this looks for feminine past-tense words in a sentence that also talks TO him +// ("ты", "тебя", "тебе", "твой", …). A feminine verb that belongs to her ("я +// заметила", "напомнила тебе") is skipped — that one is correct. +// +// Honest about the limits: this is a suffix rule, not a parser. +// - False positives: a feminine noun can be the subject in the same sentence +// ("зарядка была утром, ты её пропустил"). The guard below skips a verb whose +// previous word looks like a feminine noun, which helps but will not always +// be right. +// - False negatives: gender also shows up outside the past tense (short +// adjectives, "сама"), and none of that is checked here. +// +// That is acceptable for an eval check. It is a signal to read the message, not +// a grammar verdict, and every hit prints the word it tripped on so a human can +// disagree. + +// hisMarkers — words that mean the sentence is addressed to him. +var hisMarkers = map[string]bool{ + "ты": true, "тебя": true, "тебе": true, "тобой": true, "тобою": true, + "твой": true, "твоя": true, "твоё": true, "твое": true, "твои": true, "твою": true, +} + +// notFeminineVerb — ordinary words ending in "-ла" that are not verbs. Small on +// purpose: it only has to cover words a nudge might actually use. +var notFeminineVerb = map[string]bool{ + "школа": true, "скала": true, "игла": true, "метла": true, "смола": true, + "дела": true, "тела": true, "масла": true, "стекла": true, "весла": true, + "зола": true, "пчела": true, "числа": true, "села": true, "мыла": true, +} + +// femininePast reports whether a word looks like a feminine past-tense verb: +// "отдыхала", "поела", "выспалась". +func femininePast(w string) bool { + if len([]rune(w)) < 3 || notFeminineVerb[w] { + return false + } + return strings.HasSuffix(w, "ла") || strings.HasSuffix(w, "лась") +} + +// looksFeminineNoun — a crude guard against "зарядка была": a word right before +// the verb that ends in "а"/"я" and is not itself a verb is probably the subject. +func looksFeminineNoun(w string) bool { + if femininePast(w) || len([]rune(w)) < 3 { + return false + } + return strings.HasSuffix(w, "а") || strings.HasSuffix(w, "я") +} + +// sentenceRE splits on sentence-ending punctuation, so a feminine verb in one +// sentence is not blamed on a "ты" in the next. +var sentenceRE = regexp.MustCompile(`[.!?;…]+`) + +func checkHisGender(body string) Result { + for _, sentence := range sentenceRE.Split(strings.ToLower(body), -1) { + words := wordRE.FindAllString(sentence, -1) + addressed := false + for _, w := range words { + if hisMarkers[w] { + addressed = true + } + } + if !addressed { + continue + } + for i, w := range words { + if !femininePast(w) || hersNotHis(words, i) { + continue + } + if i > 0 && looksFeminineNoun(prevWord(words, i)) { + continue + } + return Result{CheckHisGender, false, + fmt.Sprintf("feminine %q addressed to him — he is male", w)} + } + } + return Result{CheckHisGender, true, ""} +} + +// hersNotHis — the verb is Maven's own if "я" comes shortly before it, or if the +// thing she did was done to him ("напомнила тебе", "проверила за тебя"). +func hersNotHis(words []string, i int) bool { + for j := i - 1; j >= 0 && j >= i-3; j-- { + if words[j] == "я" { + return true + } + } + if i+1 < len(words) { + switch words[i+1] { + case "тебе", "тебя", "за", "тобой": + return true + } + } + return false +} + +// prevWord — the word before i, skipping "не" and punctuation, so "не отдыхала" +// still sees the subject. +func prevWord(words []string, i int) string { + for j := i - 1; j >= 0; j-- { + w := words[j] + if w == "не" || w == "ни" || !unicode.Is(unicode.Cyrillic, []rune(w)[0]) { + continue + } + return w + } + return "" +} + // --- the cringe checks --------------------------------------------------- // // "Think Jarvis without the cringe part". DESIGN.md § Non-goals: "Not a diff --git a/internal/phraser/eval/eval.go b/internal/phraser/eval/eval.go index 26c9415..f199ebb 100644 --- a/internal/phraser/eval/eval.go +++ b/internal/phraser/eval/eval.go @@ -281,7 +281,7 @@ func (r Report) String() string { fmt.Fprintf(&b, "%s: %d/%d cases pass every check (%.1f%%), %d errors\n", r.Name, r.Passed, r.Total, 100*r.Accuracy(), r.Errors) for _, name := range CheckNames { - fmt.Fprintf(&b, " %-9s %d/%d\n", name, r.ByCheck[name], r.Total) + fmt.Fprintf(&b, " %-10s %d/%d\n", name, r.ByCheck[name], r.Total) } fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max) fmt.Fprintf(&b, " by rule: %s\n", renderStats(r.ByRule)) diff --git a/internal/phraser/eval/eval_test.go b/internal/phraser/eval/eval_test.go index a3f3389..72932bd 100644 --- a/internal/phraser/eval/eval_test.go +++ b/internal/phraser/eval/eval_test.go @@ -72,10 +72,11 @@ func TestStubBaseline(t *testing.T) { // ceiling ("you've been at your desk for 4 hours without a break — step // away for a bit." is 76 chars but 16 words). Left failing rather than // raising the ceiling to hide it. - CheckLength: 12, - CheckFeminine: 15, - CheckCringe: 15, - CheckOnTopic: 12, + CheckLength: 12, + CheckFeminine: 15, + CheckHisGender: 15, + CheckCringe: 15, + CheckOnTopic: 12, } for name, floor := range floors { if rep.ByCheck[name] < floor { @@ -104,6 +105,14 @@ func TestChecksCatchWhatTheyClaim(t *testing.T) { {"masculine predicative", "я должен сказать: попей воды.", CheckFeminine}, // The other direction: HE is male, so second-person masculine is right. {"second person masculine ok", "ты не пил воду четыре часа.", ""}, + // The real observed failure: she addressed him as a woman. + {"feminine second person", "ты давно не отдыхала — попей воды.", CheckHisGender}, + {"feminine second person no dash", "ты пила воду четыре часа назад.", CheckHisGender}, + // Her own feminine verb next to "ты" is correct and must not be flagged. + {"her feminine verb near ты", "я заметила, что ты не пил воду.", ""}, + {"her feminine verb about him", "напомнила тебе про воду.", ""}, + // A feminine noun subject in the same sentence is not him. + {"feminine noun subject ok", "зарядка была утром, ты её пропустил, попей воды.", ""}, {"feminine self ok", "я заметила: воды не было четыре часа.", ""}, {"pet name", "милый, попей воды.", CheckCringe}, {"emoji", "попей воды 💧", CheckCringe},