Merge the second-person gender check

This commit is contained in:
kami
2026-07-31 12:54:48 +04:00
4 changed files with 145 additions and 13 deletions
+9 -7
View File
@@ -105,11 +105,14 @@ that never says food.
## Broken, found, not fixed
1. **`checkFeminine` only catches half the constraint.** It scans for masculine
self-reference and passed 15/15 both runs — but three messages address the *owner* in
the feminine: "ты давно не отдыхал**а**", "он не ел". The owner is a man. The check has
no second-person gender test, so this scores clean while being exactly the persona
failure the constraint exists to prevent. This is the most important gap in the harness.
1. ~~**`checkFeminine` only catches half the constraint.**~~ **Fixed** (#381). It scanned for
masculine self-reference only, so three messages that addressed the *owner* in the feminine
("ты давно не отдыхал**а**") scored clean. There is now a second check, `hisgender`: a
feminine past-tense verb (-ла/-лась) in a sentence addressed to him ("ты", "тебе", "твой")
fails, unless the verb is hers ("я заметила", "напомнила тебе"). It is a suffix rule, not a
parser — see the comment in `checks.go` for what it misses. A fresh 15-case run after adding
it scored **12/15** with `hisgender` 15/15; the model did not repeat the feminine address in
that sample, and the check is pinned by unit tests on the recorded bad strings instead.
2. **Grammar is not checked at all, and it is bad.** `"Он не ел 11 дней"` (it was 11 hours),
`"Сонуждились 7 дней"` (not a word), `"Они забыли воду"` (wrong person entirely). Every
one of these passes all six checks. The fixture measures properties, not fluency, and at
@@ -125,8 +128,7 @@ that never says food.
## Next steps
1. **Add a second-person gender check** to `checks.go`. Finding 1 above. Until it exists the
feminine column means less than it looks like.
1. ~~**Add a second-person gender check**~~ — done, `hisgender` in `checks.go` (#381).
2. **Decide whether the fallback should count as a pass.** Right now `Score` cannot tell a
model answer from a fallback. Either mark fallback bodies in `PhrasedNudge` or count them
in their own column. Without that, any future prompt change can score well by failing
+122 -1
View File
@@ -17,10 +17,14 @@ const (
CheckFeminine = "feminine" // her self-reference is feminine (hard constraint)
CheckCringe = "cringe" // DESIGN.md § Non-goals, "not a relationship"
CheckOnTopic = "ontopic" // says the thing the rule is about
// CheckHisGender — the other half of the persona rule: SHE is feminine, HE
// is male. "ты давно не отдыхала" addresses the operator as a woman.
CheckHisGender = "hisgender"
)
// CheckNames — report order.
var CheckNames = []string{CheckMood, CheckLang, CheckLength, CheckFeminine, CheckCringe, CheckOnTopic}
var CheckNames = []string{CheckMood, CheckLang, CheckLength, CheckFeminine, CheckHisGender, CheckCringe, CheckOnTopic}
// Result — one check on one message.
type Result struct {
@@ -52,6 +56,7 @@ func RunChecks(c Case, body, mood string) []Result {
checkLang(body),
checkLength(body),
checkFeminine(body),
checkHisGender(body),
checkCringe(body),
checkOnTopic(c, body),
}
@@ -176,6 +181,122 @@ func checkFeminine(body string) Result {
return Result{CheckFeminine, true, ""}
}
// --- he is male ----------------------------------------------------------
//
// The mirror of checkFeminine, and the failure it was written for: the model
// wrote "ты давно не отдыхала", which addresses the operator as a woman. That
// scored clean, because checkFeminine only ever looks at how SHE speaks about
// herself.
//
// How it works: Russian past tense is gendered by suffix, -л (m) / -ла (f). So
// this looks for feminine past-tense words in a sentence that also talks TO him
// ("ты", "тебя", "тебе", "твой", …). A feminine verb that belongs to her ("я
// заметила", "напомнила тебе") is skipped — that one is correct.
//
// Honest about the limits: this is a suffix rule, not a parser.
// - False positives: a feminine noun can be the subject in the same sentence
// ("зарядка была утром, ты её пропустил"). The guard below skips a verb whose
// previous word looks like a feminine noun, which helps but will not always
// be right.
// - False negatives: gender also shows up outside the past tense (short
// adjectives, "сама"), and none of that is checked here.
//
// That is acceptable for an eval check. It is a signal to read the message, not
// a grammar verdict, and every hit prints the word it tripped on so a human can
// disagree.
// hisMarkers — words that mean the sentence is addressed to him.
var hisMarkers = map[string]bool{
"ты": true, "тебя": true, "тебе": true, "тобой": true, "тобою": true,
"твой": true, "твоя": true, "твоё": true, "твое": true, "твои": true, "твою": true,
}
// notFeminineVerb — ordinary words ending in "-ла" that are not verbs. Small on
// purpose: it only has to cover words a nudge might actually use.
var notFeminineVerb = map[string]bool{
"школа": true, "скала": true, "игла": true, "метла": true, "смола": true,
"дела": true, "тела": true, "масла": true, "стекла": true, "весла": true,
"зола": true, "пчела": true, "числа": true, "села": true, "мыла": true,
}
// femininePast reports whether a word looks like a feminine past-tense verb:
// "отдыхала", "поела", "выспалась".
func femininePast(w string) bool {
if len([]rune(w)) < 3 || notFeminineVerb[w] {
return false
}
return strings.HasSuffix(w, "ла") || strings.HasSuffix(w, "лась")
}
// looksFeminineNoun — a crude guard against "зарядка была": a word right before
// the verb that ends in "а"/"я" and is not itself a verb is probably the subject.
func looksFeminineNoun(w string) bool {
if femininePast(w) || len([]rune(w)) < 3 {
return false
}
return strings.HasSuffix(w, "а") || strings.HasSuffix(w, "я")
}
// sentenceRE splits on sentence-ending punctuation, so a feminine verb in one
// sentence is not blamed on a "ты" in the next.
var sentenceRE = regexp.MustCompile(`[.!?;…]+`)
func checkHisGender(body string) Result {
for _, sentence := range sentenceRE.Split(strings.ToLower(body), -1) {
words := wordRE.FindAllString(sentence, -1)
addressed := false
for _, w := range words {
if hisMarkers[w] {
addressed = true
}
}
if !addressed {
continue
}
for i, w := range words {
if !femininePast(w) || hersNotHis(words, i) {
continue
}
if i > 0 && looksFeminineNoun(prevWord(words, i)) {
continue
}
return Result{CheckHisGender, false,
fmt.Sprintf("feminine %q addressed to him — he is male", w)}
}
}
return Result{CheckHisGender, true, ""}
}
// hersNotHis — the verb is Maven's own if "я" comes shortly before it, or if the
// thing she did was done to him ("напомнила тебе", "проверила за тебя").
func hersNotHis(words []string, i int) bool {
for j := i - 1; j >= 0 && j >= i-3; j-- {
if words[j] == "я" {
return true
}
}
if i+1 < len(words) {
switch words[i+1] {
case "тебе", "тебя", "за", "тобой":
return true
}
}
return false
}
// prevWord — the word before i, skipping "не" and punctuation, so "не отдыхала"
// still sees the subject.
func prevWord(words []string, i int) string {
for j := i - 1; j >= 0; j-- {
w := words[j]
if w == "не" || w == "ни" || !unicode.Is(unicode.Cyrillic, []rune(w)[0]) {
continue
}
return w
}
return ""
}
// --- the cringe checks ---------------------------------------------------
//
// "Think Jarvis without the cringe part". DESIGN.md § Non-goals: "Not a
+1 -1
View File
@@ -281,7 +281,7 @@ func (r Report) String() string {
fmt.Fprintf(&b, "%s: %d/%d cases pass every check (%.1f%%), %d errors\n",
r.Name, r.Passed, r.Total, 100*r.Accuracy(), r.Errors)
for _, name := range CheckNames {
fmt.Fprintf(&b, " %-9s %d/%d\n", name, r.ByCheck[name], r.Total)
fmt.Fprintf(&b, " %-10s %d/%d\n", name, r.ByCheck[name], r.Total)
}
fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max)
fmt.Fprintf(&b, " by rule: %s\n", renderStats(r.ByRule))
+13 -4
View File
@@ -72,10 +72,11 @@ func TestStubBaseline(t *testing.T) {
// ceiling ("you've been at your desk for 4 hours without a break — step
// away for a bit." is 76 chars but 16 words). Left failing rather than
// raising the ceiling to hide it.
CheckLength: 12,
CheckFeminine: 15,
CheckCringe: 15,
CheckOnTopic: 12,
CheckLength: 12,
CheckFeminine: 15,
CheckHisGender: 15,
CheckCringe: 15,
CheckOnTopic: 12,
}
for name, floor := range floors {
if rep.ByCheck[name] < floor {
@@ -104,6 +105,14 @@ func TestChecksCatchWhatTheyClaim(t *testing.T) {
{"masculine predicative", "я должен сказать: попей воды.", CheckFeminine},
// The other direction: HE is male, so second-person masculine is right.
{"second person masculine ok", "ты не пил воду четыре часа.", ""},
// The real observed failure: she addressed him as a woman.
{"feminine second person", "ты давно не отдыхала — попей воды.", CheckHisGender},
{"feminine second person no dash", "ты пила воду четыре часа назад.", CheckHisGender},
// Her own feminine verb next to "ты" is correct and must not be flagged.
{"her feminine verb near ты", "я заметила, что ты не пил воду.", ""},
{"her feminine verb about him", "напомнила тебе про воду.", ""},
// A feminine noun subject in the same sentence is not him.
{"feminine noun subject ok", "зарядка была утром, ты её пропустил, попей воды.", ""},
{"feminine self ok", "я заметила: воды не было четыре часа.", ""},
{"pet name", "милый, попей воды.", CheckCringe},
{"emoji", "попей воды 💧", CheckCringe},