Check that she doesn't address you as a woman (#381) #24
@@ -105,11 +105,14 @@ that never says food.
|
|||||||
|
|
||||||
## Broken, found, not fixed
|
## Broken, found, not fixed
|
||||||
|
|
||||||
1. **`checkFeminine` only catches half the constraint.** It scans for masculine
|
1. ~~**`checkFeminine` only catches half the constraint.**~~ **Fixed** (#381). It scanned for
|
||||||
self-reference and passed 15/15 both runs — but three messages address the *owner* in
|
masculine self-reference only, so three messages that addressed the *owner* in the feminine
|
||||||
the feminine: "ты давно не отдыхал**а**", "он не ел". The owner is a man. The check has
|
("ты давно не отдыхал**а**") scored clean. There is now a second check, `hisgender`: a
|
||||||
no second-person gender test, so this scores clean while being exactly the persona
|
feminine past-tense verb (-ла/-лась) in a sentence addressed to him ("ты", "тебе", "твой")
|
||||||
failure the constraint exists to prevent. This is the most important gap in the harness.
|
fails, unless the verb is hers ("я заметила", "напомнила тебе"). It is a suffix rule, not a
|
||||||
|
parser — see the comment in `checks.go` for what it misses. A fresh 15-case run after adding
|
||||||
|
it scored **12/15** with `hisgender` 15/15; the model did not repeat the feminine address in
|
||||||
|
that sample, and the check is pinned by unit tests on the recorded bad strings instead.
|
||||||
2. **Grammar is not checked at all, and it is bad.** `"Он не ел 11 дней"` (it was 11 hours),
|
2. **Grammar is not checked at all, and it is bad.** `"Он не ел 11 дней"` (it was 11 hours),
|
||||||
`"Сонуждились 7 дней"` (not a word), `"Они забыли воду"` (wrong person entirely). Every
|
`"Сонуждились 7 дней"` (not a word), `"Они забыли воду"` (wrong person entirely). Every
|
||||||
one of these passes all six checks. The fixture measures properties, not fluency, and at
|
one of these passes all six checks. The fixture measures properties, not fluency, and at
|
||||||
@@ -125,8 +128,7 @@ that never says food.
|
|||||||
|
|
||||||
## Next steps
|
## Next steps
|
||||||
|
|
||||||
1. **Add a second-person gender check** to `checks.go`. Finding 1 above. Until it exists the
|
1. ~~**Add a second-person gender check**~~ — done, `hisgender` in `checks.go` (#381).
|
||||||
feminine column means less than it looks like.
|
|
||||||
2. **Decide whether the fallback should count as a pass.** Right now `Score` cannot tell a
|
2. **Decide whether the fallback should count as a pass.** Right now `Score` cannot tell a
|
||||||
model answer from a fallback. Either mark fallback bodies in `PhrasedNudge` or count them
|
model answer from a fallback. Either mark fallback bodies in `PhrasedNudge` or count them
|
||||||
in their own column. Without that, any future prompt change can score well by failing
|
in their own column. Without that, any future prompt change can score well by failing
|
||||||
|
|||||||
@@ -17,10 +17,14 @@ const (
|
|||||||
CheckFeminine = "feminine" // her self-reference is feminine (hard constraint)
|
CheckFeminine = "feminine" // her self-reference is feminine (hard constraint)
|
||||||
CheckCringe = "cringe" // DESIGN.md § Non-goals, "not a relationship"
|
CheckCringe = "cringe" // DESIGN.md § Non-goals, "not a relationship"
|
||||||
CheckOnTopic = "ontopic" // says the thing the rule is about
|
CheckOnTopic = "ontopic" // says the thing the rule is about
|
||||||
|
|
||||||
|
// CheckHisGender — the other half of the persona rule: SHE is feminine, HE
|
||||||
|
// is male. "ты давно не отдыхала" addresses the operator as a woman.
|
||||||
|
CheckHisGender = "hisgender"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CheckNames — report order.
|
// CheckNames — report order.
|
||||||
var CheckNames = []string{CheckMood, CheckLang, CheckLength, CheckFeminine, CheckCringe, CheckOnTopic}
|
var CheckNames = []string{CheckMood, CheckLang, CheckLength, CheckFeminine, CheckHisGender, CheckCringe, CheckOnTopic}
|
||||||
|
|
||||||
// Result — one check on one message.
|
// Result — one check on one message.
|
||||||
type Result struct {
|
type Result struct {
|
||||||
@@ -52,6 +56,7 @@ func RunChecks(c Case, body, mood string) []Result {
|
|||||||
checkLang(body),
|
checkLang(body),
|
||||||
checkLength(body),
|
checkLength(body),
|
||||||
checkFeminine(body),
|
checkFeminine(body),
|
||||||
|
checkHisGender(body),
|
||||||
checkCringe(body),
|
checkCringe(body),
|
||||||
checkOnTopic(c, body),
|
checkOnTopic(c, body),
|
||||||
}
|
}
|
||||||
@@ -176,6 +181,127 @@ func checkFeminine(body string) Result {
|
|||||||
return Result{CheckFeminine, true, ""}
|
return Result{CheckFeminine, true, ""}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- he is male ----------------------------------------------------------
|
||||||
|
//
|
||||||
|
// The mirror of checkFeminine, and the failure it was written for: the model
|
||||||
|
// wrote "ты давно не отдыхала", which addresses the operator as a woman. That
|
||||||
|
// scored clean, because checkFeminine only ever looks at how SHE speaks about
|
||||||
|
// herself.
|
||||||
|
//
|
||||||
|
// How it works: Russian past tense is gendered by suffix, -л (m) / -ла (f). So
|
||||||
|
// this looks for feminine past-tense words in a sentence that also talks TO him
|
||||||
|
// ("ты", "тебя", "тебе", "твой", …). A feminine verb that belongs to her ("я
|
||||||
|
// заметила", "напомнила тебе") is skipped — that one is correct.
|
||||||
|
//
|
||||||
|
// Honest about the limits: this is a suffix rule, not a parser.
|
||||||
|
// - False positives: a feminine noun can be the subject in the same sentence
|
||||||
|
// ("зарядка была утром, ты её пропустил"). The guard below skips a verb whose
|
||||||
|
// previous word looks like a feminine noun, which helps but will not always
|
||||||
|
// be right.
|
||||||
|
// - False negatives: gender also shows up outside the past tense (short
|
||||||
|
// adjectives, "сама"), and none of that is checked here.
|
||||||
|
//
|
||||||
|
// That is acceptable for an eval check. It is a signal to read the message, not
|
||||||
|
// a grammar verdict, and every hit prints the word it tripped on so a human can
|
||||||
|
// disagree.
|
||||||
|
|
||||||
|
// hisMarkers — words that mean the sentence is addressed to him.
|
||||||
|
var hisMarkers = map[string]bool{
|
||||||
|
"ты": true, "тебя": true, "тебе": true, "тобой": true, "тобою": true,
|
||||||
|
"твой": true, "твоя": true, "твоё": true, "твое": true, "твои": true, "твою": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// notFeminineVerb — ordinary words ending in "-ла" that are not verbs. Small on
|
||||||
|
// purpose: it only has to cover words a nudge might actually use.
|
||||||
|
//
|
||||||
|
// Words that are both a noun and a verb are deliberately NOT here. "села",
|
||||||
|
// "мыла" and "стекла" are nouns on paper, but in a nudge they are almost always
|
||||||
|
// verbs ("ты села", "ты мыла"), and listing them would make the check miss the
|
||||||
|
// exact thing it is for. Missing a real hit is worse than one false alarm.
|
||||||
|
var notFeminineVerb = map[string]bool{
|
||||||
|
"школа": true, "скала": true, "игла": true, "метла": true, "смола": true,
|
||||||
|
"дела": true, "тела": true, "масла": true, "весла": true,
|
||||||
|
"зола": true, "пчела": true, "числа": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// femininePast reports whether a word looks like a feminine past-tense verb:
|
||||||
|
// "отдыхала", "поела", "выспалась".
|
||||||
|
func femininePast(w string) bool {
|
||||||
|
if len([]rune(w)) < 3 || notFeminineVerb[w] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.HasSuffix(w, "ла") || strings.HasSuffix(w, "лась")
|
||||||
|
}
|
||||||
|
|
||||||
|
// looksFeminineNoun — a crude guard against "зарядка была": a word right before
|
||||||
|
// the verb that ends in "а"/"я" and is not itself a verb is probably the subject.
|
||||||
|
func looksFeminineNoun(w string) bool {
|
||||||
|
if femininePast(w) || len([]rune(w)) < 3 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.HasSuffix(w, "а") || strings.HasSuffix(w, "я")
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentenceRE splits on sentence-ending punctuation, so a feminine verb in one
|
||||||
|
// sentence is not blamed on a "ты" in the next.
|
||||||
|
var sentenceRE = regexp.MustCompile(`[.!?;…]+`)
|
||||||
|
|
||||||
|
func checkHisGender(body string) Result {
|
||||||
|
for _, sentence := range sentenceRE.Split(strings.ToLower(body), -1) {
|
||||||
|
words := wordRE.FindAllString(sentence, -1)
|
||||||
|
addressed := false
|
||||||
|
for _, w := range words {
|
||||||
|
if hisMarkers[w] {
|
||||||
|
addressed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !addressed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i, w := range words {
|
||||||
|
if !femininePast(w) || hersNotHis(words, i) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if i > 0 && looksFeminineNoun(prevWord(words, i)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return Result{CheckHisGender, false,
|
||||||
|
fmt.Sprintf("feminine %q addressed to him — he is male", w)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Result{CheckHisGender, true, ""}
|
||||||
|
}
|
||||||
|
|
||||||
|
// hersNotHis — the verb is Maven's own if "я" comes shortly before it, or if the
|
||||||
|
// thing she did was done to him ("напомнила тебе", "проверила за тебя").
|
||||||
|
func hersNotHis(words []string, i int) bool {
|
||||||
|
for j := i - 1; j >= 0 && j >= i-3; j-- {
|
||||||
|
if words[j] == "я" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if i+1 < len(words) {
|
||||||
|
switch words[i+1] {
|
||||||
|
case "тебе", "тебя", "за", "тобой":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// prevWord — the word before i, skipping "не" and punctuation, so "не отдыхала"
|
||||||
|
// still sees the subject.
|
||||||
|
func prevWord(words []string, i int) string {
|
||||||
|
for j := i - 1; j >= 0; j-- {
|
||||||
|
w := words[j]
|
||||||
|
if w == "не" || w == "ни" || !unicode.Is(unicode.Cyrillic, []rune(w)[0]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// --- the cringe checks ---------------------------------------------------
|
// --- the cringe checks ---------------------------------------------------
|
||||||
//
|
//
|
||||||
// "Think Jarvis without the cringe part". DESIGN.md § Non-goals: "Not a
|
// "Think Jarvis without the cringe part". DESIGN.md § Non-goals: "Not a
|
||||||
|
|||||||
@@ -281,7 +281,7 @@ func (r Report) String() string {
|
|||||||
fmt.Fprintf(&b, "%s: %d/%d cases pass every check (%.1f%%), %d errors\n",
|
fmt.Fprintf(&b, "%s: %d/%d cases pass every check (%.1f%%), %d errors\n",
|
||||||
r.Name, r.Passed, r.Total, 100*r.Accuracy(), r.Errors)
|
r.Name, r.Passed, r.Total, 100*r.Accuracy(), r.Errors)
|
||||||
for _, name := range CheckNames {
|
for _, name := range CheckNames {
|
||||||
fmt.Fprintf(&b, " %-9s %d/%d\n", name, r.ByCheck[name], r.Total)
|
fmt.Fprintf(&b, " %-10s %d/%d\n", name, r.ByCheck[name], r.Total)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max)
|
fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max)
|
||||||
fmt.Fprintf(&b, " by rule: %s\n", renderStats(r.ByRule))
|
fmt.Fprintf(&b, " by rule: %s\n", renderStats(r.ByRule))
|
||||||
|
|||||||
@@ -72,10 +72,11 @@ func TestStubBaseline(t *testing.T) {
|
|||||||
// ceiling ("you've been at your desk for 4 hours without a break — step
|
// ceiling ("you've been at your desk for 4 hours without a break — step
|
||||||
// away for a bit." is 76 chars but 16 words). Left failing rather than
|
// away for a bit." is 76 chars but 16 words). Left failing rather than
|
||||||
// raising the ceiling to hide it.
|
// raising the ceiling to hide it.
|
||||||
CheckLength: 12,
|
CheckLength: 12,
|
||||||
CheckFeminine: 15,
|
CheckFeminine: 15,
|
||||||
CheckCringe: 15,
|
CheckHisGender: 15,
|
||||||
CheckOnTopic: 12,
|
CheckCringe: 15,
|
||||||
|
CheckOnTopic: 12,
|
||||||
}
|
}
|
||||||
for name, floor := range floors {
|
for name, floor := range floors {
|
||||||
if rep.ByCheck[name] < floor {
|
if rep.ByCheck[name] < floor {
|
||||||
@@ -104,6 +105,14 @@ func TestChecksCatchWhatTheyClaim(t *testing.T) {
|
|||||||
{"masculine predicative", "я должен сказать: попей воды.", CheckFeminine},
|
{"masculine predicative", "я должен сказать: попей воды.", CheckFeminine},
|
||||||
// The other direction: HE is male, so second-person masculine is right.
|
// The other direction: HE is male, so second-person masculine is right.
|
||||||
{"second person masculine ok", "ты не пил воду четыре часа.", ""},
|
{"second person masculine ok", "ты не пил воду четыре часа.", ""},
|
||||||
|
// The real observed failure: she addressed him as a woman.
|
||||||
|
{"feminine second person", "ты давно не отдыхала — попей воды.", CheckHisGender},
|
||||||
|
{"feminine second person no dash", "ты пила воду четыре часа назад.", CheckHisGender},
|
||||||
|
// Her own feminine verb next to "ты" is correct and must not be flagged.
|
||||||
|
{"her feminine verb near ты", "я заметила, что ты не пил воду.", ""},
|
||||||
|
{"her feminine verb about him", "напомнила тебе про воду.", ""},
|
||||||
|
// A feminine noun subject in the same sentence is not him.
|
||||||
|
{"feminine noun subject ok", "зарядка была утром, ты её пропустил, попей воды.", ""},
|
||||||
{"feminine self ok", "я заметила: воды не было четыре часа.", ""},
|
{"feminine self ok", "я заметила: воды не было четыре часа.", ""},
|
||||||
{"pet name", "милый, попей воды.", CheckCringe},
|
{"pet name", "милый, попей воды.", CheckCringe},
|
||||||
{"emoji", "попей воды 💧", CheckCringe},
|
{"emoji", "попей воды 💧", CheckCringe},
|
||||||
|
|||||||
Reference in New Issue
Block a user