kami
414 lines
16 KiB
Go
414 lines
16 KiB
Go
package eval
|
||
|
||
import (
|
||
"fmt"
|
||
"regexp"
|
||
"strings"
|
||
"unicode"
|
||
"unicode/utf8"
|
||
)
|
||
|
||
// The check names, in report order. Every check is a string or length test — no
|
||
// model grades another model here.
|
||
const (
|
||
CheckMood = "mood" // mood is in the documented enum
|
||
CheckLang = "lang" // the operator's language, not the prompt's
|
||
CheckLength = "length" // a nudge is one sentence, not a paragraph
|
||
CheckFeminine = "feminine" // her self-reference is feminine (hard constraint)
|
||
CheckCringe = "cringe" // DESIGN.md § Non-goals, "not a relationship"
|
||
CheckOnTopic = "ontopic" // says the thing the rule is about
|
||
|
||
// CheckHisGender — the other half of the persona rule: SHE is feminine, HE
|
||
// is male. "ты давно не отдыхала" addresses the operator as a woman.
|
||
CheckHisGender = "hisgender"
|
||
)
|
||
|
||
// CheckNames — report order.
|
||
var CheckNames = []string{CheckMood, CheckLang, CheckLength, CheckFeminine, CheckHisGender, CheckCringe, CheckOnTopic}
|
||
|
||
// Result — one check on one message.
|
||
type Result struct {
|
||
Name string
|
||
Pass bool
|
||
Detail string
|
||
}
|
||
|
||
// Moods — the fixed enum from the LLM output contract. Not extended here; the
|
||
// contract lives in the daemon and the harness only reads it.
|
||
var Moods = map[string]bool{
|
||
"neutral": true, "happy": true, "thinking": true, "tired": true, "confused": true,
|
||
}
|
||
|
||
// Length ceilings. Justification: the nudge is spoken by piper at roughly 14
|
||
// characters per second, so 120 characters is about 8 seconds of speech. The
|
||
// operator has AuDHD — past one short sentence a nudge stops being a nudge and
|
||
// becomes something to tune out, which is exactly the "not a nag" failure. The
|
||
// word ceiling catches the same thing for languages that pack more per byte.
|
||
const (
|
||
MaxChars = 120
|
||
MaxWords = 16
|
||
)
|
||
|
||
// RunChecks scores one message. Order matches CheckNames.
|
||
func RunChecks(c Case, body, mood string) []Result {
|
||
return []Result{
|
||
checkMood(mood),
|
||
checkLang(body),
|
||
checkLength(body),
|
||
checkFeminine(body),
|
||
checkHisGender(body),
|
||
checkCringe(body),
|
||
checkOnTopic(c, body),
|
||
}
|
||
}
|
||
|
||
func checkMood(mood string) Result {
|
||
if Moods[mood] {
|
||
return Result{CheckMood, true, ""}
|
||
}
|
||
return Result{CheckMood, false, fmt.Sprintf("mood %q not in the enum", mood)}
|
||
}
|
||
|
||
// checkLang — the operator is Russian-speaking and the nudge is spoken aloud by
|
||
// a Russian piper voice. An English nudge is not a tone problem, it is an
|
||
// unusable one.
|
||
func checkLang(body string) Result {
|
||
cyr, lat := 0, 0
|
||
for _, r := range body {
|
||
switch {
|
||
case unicode.Is(unicode.Cyrillic, r):
|
||
cyr++
|
||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z':
|
||
lat++
|
||
}
|
||
}
|
||
if cyr > lat {
|
||
return Result{CheckLang, true, ""}
|
||
}
|
||
return Result{CheckLang, false, fmt.Sprintf("not Russian (%d cyrillic vs %d latin letters)", cyr, lat)}
|
||
}
|
||
|
||
func checkLength(body string) Result {
|
||
chars := utf8.RuneCountInString(body)
|
||
words := len(strings.Fields(body))
|
||
if chars <= MaxChars && words <= MaxWords {
|
||
return Result{CheckLength, true, ""}
|
||
}
|
||
return Result{CheckLength, false,
|
||
fmt.Sprintf("%d chars / %d words, ceiling %d / %d", chars, words, MaxChars, MaxWords)}
|
||
}
|
||
|
||
// --- feminine self-reference ---------------------------------------------
|
||
//
|
||
// The hard constraint (CLAUDE.md, DESIGN.md § Identity): Maven's Russian
|
||
// self-reference is feminine. The operator is male, so second-person forms
|
||
// addressed to him are MASCULINE and must not be flagged — "ты не пил воду" is
|
||
// correct, "я напомнил" is not. Both directions matter, which is why this is a
|
||
// windowed scan around "я" and not a bare search for masculine endings.
|
||
|
||
var wordRE = regexp.MustCompile(`[\p{Cyrillic}]+|[,.;:!?…—-]`)
|
||
|
||
// secondPerson — pronouns that end the self-reference window. Everything after
|
||
// one of these is about him, not about her.
|
||
var secondPerson = map[string]bool{
|
||
"ты": true, "тебе": true, "тебя": true, "тобой": true,
|
||
"вы": true, "вам": true, "вас": true,
|
||
"он": true, "она": true, "оно": true, "они": true,
|
||
}
|
||
|
||
// masculinePredicative — short adjectives with no verb ending to key off.
|
||
var masculinePredicative = map[string]bool{
|
||
"должен": true, "готов": true, "рад": true, "уверен": true,
|
||
"обязан": true, "сам": true, "занят": true, "прав": true,
|
||
}
|
||
|
||
// nounsEndingInL — the false positives of "ends in л ⇒ masculine past tense".
|
||
// Small on purpose: it only has to cover nouns a nudge might actually use.
|
||
var nounsEndingInL = map[string]bool{
|
||
"стол": true, "стул": true, "пол": true, "зал": true, "гол": true,
|
||
"узел": true, "отдел": true, "файл": true, "канал": true, "угол": true,
|
||
"футбол": true, "вокзал": true, "металл": true, "интервал": true,
|
||
"уровень": true, "мускул": true, "апрель": true, "июль": true, "рубль": true,
|
||
}
|
||
|
||
// masculinePast reports whether a word looks like a masculine past-tense verb.
|
||
// Russian past tense is gendered by suffix: -л (m), -ла (f). A 0.8B with weak
|
||
// Russian defaults to the masculine form, which is the exact drift being
|
||
// measured.
|
||
func masculinePast(w string) bool {
|
||
if len([]rune(w)) < 3 || nounsEndingInL[w] {
|
||
return false
|
||
}
|
||
return strings.HasSuffix(w, "л") || strings.HasSuffix(w, "лся")
|
||
}
|
||
|
||
func checkFeminine(body string) Result {
|
||
words := wordRE.FindAllString(strings.ToLower(body), -1)
|
||
for i, w := range words {
|
||
if w != "я" {
|
||
continue
|
||
}
|
||
// Scan the next few words. Stop at punctuation or at a second-person
|
||
// pronoun: past that point the sentence is about him and masculine is
|
||
// correct.
|
||
for j := i + 1; j < len(words) && j <= i+3; j++ {
|
||
nw := words[j]
|
||
if len(nw) == 1 && !unicode.Is(unicode.Cyrillic, []rune(nw)[0]) {
|
||
break
|
||
}
|
||
if secondPerson[nw] {
|
||
break
|
||
}
|
||
if masculinePast(nw) || masculinePredicative[nw] {
|
||
return Result{CheckFeminine, false,
|
||
fmt.Sprintf("masculine self-reference %q after \"я\"", nw)}
|
||
}
|
||
}
|
||
}
|
||
// Second pass: self-reference with the pronoun dropped — "напомнил тебе",
|
||
// "проверил за тебя". A masculine past-tense verb whose object is HIM can
|
||
// only be her speaking about herself.
|
||
for i, w := range words {
|
||
if !masculinePast(w) || i+1 >= len(words) {
|
||
continue
|
||
}
|
||
next := words[i+1]
|
||
if next == "тебе" || next == "тебя" || next == "за" {
|
||
return Result{CheckFeminine, false,
|
||
fmt.Sprintf("masculine self-reference %q before %q", w, next)}
|
||
}
|
||
}
|
||
return Result{CheckFeminine, true, ""}
|
||
}
|
||
|
||
// --- he is male ----------------------------------------------------------
|
||
//
|
||
// The mirror of checkFeminine, and the failure it was written for: the model
|
||
// wrote "ты давно не отдыхала", which addresses the operator as a woman. That
|
||
// scored clean, because checkFeminine only ever looks at how SHE speaks about
|
||
// herself.
|
||
//
|
||
// How it works: Russian past tense is gendered by suffix, -л (m) / -ла (f). So
|
||
// this looks for feminine past-tense words in a sentence that also talks TO him
|
||
// ("ты", "тебя", "тебе", "твой", …). A feminine verb that belongs to her ("я
|
||
// заметила", "напомнила тебе") is skipped — that one is correct.
|
||
//
|
||
// Honest about the limits: this is a suffix rule, not a parser.
|
||
// - False positives: a feminine noun can be the subject in the same sentence
|
||
// ("зарядка была утром, ты её пропустил"). The guard below skips a verb whose
|
||
// previous word looks like a feminine noun, which helps but will not always
|
||
// be right.
|
||
// - False negatives: gender also shows up outside the past tense (short
|
||
// adjectives, "сама"), and none of that is checked here.
|
||
//
|
||
// That is acceptable for an eval check. It is a signal to read the message, not
|
||
// a grammar verdict, and every hit prints the word it tripped on so a human can
|
||
// disagree.
|
||
|
||
// hisMarkers — words that mean the sentence is addressed to him.
|
||
var hisMarkers = map[string]bool{
|
||
"ты": true, "тебя": true, "тебе": true, "тобой": true, "тобою": true,
|
||
"твой": true, "твоя": true, "твоё": true, "твое": true, "твои": true, "твою": true,
|
||
}
|
||
|
||
// notFeminineVerb — ordinary words ending in "-ла" that are not verbs. Small on
|
||
// purpose: it only has to cover words a nudge might actually use.
|
||
//
|
||
// Words that are both a noun and a verb are deliberately NOT here. "села",
|
||
// "мыла" and "стекла" are nouns on paper, but in a nudge they are almost always
|
||
// verbs ("ты села", "ты мыла"), and listing them would make the check miss the
|
||
// exact thing it is for. Missing a real hit is worse than one false alarm.
|
||
var notFeminineVerb = map[string]bool{
|
||
"школа": true, "скала": true, "игла": true, "метла": true, "смола": true,
|
||
"дела": true, "тела": true, "масла": true, "весла": true,
|
||
"зола": true, "пчела": true, "числа": true,
|
||
}
|
||
|
||
// femininePast reports whether a word looks like a feminine past-tense verb:
|
||
// "отдыхала", "поела", "выспалась".
|
||
func femininePast(w string) bool {
|
||
if len([]rune(w)) < 3 || notFeminineVerb[w] {
|
||
return false
|
||
}
|
||
return strings.HasSuffix(w, "ла") || strings.HasSuffix(w, "лась")
|
||
}
|
||
|
||
// looksFeminineNoun — a crude guard against "зарядка была": a word right before
|
||
// the verb that ends in "а"/"я" and is not itself a verb is probably the subject.
|
||
func looksFeminineNoun(w string) bool {
|
||
if femininePast(w) || len([]rune(w)) < 3 {
|
||
return false
|
||
}
|
||
return strings.HasSuffix(w, "а") || strings.HasSuffix(w, "я")
|
||
}
|
||
|
||
// sentenceRE splits on sentence-ending punctuation, so a feminine verb in one
|
||
// sentence is not blamed on a "ты" in the next.
|
||
var sentenceRE = regexp.MustCompile(`[.!?;…]+`)
|
||
|
||
func checkHisGender(body string) Result {
|
||
for _, sentence := range sentenceRE.Split(strings.ToLower(body), -1) {
|
||
words := wordRE.FindAllString(sentence, -1)
|
||
addressed := false
|
||
for _, w := range words {
|
||
if hisMarkers[w] {
|
||
addressed = true
|
||
}
|
||
}
|
||
if !addressed {
|
||
continue
|
||
}
|
||
for i, w := range words {
|
||
if !femininePast(w) || hersNotHis(words, i) {
|
||
continue
|
||
}
|
||
if i > 0 && looksFeminineNoun(prevWord(words, i)) {
|
||
continue
|
||
}
|
||
return Result{CheckHisGender, false,
|
||
fmt.Sprintf("feminine %q addressed to him — he is male", w)}
|
||
}
|
||
}
|
||
return Result{CheckHisGender, true, ""}
|
||
}
|
||
|
||
// hersNotHis — the verb is Maven's own if "я" comes shortly before it, or if the
|
||
// thing she did was done to him ("напомнила тебе", "проверила за тебя").
|
||
func hersNotHis(words []string, i int) bool {
|
||
for j := i - 1; j >= 0 && j >= i-3; j-- {
|
||
if words[j] == "я" {
|
||
return true
|
||
}
|
||
}
|
||
if i+1 < len(words) {
|
||
switch words[i+1] {
|
||
case "тебе", "тебя", "за", "тобой":
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// prevWord — the word before i, skipping "не" and punctuation, so "не отдыхала"
|
||
// still sees the subject.
|
||
func prevWord(words []string, i int) string {
|
||
for j := i - 1; j >= 0; j-- {
|
||
w := words[j]
|
||
if w == "не" || w == "ни" || !unicode.Is(unicode.Cyrillic, []rune(w)[0]) {
|
||
continue
|
||
}
|
||
return w
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// --- the cringe checks ---------------------------------------------------
|
||
//
|
||
// "Think Jarvis without the cringe part". DESIGN.md § Non-goals: "Not a
|
||
// relationship — mom-tone is a function that makes nudges land, not emotional
|
||
// company. Names the drift a warm small model falls into." Each pattern below
|
||
// is one shape of that drift. They are deliberately specific: a check that
|
||
// flags any warmth at all would make the nudges robotic, which is the other
|
||
// failure.
|
||
|
||
type cringePattern struct {
|
||
// what the pattern is defending against, shown in the failure detail.
|
||
why string
|
||
pat *regexp.Regexp
|
||
}
|
||
|
||
var cringePatterns = []cringePattern{
|
||
{
|
||
// Endearments. "Not a relationship" — a pet name reframes a nudge as
|
||
// intimacy, and the operator asked for mother-like, not girlfriend-like.
|
||
why: "pet name / endearment",
|
||
// No \b around the Russian alternatives: Go's RE2 \b is ASCII-only and
|
||
// never matches at a Cyrillic boundary, so anchoring them would make
|
||
// this check silently always pass.
|
||
pat: regexp.MustCompile(`(?i)(милый|дорогой|солнышко|солнце моё|солнце мое|зайчик|котик|сладкий|малыш|дружок|родной|любимый|\bhoney\b|\bsweetie\b|\bdarling\b|\bbuddy\b)`),
|
||
},
|
||
{
|
||
// Emoji. The nudge is spoken aloud; an emoji is either silence or a TTS
|
||
// artefact. Also the single loudest cringe signal in a small model.
|
||
why: "emoji",
|
||
pat: nil, // handled by hasEmoji, ranges don't fit a regexp cleanly
|
||
},
|
||
{
|
||
// Exclamation pileup. One "!" is emphasis; two is a cheerleader.
|
||
why: "more than one exclamation mark",
|
||
pat: regexp.MustCompile(`!.*!|!!`),
|
||
},
|
||
{
|
||
// Fake concern. She has no feelings to report, and reporting them makes
|
||
// the nudge about her instead of about the water.
|
||
why: "fake concern opener",
|
||
pat: regexp.MustCompile(`(?i)(я волну|я беспоко|беспокоюсь|переживаю|я забочусь|я тревож|мне тревожно|i'?m worried)`),
|
||
},
|
||
{
|
||
// Apologising. The rule decided she speaks. Apologising for a greenlit
|
||
// nudge undermines the one thing that makes nudges land.
|
||
why: "apology",
|
||
pat: regexp.MustCompile(`(?i)(извини|прости|сожалею|прошу прощения|не хочу мешать|не хочу отвлекать|sorry|apolog)`),
|
||
},
|
||
{
|
||
// Offering emotional support. The explicit "not emotional company" line.
|
||
why: "offer of emotional support",
|
||
pat: regexp.MustCompile(`(?i)(я рядом|я здесь для теб|ты не один|всё будет хорошо|все будет хорошо|не переживай|я с тобой|обнимаю|я поддерж|держись)`),
|
||
},
|
||
{
|
||
// Asking how he feels. Turns a one-way nudge into a conversation he now
|
||
// owes an answer to — the most reliable way to make him mute it.
|
||
why: "asking how he feels",
|
||
pat: regexp.MustCompile(`(?i)(как ты\s*[?.!]|как ты себя|как самочувств|как настроение|всё ли в порядке|все ли в порядке|ты в порядке|how are you)`),
|
||
},
|
||
{
|
||
// Praise for compliance. Rewards make the nudge a training exercise;
|
||
// "not a relationship" again, from the other side.
|
||
why: "praise / reward framing",
|
||
pat: regexp.MustCompile(`(?i)(молодец|умница|ты справ|гордюсь|горжусь|отличная работа|так держать|good job|proud of you)`),
|
||
},
|
||
}
|
||
|
||
// hasEmoji — the pictographic ranges plus the variation selector. Cyrillic and
|
||
// ordinary punctuation are far below all of these.
|
||
func hasEmoji(s string) bool {
|
||
for _, r := range s {
|
||
switch {
|
||
case r >= 0x1F000 && r <= 0x1FAFF,
|
||
r >= 0x2600 && r <= 0x27BF,
|
||
r >= 0x2B00 && r <= 0x2BFF,
|
||
r == 0xFE0F, r == 0x203C, r == 0x2049:
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func checkCringe(body string) Result {
|
||
for _, c := range cringePatterns {
|
||
if c.pat == nil {
|
||
if hasEmoji(body) {
|
||
return Result{CheckCringe, false, c.why}
|
||
}
|
||
continue
|
||
}
|
||
if m := c.pat.FindString(body); m != "" {
|
||
return Result{CheckCringe, false, fmt.Sprintf("%s (%q)", c.why, m)}
|
||
}
|
||
}
|
||
return Result{CheckCringe, true, ""}
|
||
}
|
||
|
||
// checkOnTopic — the message must name the thing the rule is about. A nudge
|
||
// that never mentions water leaves the operator with a chime and no action.
|
||
func checkOnTopic(c Case, body string) Result {
|
||
low := strings.ToLower(body)
|
||
for _, want := range c.WantAny {
|
||
if strings.Contains(low, strings.ToLower(want)) {
|
||
return Result{CheckOnTopic, true, ""}
|
||
}
|
||
}
|
||
return Result{CheckOnTopic, false,
|
||
fmt.Sprintf("mentions none of %v", c.WantAny)}
|
||
}
|