kami
aa8f5b2ee2
It scored 27/27 on a run where two replies were "{" and "{\n \"". It only
tested that the string was not blank, so punctuation counted as content and
the worst replies of the run passed the first check.
Now a reply needs at least one letter, Cyrillic or Latin. Latin counts
because answers about ssd or vpn are legitimately part English.
Digits alone fail too. The same run answered "сколько варить яйцо
вкрутую?" with "15-16" — no unit, no words, and the wrong number as well.
That is not something she said.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
66 lines
1.9 KiB
Go
66 lines
1.9 KiB
Go
package eval
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// TestAddressReportsEveryBreak — the real reply from a nudge eval run broke in
|
||
// two ways at once and the check named only the plural. Both must print: a
|
||
// half-reported failure reads as a milder problem than it is.
|
||
func TestAddressReportsEveryBreak(t *testing.T) {
|
||
body := "Смотрите на его потребление воды."
|
||
res := checkAddress(body)
|
||
if res.Pass {
|
||
t.Fatalf("checkAddress passed %q", body)
|
||
}
|
||
for _, want := range []string{"смотрите", "его"} {
|
||
if !strings.Contains(res.Detail, want) {
|
||
t.Errorf("detail %q does not name %q", res.Detail, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// One word repeated is one problem, so the detail must not say it twice.
|
||
func TestAddressDeduplicates(t *testing.T) {
|
||
res := checkAddress("Вам стоит поесть, вам это нужно.")
|
||
if res.Pass {
|
||
t.Fatal("expected failure")
|
||
}
|
||
if n := strings.Count(res.Detail, "formal"); n != 1 {
|
||
t.Errorf("detail repeats the same break %d times: %q", n, res.Detail)
|
||
}
|
||
}
|
||
|
||
// The fragments a real run produced. All of them scored as non-empty replies
|
||
// before checkNonEmpty looked for letters.
|
||
func TestNonEmptyNeedsLetters(t *testing.T) {
|
||
for _, body := range []string{
|
||
"{",
|
||
"{\n \"",
|
||
"15-16",
|
||
`{"`,
|
||
" ",
|
||
"...",
|
||
} {
|
||
if got := checkNonEmpty(body); got.Pass {
|
||
t.Errorf("checkNonEmpty(%q) passed — that is not a reply", body)
|
||
}
|
||
}
|
||
}
|
||
|
||
// And it must not start failing real replies. Latin counts as well as Cyrillic:
|
||
// answers about ssd or vpn are legitimately part English.
|
||
func TestNonEmptyAcceptsRealReplies(t *testing.T) {
|
||
for _, body := range []string{
|
||
"норм, а ты как?",
|
||
"вот что я нашла: ключ у соседа",
|
||
"ssd быстрее hdd.",
|
||
"9 минут.",
|
||
} {
|
||
if got := checkNonEmpty(body); !got.Pass {
|
||
t.Errorf("checkNonEmpty(%q) failed: %s", body, got.Detail)
|
||
}
|
||
}
|
||
}
|