Make the nonempty check look for actual words
It scored 27/27 on a run where two replies were "{" and "{\n \"". It only
tested that the string was not blank, so punctuation counted as content and
the worst replies of the run passed the first check.
Now a reply needs at least one letter, Cyrillic or Latin. Latin counts
because answers about ssd or vpn are legitimately part English.
Digits alone fail too. The same run answered "сколько варить яйцо
вкрутую?" with "15-16" — no unit, no words, and the wrong number as well.
That is not something she said.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
This commit is contained in:
@@ -31,3 +31,35 @@ func TestAddressDeduplicates(t *testing.T) {
|
||||
t.Errorf("detail repeats the same break %d times: %q", n, res.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// The fragments a real run produced. All of them scored as non-empty replies
|
||||
// before checkNonEmpty looked for letters.
|
||||
func TestNonEmptyNeedsLetters(t *testing.T) {
|
||||
for _, body := range []string{
|
||||
"{",
|
||||
"{\n \"",
|
||||
"15-16",
|
||||
`{"`,
|
||||
" ",
|
||||
"...",
|
||||
} {
|
||||
if got := checkNonEmpty(body); got.Pass {
|
||||
t.Errorf("checkNonEmpty(%q) passed — that is not a reply", body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And it must not start failing real replies. Latin counts as well as Cyrillic:
|
||||
// answers about ssd or vpn are legitimately part English.
|
||||
func TestNonEmptyAcceptsRealReplies(t *testing.T) {
|
||||
for _, body := range []string{
|
||||
"норм, а ты как?",
|
||||
"вот что я нашла: ключ у соседа",
|
||||
"ssd быстрее hdd.",
|
||||
"9 минут.",
|
||||
} {
|
||||
if got := checkNonEmpty(body); !got.Pass {
|
||||
t.Errorf("checkNonEmpty(%q) failed: %s", body, got.Detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -623,11 +623,24 @@ const (
|
||||
CheckEllipsis = "ellipsis" // she finished the sentence
|
||||
)
|
||||
|
||||
// A reply needs words in it, not just characters. This check used to test for a
|
||||
// non-empty string, which scored 27/27 on a run where two replies were "{" and
|
||||
// "{\n \"" — punctuation passed as content. Braces, quotes, digits and spaces
|
||||
// are all empty in the only sense that matters.
|
||||
//
|
||||
// Digits alone fail too, and that is deliberate: the same run answered "сколько
|
||||
// варить яйцо вкрутую?" with "15-16". No unit, no words, and it is also the
|
||||
// wrong number. Whatever that is, it is not something she said.
|
||||
func checkNonEmpty(body string) Result {
|
||||
if strings.TrimSpace(body) == "" {
|
||||
return Result{CheckNonEmpty, false, "empty reply"}
|
||||
}
|
||||
return Result{CheckNonEmpty, true, ""}
|
||||
for _, r := range body {
|
||||
if unicode.IsLetter(r) {
|
||||
return Result{CheckNonEmpty, true, ""}
|
||||
}
|
||||
}
|
||||
return Result{CheckNonEmpty, false, fmt.Sprintf("no letters in the reply %q — punctuation or digits only", strings.TrimSpace(body))}
|
||||
}
|
||||
|
||||
// checkEllipsis — a reply ending in "…" or "..." is a generation that ran out of
|
||||
|
||||
Reference in New Issue
Block a user