claude
32d5f68710
Sixty of the failures in the 2026-08-05 temperature sweep were one error,
`phraser: model output starts as JSON but does not parse`, all of them in the
reply family and two of them in all twelve runs. The write-up read that as
truncation. It is not: no run hit the token cap.
The string rule in both grammars was `[^"\\]`, which admits a literal
newline. A model that wants two lines writes one, the generation satisfies the
grammar, and json.Unmarshal then rejects it with "invalid character '\n' in
string literal". The object starts with "{", so it came back as errBrokenJSON
and the reply was an empty string. The router's rule also admitted `"\\" .`,
so \q satisfied it and failed to parse the same way.
Both string rules are now llama.cpp's own json.gbnf class: the control range is
out and the escape alternatives are exact. Verified against the resident model
on 8899 — llama-server accepts both grammars and both still emit what they did.
escapeRawControls is the second line, for NoGrammar and for a remote server that
ignores a grammar: a reply whose only fault is a raw newline is readable, so it
is read rather than dropped.
105 lines
3.9 KiB
Go
105 lines
3.9 KiB
Go
package phraser
|
||
|
||
import (
|
||
"errors"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// A reply that starts a JSON object and never finishes it is a failed
|
||
// generation, not a reply. Before this, the parser returned ("", "") for these
|
||
// and every caller then shipped the raw fragment as the thing Maven said. A
|
||
// real run produced replies of literally "{" and "{\n \"".
|
||
func TestParseResponseMoodRejectsUnfinishedJSON(t *testing.T) {
|
||
for _, raw := range []string{
|
||
`{`,
|
||
"{\n \"",
|
||
`{"response": "неполн`,
|
||
`{"response": "текст", "mood":`,
|
||
} {
|
||
text, mood, err := parseResponseMood(raw)
|
||
if !errors.Is(err, errBrokenJSON) {
|
||
t.Errorf("parseResponseMood(%q) err = %v, want errBrokenJSON", raw, err)
|
||
}
|
||
if text != "" || mood != "" {
|
||
t.Errorf("parseResponseMood(%q) leaked %q/%q — a fragment must never come back as a reply", raw, text, mood)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Bare prose is still fine. Small models sometimes answer without any JSON at
|
||
// all, and that reply is usable — so the new error must not swallow it.
|
||
func TestParseResponseMoodAllowsBareProse(t *testing.T) {
|
||
for _, raw := range []string{
|
||
"норм, а ты как?",
|
||
"вот что я нашла: ключ у соседа",
|
||
} {
|
||
text, mood, err := parseResponseMood(raw)
|
||
if err != nil {
|
||
t.Errorf("parseResponseMood(%q) err = %v, want nil", raw, err)
|
||
}
|
||
// No JSON means no fields; the caller ships raw as-is.
|
||
if text != "" || mood != "" {
|
||
t.Errorf("parseResponseMood(%q) = %q/%q, want empty", raw, text, mood)
|
||
}
|
||
}
|
||
}
|
||
|
||
// The measured failure: the model wants more than 400 characters and the old
|
||
// grammar cut it off mid-word. Guards the bound against being tightened back.
|
||
func TestGrammarStringBoundHasRoomForARealAnswer(t *testing.T) {
|
||
if !strings.Contains(responseGrammar, "{0,1000}") {
|
||
t.Error("grammar string bound is not 1000; 400 truncated real replies mid-word (see the comment on responseGrammar)")
|
||
}
|
||
}
|
||
|
||
// A multi-line reply is the sixty-failure shape from the 2026-08-05 temperature
|
||
// sweep (Vikunja #537). The model wrote a literal newline inside the string,
|
||
// which the old grammar allowed and json.Unmarshal rejects with "invalid
|
||
// character '\n' in string literal". The object starts with "{", so it came back
|
||
// as errBrokenJSON and the case answered with nothing at all.
|
||
//
|
||
// The grammar is the fix. This is the parser's own second line, for the paths
|
||
// that send no grammar: the reply is readable, so read it.
|
||
func TestParseResponseMoodRepairsARawNewline(t *testing.T) {
|
||
raw := "{\"response\": \"первая строка\nвторая строка\", \"mood\": \"neutral\"}"
|
||
text, mood, err := parseResponseMood(raw)
|
||
if err != nil {
|
||
t.Fatalf("err = %v, want nil — a raw newline is repairable, not a failed generation", err)
|
||
}
|
||
if want := "первая строка\nвторая строка"; text != want {
|
||
t.Errorf("response = %q, want %q", text, want)
|
||
}
|
||
if mood != "neutral" {
|
||
t.Errorf("mood = %q, want neutral", mood)
|
||
}
|
||
}
|
||
|
||
// Repairing must not rewrite JSON that was already fine: an escaped newline
|
||
// stays one newline, and a backslash the model escaped properly is left alone.
|
||
func TestEscapeRawControlsLeavesValidJSONAlone(t *testing.T) {
|
||
raw := `{"response": "строка\nдве \\ и \"кавычки\"", "mood": "happy"}`
|
||
if got := escapeRawControls(raw); got != raw {
|
||
t.Errorf("escapeRawControls rewrote valid JSON:\n got %q\nwant %q", got, raw)
|
||
}
|
||
text, _, err := parseResponseMood(raw)
|
||
if err != nil {
|
||
t.Fatalf("err = %v", err)
|
||
}
|
||
if want := "строка\nдве \\ и \"кавычки\""; text != want {
|
||
t.Errorf("response = %q, want %q", text, want)
|
||
}
|
||
}
|
||
|
||
// A tab and a bare control byte take the same path as the newline.
|
||
func TestParseResponseMoodRepairsOtherControls(t *testing.T) {
|
||
raw := "{\"response\": \"таб\tи \x01байт\", \"mood\": \"tired\"}"
|
||
text, _, err := parseResponseMood(raw)
|
||
if err != nil {
|
||
t.Fatalf("err = %v, want nil", err)
|
||
}
|
||
if want := "таб\tи \x01байт"; text != want {
|
||
t.Errorf("response = %q, want %q", text, want)
|
||
}
|
||
}
|