escape control characters inside the string, not around it (V-44)

Qwen3-1.7B pretty-prints its JSON: it opens the object and writes three
newlines before the first key. escapeRawControls rewrote those structural
newlines into a literal backslash-n, which is legal nowhere outside a string,
so the object stopped parsing and came back as errBrokenJSON.

The comment claimed escaping unconditionally could not turn valid JSON into
anything else, on the grounds that JSON permits no control character outside a
string. It permits three: newline, tab and return are whitespace between
tokens, and that is what pretty-printing is made of.

Measured on the talk fixture against the resident model: 31 of 36 conversational
cases were failing generations and answered from the stub. Every chat reply and
every knowledge answer the resident model wrote was being discarded. Now 25/36
pass every check, 0 errors, and the 15 nudges stay at 15/15.
This commit is contained in:
2026-08-05 14:02:53 +04:00
parent 9e15ff36aa
commit 4dbeca5a2e
2 changed files with 63 additions and 6 deletions
+33
View File
@@ -91,6 +91,39 @@ func TestEscapeRawControlsLeavesValidJSONAlone(t *testing.T) {
}
}
// Pretty-printed JSON is what Qwen3-1.7B writes: it opens the object and puts
// three newlines before the first key. Escaping those structural newlines made
// the object unparseable, so 31 of 36 conversational cases in the talk fixture
// answered from the stub (Vikunja #44, measured 2026-08-05).
func TestParseResponseMoodReadsPrettyPrintedJSON(t *testing.T) {
raw := "{\n\n\n \"response\": \"Хорошо настроение.\",\n \"mood\": \"neutral\"\n}"
text, mood, err := parseResponseMood(raw)
if err != nil {
t.Fatalf("err = %v, want nil — this is valid JSON, not a failed generation", err)
}
if want := "Хорошо настроение."; text != want {
t.Errorf("response = %q, want %q", text, want)
}
if mood != "neutral" {
t.Errorf("mood = %q, want neutral", mood)
}
}
// Both at once: structural newlines outside the strings, a raw one inside.
func TestParseResponseMoodRepairsInsideAndKeepsOutside(t *testing.T) {
raw := "{\n\t\"response\": \"первая\nвторая\",\n\t\"mood\": \"tired\"\n}"
text, mood, err := parseResponseMood(raw)
if err != nil {
t.Fatalf("err = %v, want nil", err)
}
if want := "первая\nвторая"; text != want {
t.Errorf("response = %q, want %q", text, want)
}
if mood != "tired" {
t.Errorf("mood = %q, want tired", mood)
}
}
// A tab and a bare control byte take the same path as the newline.
func TestParseResponseMoodRepairsOtherControls(t *testing.T) {
raw := "{\"response\": \"таб\tи \x01байт\", \"mood\": \"tired\"}"
+30 -6
View File
@@ -1155,27 +1155,51 @@ func parseResponseMood(raw string) (response, mood string, err error) {
// second line, for the paths that send no grammar at all — NoGrammar, and any
// remote model whose server ignores one. A raw newline is the shape that was
// measured; the rest of the range is here because the same argument covers it.
// Only characters outside a string are affected in principle, and JSON permits
// none of this range outside a string either, so escaping unconditionally
// cannot turn valid JSON into anything else.
//
// Inside a string only. The first version escaped the whole object on the
// argument that JSON permits no control character outside a string either, so
// rewriting one could not do harm. That argument is wrong: JSON permits a
// newline, a tab and a return BETWEEN tokens, which is what pretty-printing is.
// Qwen3-1.7B pretty-prints — it opens `{` and writes three newlines before the
// first key — and escaping those into a literal backslash-n broke every reply
// it wrote. Measured 2026-08-05 on the talk fixture: 31 of 36 conversational
// cases came back as errBrokenJSON and answered from the stub (Vikunja #44).
func escapeRawControls(s string) string {
if !strings.ContainsFunc(s, func(r rune) bool { return r < 0x20 }) {
return s
}
var b strings.Builder
b.Grow(len(s) + 8)
inString := false
escaped := false
for _, r := range s {
switch {
case escaped:
// The character after a backslash is the model's own escape and is
// already whatever it meant to write.
escaped = false
b.WriteRune(r)
continue
case inString && r == '\\':
escaped = true
b.WriteRune(r)
continue
case r == '"':
inString = !inString
b.WriteRune(r)
continue
}
switch {
case !inString || r >= 0x20:
b.WriteRune(r)
case r == '\n':
b.WriteString(`\n`)
case r == '\r':
b.WriteString(`\r`)
case r == '\t':
b.WriteString(`\t`)
case r < 0x20:
fmt.Fprintf(&b, `\u%04x`, r)
default:
b.WriteRune(r)
fmt.Fprintf(&b, `\u%04x`, r)
}
}
return b.String()