From 4dbeca5a2e5417c07dd51876115216146aafff85 Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 5 Aug 2026 14:02:53 +0400 Subject: [PATCH 1/2] escape control characters inside the string, not around it (V-44) Qwen3-1.7B pretty-prints its JSON: it opens the object and writes three newlines before the first key. escapeRawControls rewrote those structural newlines into a literal backslash-n, which is legal nowhere outside a string, so the object stopped parsing and came back as errBrokenJSON. The comment claimed escaping unconditionally could not turn valid JSON into anything else, on the grounds that JSON permits no control character outside a string. It permits three: newline, tab and return are whitespace between tokens, and that is what pretty-printing is made of. Measured on the talk fixture against the resident model: 31 of 36 conversational cases were failing generations and answered from the stub. Every chat reply and every knowledge answer the resident model wrote was being discarded. Now 25/36 pass every check, 0 errors, and the 15 nudges stay at 15/15. --- internal/phraser/broken_json_test.go | 33 +++++++++++++++++++++++++ internal/phraser/llmphraser.go | 36 +++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/internal/phraser/broken_json_test.go b/internal/phraser/broken_json_test.go index d766b8a..ab20983 100644 --- a/internal/phraser/broken_json_test.go +++ b/internal/phraser/broken_json_test.go @@ -91,6 +91,39 @@ func TestEscapeRawControlsLeavesValidJSONAlone(t *testing.T) { } } +// Pretty-printed JSON is what Qwen3-1.7B writes: it opens the object and puts +// three newlines before the first key. Escaping those structural newlines made +// the object unparseable, so 31 of 36 conversational cases in the talk fixture +// answered from the stub (Vikunja #44, measured 2026-08-05). +func TestParseResponseMoodReadsPrettyPrintedJSON(t *testing.T) { + raw := "{\n\n\n \"response\": \"Хорошо настроение.\",\n \"mood\": \"neutral\"\n}" + text, mood, err := parseResponseMood(raw) + if err != nil { + t.Fatalf("err = %v, want nil — this is valid JSON, not a failed generation", err) + } + if want := "Хорошо настроение."; text != want { + t.Errorf("response = %q, want %q", text, want) + } + if mood != "neutral" { + t.Errorf("mood = %q, want neutral", mood) + } +} + +// Both at once: structural newlines outside the strings, a raw one inside. +func TestParseResponseMoodRepairsInsideAndKeepsOutside(t *testing.T) { + raw := "{\n\t\"response\": \"первая\nвторая\",\n\t\"mood\": \"tired\"\n}" + text, mood, err := parseResponseMood(raw) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if want := "первая\nвторая"; text != want { + t.Errorf("response = %q, want %q", text, want) + } + if mood != "tired" { + t.Errorf("mood = %q, want tired", mood) + } +} + // A tab and a bare control byte take the same path as the newline. func TestParseResponseMoodRepairsOtherControls(t *testing.T) { raw := "{\"response\": \"таб\tи \x01байт\", \"mood\": \"tired\"}" diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index 7647567..913210c 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -1155,27 +1155,51 @@ func parseResponseMood(raw string) (response, mood string, err error) { // second line, for the paths that send no grammar at all — NoGrammar, and any // remote model whose server ignores one. A raw newline is the shape that was // measured; the rest of the range is here because the same argument covers it. -// Only characters outside a string are affected in principle, and JSON permits -// none of this range outside a string either, so escaping unconditionally -// cannot turn valid JSON into anything else. +// +// Inside a string only. The first version escaped the whole object on the +// argument that JSON permits no control character outside a string either, so +// rewriting one could not do harm. That argument is wrong: JSON permits a +// newline, a tab and a return BETWEEN tokens, which is what pretty-printing is. +// Qwen3-1.7B pretty-prints — it opens `{` and writes three newlines before the +// first key — and escaping those into a literal backslash-n broke every reply +// it wrote. Measured 2026-08-05 on the talk fixture: 31 of 36 conversational +// cases came back as errBrokenJSON and answered from the stub (Vikunja #44). func escapeRawControls(s string) string { if !strings.ContainsFunc(s, func(r rune) bool { return r < 0x20 }) { return s } var b strings.Builder b.Grow(len(s) + 8) + inString := false + escaped := false for _, r := range s { switch { + case escaped: + // The character after a backslash is the model's own escape and is + // already whatever it meant to write. + escaped = false + b.WriteRune(r) + continue + case inString && r == '\\': + escaped = true + b.WriteRune(r) + continue + case r == '"': + inString = !inString + b.WriteRune(r) + continue + } + switch { + case !inString || r >= 0x20: + b.WriteRune(r) case r == '\n': b.WriteString(`\n`) case r == '\r': b.WriteString(`\r`) case r == '\t': b.WriteString(`\t`) - case r < 0x20: - fmt.Fprintf(&b, `\u%04x`, r) default: - b.WriteRune(r) + fmt.Fprintf(&b, `\u%04x`, r) } } return b.String() From b752ec037e03875c2d65d2ff0e655d8d402be71b Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 5 Aug 2026 14:03:45 +0400 Subject: [PATCH 2/2] talk fixture on the resident model: 2/36 to 25/36 (V-44) --- .../evals/2026-08-05-talk-fixture-resident.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 docs/evals/2026-08-05-talk-fixture-resident.md diff --git a/docs/evals/2026-08-05-talk-fixture-resident.md b/docs/evals/2026-08-05-talk-fixture-resident.md new file mode 100644 index 0000000..e32b0b6 --- /dev/null +++ b/docs/evals/2026-08-05-talk-fixture-resident.md @@ -0,0 +1,60 @@ +# Talk fixture against the resident model, 2026-08-05 + +Vikunja #44 step 1. `MAVEN_LLM_URL=http://127.0.0.1:8899 make eval-phrasing`, +Qwen3-1.7B-UD-Q4_K_XL on the host, no workstation in the run. The fixture holds +36 cases now, against 27 when the bakeoff measured it. So the old score is not +a column in this table. + +## Result + +| | before the escape fix | after | +|---|---|---| +| talk, passes every check | 2/36 (5.6%) | 25/36 (69.4%) | +| failed generations | 31 | 0 | +| by path: chat | 0/9 | 4/9 | +| by path: knowledge | 1/9 | 6/9 | +| by path: query | 0/9 | 9/9 | +| by path: reply | 1/9 | 6/9 | +| feminine | 5/36 | 36/36 | +| address | 5/36 | 33/36 | +| ontopic | 2/36 | 28/36 | +| p50 latency | 3.05s | 2.97s | +| nudges (15 cases) | 15/15 | 15/15 | + +## What the 31 errors were + +Not the model. `escapeRawControls` in `internal/phraser/llmphraser.go`, added +for #537 to repair a raw newline written inside a string, escaped the whole +object. Qwen3-1.7B pretty-prints: it opens `{` and writes three newlines before +the first key. Those newlines became a literal backslash-n, which is legal +nowhere outside a string, so the object stopped parsing and `parseResponseMood` +reported `errBrokenJSON`. + +The comment said escaping unconditionally could not turn valid JSON into +anything else, because JSON permits no control character outside a string. It +permits three. Newline, tab and return are whitespace between tokens, and that +is what pretty-printing is made of. + +Every chat reply and every knowledge answer the resident model wrote was being +discarded for a stub line. The nudge path never showed it, because the nudge +prompt gets compact JSON back. + +## The 11 that still fail + +Eight are `ontopic`, three are `address`. + +The address failures are all plural imperatives written to a formal listener: +`держите`, `уточните`, `попробуйте`. Feminine self-reference held in all 36, +which is the half #122 is training for. So the persona gap the CPT is aimed at +is now the address half, not the gender half. + +The ontopic failures are the resident model answering next to the question +rather than in it. `chat-joke` describes crying dolls instead of telling one, +`know-hiccups` calls hiccups an icon, `know-boil-egg` answers about an omelette. +`chat-about-me` answers "Я - записка", which is the same confabulation the +bakeoff recorded. + +## Not measured here + +The workstation. Every number above is the homesrv floor. `make eval-phrasing` +points at one URL, so a gemma-4-12b column needs its own run.