diff --git a/docs/evals/2026-08-05-talk-fixture-resident.md b/docs/evals/2026-08-05-talk-fixture-resident.md new file mode 100644 index 0000000..e32b0b6 --- /dev/null +++ b/docs/evals/2026-08-05-talk-fixture-resident.md @@ -0,0 +1,60 @@ +# Talk fixture against the resident model, 2026-08-05 + +Vikunja #44 step 1. `MAVEN_LLM_URL=http://127.0.0.1:8899 make eval-phrasing`, +Qwen3-1.7B-UD-Q4_K_XL on the host, no workstation in the run. The fixture holds +36 cases now, against 27 when the bakeoff measured it. So the old score is not +a column in this table. + +## Result + +| | before the escape fix | after | +|---|---|---| +| talk, passes every check | 2/36 (5.6%) | 25/36 (69.4%) | +| failed generations | 31 | 0 | +| by path: chat | 0/9 | 4/9 | +| by path: knowledge | 1/9 | 6/9 | +| by path: query | 0/9 | 9/9 | +| by path: reply | 1/9 | 6/9 | +| feminine | 5/36 | 36/36 | +| address | 5/36 | 33/36 | +| ontopic | 2/36 | 28/36 | +| p50 latency | 3.05s | 2.97s | +| nudges (15 cases) | 15/15 | 15/15 | + +## What the 31 errors were + +Not the model. `escapeRawControls` in `internal/phraser/llmphraser.go`, added +for #537 to repair a raw newline written inside a string, escaped the whole +object. Qwen3-1.7B pretty-prints: it opens `{` and writes three newlines before +the first key. Those newlines became a literal backslash-n, which is legal +nowhere outside a string, so the object stopped parsing and `parseResponseMood` +reported `errBrokenJSON`. + +The comment said escaping unconditionally could not turn valid JSON into +anything else, because JSON permits no control character outside a string. It +permits three. Newline, tab and return are whitespace between tokens, and that +is what pretty-printing is made of. + +Every chat reply and every knowledge answer the resident model wrote was being +discarded for a stub line. The nudge path never showed it, because the nudge +prompt gets compact JSON back. + +## The 11 that still fail + +Eight are `ontopic`, three are `address`. + +The address failures are all plural imperatives written to a formal listener: +`держите`, `уточните`, `попробуйте`. Feminine self-reference held in all 36, +which is the half #122 is training for. So the persona gap the CPT is aimed at +is now the address half, not the gender half. + +The ontopic failures are the resident model answering next to the question +rather than in it. `chat-joke` describes crying dolls instead of telling one, +`know-hiccups` calls hiccups an icon, `know-boil-egg` answers about an omelette. +`chat-about-me` answers "Я - записка", which is the same confabulation the +bakeoff recorded. + +## Not measured here + +The workstation. Every number above is the homesrv floor. `make eval-phrasing` +points at one URL, so a gemma-4-12b column needs its own run. diff --git a/internal/phraser/broken_json_test.go b/internal/phraser/broken_json_test.go index d766b8a..ab20983 100644 --- a/internal/phraser/broken_json_test.go +++ b/internal/phraser/broken_json_test.go @@ -91,6 +91,39 @@ func TestEscapeRawControlsLeavesValidJSONAlone(t *testing.T) { } } +// Pretty-printed JSON is what Qwen3-1.7B writes: it opens the object and puts +// three newlines before the first key. Escaping those structural newlines made +// the object unparseable, so 31 of 36 conversational cases in the talk fixture +// answered from the stub (Vikunja #44, measured 2026-08-05). +func TestParseResponseMoodReadsPrettyPrintedJSON(t *testing.T) { + raw := "{\n\n\n \"response\": \"Хорошо настроение.\",\n \"mood\": \"neutral\"\n}" + text, mood, err := parseResponseMood(raw) + if err != nil { + t.Fatalf("err = %v, want nil — this is valid JSON, not a failed generation", err) + } + if want := "Хорошо настроение."; text != want { + t.Errorf("response = %q, want %q", text, want) + } + if mood != "neutral" { + t.Errorf("mood = %q, want neutral", mood) + } +} + +// Both at once: structural newlines outside the strings, a raw one inside. +func TestParseResponseMoodRepairsInsideAndKeepsOutside(t *testing.T) { + raw := "{\n\t\"response\": \"первая\nвторая\",\n\t\"mood\": \"tired\"\n}" + text, mood, err := parseResponseMood(raw) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if want := "первая\nвторая"; text != want { + t.Errorf("response = %q, want %q", text, want) + } + if mood != "tired" { + t.Errorf("mood = %q, want tired", mood) + } +} + // A tab and a bare control byte take the same path as the newline. func TestParseResponseMoodRepairsOtherControls(t *testing.T) { raw := "{\"response\": \"таб\tи \x01байт\", \"mood\": \"tired\"}" diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index 7647567..913210c 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -1155,27 +1155,51 @@ func parseResponseMood(raw string) (response, mood string, err error) { // second line, for the paths that send no grammar at all — NoGrammar, and any // remote model whose server ignores one. A raw newline is the shape that was // measured; the rest of the range is here because the same argument covers it. -// Only characters outside a string are affected in principle, and JSON permits -// none of this range outside a string either, so escaping unconditionally -// cannot turn valid JSON into anything else. +// +// Inside a string only. The first version escaped the whole object on the +// argument that JSON permits no control character outside a string either, so +// rewriting one could not do harm. That argument is wrong: JSON permits a +// newline, a tab and a return BETWEEN tokens, which is what pretty-printing is. +// Qwen3-1.7B pretty-prints — it opens `{` and writes three newlines before the +// first key — and escaping those into a literal backslash-n broke every reply +// it wrote. Measured 2026-08-05 on the talk fixture: 31 of 36 conversational +// cases came back as errBrokenJSON and answered from the stub (Vikunja #44). func escapeRawControls(s string) string { if !strings.ContainsFunc(s, func(r rune) bool { return r < 0x20 }) { return s } var b strings.Builder b.Grow(len(s) + 8) + inString := false + escaped := false for _, r := range s { switch { + case escaped: + // The character after a backslash is the model's own escape and is + // already whatever it meant to write. + escaped = false + b.WriteRune(r) + continue + case inString && r == '\\': + escaped = true + b.WriteRune(r) + continue + case r == '"': + inString = !inString + b.WriteRune(r) + continue + } + switch { + case !inString || r >= 0x20: + b.WriteRune(r) case r == '\n': b.WriteString(`\n`) case r == '\r': b.WriteString(`\r`) case r == '\t': b.WriteString(`\t`) - case r < 0x20: - fmt.Fprintf(&b, `\u%04x`, r) default: - b.WriteRune(r) + fmt.Fprintf(&b, `\u%04x`, r) } } return b.String()