diff --git a/internal/phraser/broken_json_test.go b/internal/phraser/broken_json_test.go index d766b8a..ab20983 100644 --- a/internal/phraser/broken_json_test.go +++ b/internal/phraser/broken_json_test.go @@ -91,6 +91,39 @@ func TestEscapeRawControlsLeavesValidJSONAlone(t *testing.T) { } } +// Pretty-printed JSON is what Qwen3-1.7B writes: it opens the object and puts +// three newlines before the first key. Escaping those structural newlines made +// the object unparseable, so 31 of 36 conversational cases in the talk fixture +// answered from the stub (Vikunja #44, measured 2026-08-05). +func TestParseResponseMoodReadsPrettyPrintedJSON(t *testing.T) { + raw := "{\n\n\n \"response\": \"Хорошо настроение.\",\n \"mood\": \"neutral\"\n}" + text, mood, err := parseResponseMood(raw) + if err != nil { + t.Fatalf("err = %v, want nil — this is valid JSON, not a failed generation", err) + } + if want := "Хорошо настроение."; text != want { + t.Errorf("response = %q, want %q", text, want) + } + if mood != "neutral" { + t.Errorf("mood = %q, want neutral", mood) + } +} + +// Both at once: structural newlines outside the strings, a raw one inside. +func TestParseResponseMoodRepairsInsideAndKeepsOutside(t *testing.T) { + raw := "{\n\t\"response\": \"первая\nвторая\",\n\t\"mood\": \"tired\"\n}" + text, mood, err := parseResponseMood(raw) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if want := "первая\nвторая"; text != want { + t.Errorf("response = %q, want %q", text, want) + } + if mood != "tired" { + t.Errorf("mood = %q, want tired", mood) + } +} + // A tab and a bare control byte take the same path as the newline. func TestParseResponseMoodRepairsOtherControls(t *testing.T) { raw := "{\"response\": \"таб\tи \x01байт\", \"mood\": \"tired\"}" diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index 7647567..913210c 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -1155,27 +1155,51 @@ func parseResponseMood(raw string) (response, mood string, err error) { // second line, for the paths that send no grammar at all — NoGrammar, and any // remote model whose server ignores one. A raw newline is the shape that was // measured; the rest of the range is here because the same argument covers it. -// Only characters outside a string are affected in principle, and JSON permits -// none of this range outside a string either, so escaping unconditionally -// cannot turn valid JSON into anything else. +// +// Inside a string only. The first version escaped the whole object on the +// argument that JSON permits no control character outside a string either, so +// rewriting one could not do harm. That argument is wrong: JSON permits a +// newline, a tab and a return BETWEEN tokens, which is what pretty-printing is. +// Qwen3-1.7B pretty-prints — it opens `{` and writes three newlines before the +// first key — and escaping those into a literal backslash-n broke every reply +// it wrote. Measured 2026-08-05 on the talk fixture: 31 of 36 conversational +// cases came back as errBrokenJSON and answered from the stub (Vikunja #44). func escapeRawControls(s string) string { if !strings.ContainsFunc(s, func(r rune) bool { return r < 0x20 }) { return s } var b strings.Builder b.Grow(len(s) + 8) + inString := false + escaped := false for _, r := range s { switch { + case escaped: + // The character after a backslash is the model's own escape and is + // already whatever it meant to write. + escaped = false + b.WriteRune(r) + continue + case inString && r == '\\': + escaped = true + b.WriteRune(r) + continue + case r == '"': + inString = !inString + b.WriteRune(r) + continue + } + switch { + case !inString || r >= 0x20: + b.WriteRune(r) case r == '\n': b.WriteString(`\n`) case r == '\r': b.WriteString(`\r`) case r == '\t': b.WriteString(`\t`) - case r < 0x20: - fmt.Fprintf(&b, `\u%04x`, r) default: - b.WriteRune(r) + fmt.Fprintf(&b, `\u%04x`, r) } } return b.String()