Maven/internal/phraser/transport.go

// phraser/transport.go — the wire to one llama-server: the request and
// response shapes, the GBNF that binds the model to the reply contract, and
// the single POST every phrasing path in this package goes through.
package phraser

import (
	"log"
)

type chatMsg struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type chatReq struct {
	Model       string    `json:"model"`
	Messages    []chatMsg `json:"messages"`
	Temperature float64   `json:"temperature"`
	MaxTokens   int       `json:"max_tokens"`
	// Grammar is llama-server's `grammar` field (GBNF). Same wiring as
	// internal/llm.Req.Grammar. Empty ⇒ unconstrained sampling.
	Grammar string `json:"grammar,omitempty"`
	// RepeatPenalty — defence in depth behind the bounded grammar, not the fix
	// for #531. This struct had no such field, so every caller through
	// chatWithSystem ran at the server default of 1.0 while Replier.PhraseReply
	// sent 1.3 through internal/llm and was protected by accident. Two wire
	// structs that disagree about the sampler is the condition that let one
	// path run away and the other not, and it should not survive as a
	// difference nobody chose.
	RepeatPenalty float64 `json:"repeat_penalty,omitempty"`
}

// phraseRepeatPenalty — matches Replier.PhraseReply, which has sent 1.3 since
// it was written. The value is not tuned here and is not what stops the
// whitespace loop; the bounded ws rule is. It is here so the two phrasing
// paths sample alike.
const phraseRepeatPenalty = 1.3

// responseGrammar — GBNF constraining the model to the documented phrasing
// contract and nothing else: {"response": "<text>", "mood": "<enum>"}.
//
// Without it a 0.8B answers roughly one chat turn in three with open reasoning
// as plain text ("Thinking Process:" …), which no tag-stripper can remove and
// which eats the token budget before the JSON closes. Modelled on
// routeGrammar in internal/router/llmrouter.go so the two read alike.
//
// text accepts ANY codepoint except the two JSON must escape and the control
// range — the replies are Russian, so an ASCII-only rule would make every reply
// empty. The escape rule is what lets the model close a string it opened with a
// quote inside. Length is bounded so a repetition loop truncates the field, not
// the JSON object.
//
// The control range is excluded because a raw newline inside a JSON string is
// not JSON (Vikunja #537). The class used to be `[^"\\]`, which let the model
// write a multi-line reply that satisfied the grammar and then failed
// json.Unmarshal with "invalid character '\n' in string literal" — the object
// starts with "{", so it came back as errBrokenJSON and the case answered with
// an empty string. Sixty of the failures in the 2026-08-05 temperature sweep
// were that one error, and it never once hit the token cap, which is why the
// truncation reading was wrong. The escape alternatives are llama.cpp's own
// json.gbnf: a model that wants a line break must write \n, which parses.
//
// That bound was 400 and 400 was too tight. Measured against Qwen3.5-0.8B: on
// "почему гром слышно позже молнии?" the reply came back exactly 400 characters
// long, cut mid-word ("Нужно записать и,"), at every token cap from 256 to 2048.
// So the token cap was never what stopped it — this rule was. 1000 characters is
// roughly six Russian sentences, still short enough to stop a repetition loop.
//
// ws is bounded for the same reason and it is the more expensive of the two.
// `*` let the model open the object and then satisfy ws with whitespace until
// max_tokens, which is 512 here: both interactive turns measured on 2026-08-04
// decoded exactly 512 tokens and spent 24-30 seconds doing it, all of it
// whitespace (Vikunja #531). Nothing on this path sends a repeat penalty —
// chatReq had no field for one — so the sampler never broke the loop. {0,4}
// was measured: three runs, three clean stops at 33 tokens, no penalty needed.
const responseGrammar = `
root   ::= "{" ws "\"response\"" ws ":" ws string ws "," ws "\"mood\"" ws ":" ws mood ws "}"
mood   ::= "\"neutral\"" | "\"happy\"" | "\"thinking\"" | "\"tired\"" | "\"confused\""
string ::= "\"" ([^"\\\x00-\x1F] | "\\" ["\\/bfnrt] | "\\u" [0-9a-fA-F]{4}){0,1000} "\""
ws     ::= [ \t\n]{0,4}
`

// ResponseGrammar exposes responseGrammar to the other callers that emit the
// same {"response","mood"} contract — cmd/mavend's reactive replier, which is
// parsed by the same two fields. One definition, so the two cannot drift.
const ResponseGrammar = responseGrammar

// grammar returns the GBNF to attach to a phrasing request, or "" when the
// operator turned it off.
func (p *LLMPhraser) grammar() string {
	if p.cfg.NoGrammar {
		return ""
	}
	return responseGrammar
}

type chatResp struct {
	Choices []struct {
		Message struct {
			Content          string `json:"content"`
			Reasoning        string `json:"reasoning"`
			ReasoningContent string `json:"reasoning_content"`
		} `json:"message"`
		// FinishReason — "stop" when the model chose to end, "length" when the
		// token cap cut it off. Parsed since #531, where two turns ran to the
		// 512-token cap and both happened to parse anyway: the grammar had
		// already closed the JSON, so a truncated generation was indistinguishable
		// from a good one at every layer above this struct.
		FinishReason string `json:"finish_reason"`
	} `json:"choices"`
}

// logIfTruncated says so when a generation stopped at the token cap.
//
// A cap hit is never routine. Either the model was looping, which is the #531
// shape, or the reply was genuinely longer than maxTokens, which means she cut
// herself off mid-sentence. Both are worth a line, and neither produced one
// before: the caller sees a parsed string and cannot tell.
func logIfTruncated(where, reason string, maxTokens int) {
	if reason == "length" {
		log.Printf("phraser: %s hit the %d-token cap (finish_reason=length) — the reply is truncated, or the model was looping", where, maxTokens)
	}
}