// phraser/transport.go — the wire to one llama-server: the request and // response shapes, the GBNF that binds the model to the reply contract, and // the single POST every phrasing path in this package goes through. package phraser import ( "bytes" "context" "encoding/json" "fmt" "io" "log" "net/http" "strings" ) type chatMsg struct { Role string `json:"role"` Content string `json:"content"` } type chatReq struct { Model string `json:"model"` Messages []chatMsg `json:"messages"` Temperature float64 `json:"temperature"` MaxTokens int `json:"max_tokens"` // Grammar is llama-server's `grammar` field (GBNF). Same wiring as // internal/llm.Req.Grammar. Empty ⇒ unconstrained sampling. Grammar string `json:"grammar,omitempty"` // RepeatPenalty — defence in depth behind the bounded grammar, not the fix // for #531. This struct had no such field, so every caller through // chatWithSystem ran at the server default of 1.0 while Replier.PhraseReply // sent 1.3 through internal/llm and was protected by accident. Two wire // structs that disagree about the sampler is the condition that let one // path run away and the other not, and it should not survive as a // difference nobody chose. RepeatPenalty float64 `json:"repeat_penalty,omitempty"` } // phraseRepeatPenalty — matches Replier.PhraseReply, which has sent 1.3 since // it was written. The value is not tuned here and is not what stops the // whitespace loop; the bounded ws rule is. It is here so the two phrasing // paths sample alike. const phraseRepeatPenalty = 1.3 // responseGrammar — GBNF constraining the model to the documented phrasing // contract and nothing else: {"response": "", "mood": ""}. // // Without it a 0.8B answers roughly one chat turn in three with open reasoning // as plain text ("Thinking Process:" …), which no tag-stripper can remove and // which eats the token budget before the JSON closes. Modelled on // routeGrammar in internal/router/llmrouter.go so the two read alike. // // text accepts ANY codepoint except the two JSON must escape and the control // range — the replies are Russian, so an ASCII-only rule would make every reply // empty. The escape rule is what lets the model close a string it opened with a // quote inside. Length is bounded so a repetition loop truncates the field, not // the JSON object. // // The control range is excluded because a raw newline inside a JSON string is // not JSON (Vikunja #537). The class used to be `[^"\\]`, which let the model // write a multi-line reply that satisfied the grammar and then failed // json.Unmarshal with "invalid character '\n' in string literal" — the object // starts with "{", so it came back as errBrokenJSON and the case answered with // an empty string. Sixty of the failures in the 2026-08-05 temperature sweep // were that one error, and it never once hit the token cap, which is why the // truncation reading was wrong. The escape alternatives are llama.cpp's own // json.gbnf: a model that wants a line break must write \n, which parses. // // That bound was 400 and 400 was too tight. Measured against Qwen3.5-0.8B: on // "почему гром слышно позже молнии?" the reply came back exactly 400 characters // long, cut mid-word ("Нужно записать и,"), at every token cap from 256 to 2048. // So the token cap was never what stopped it — this rule was. 1000 characters is // roughly six Russian sentences, still short enough to stop a repetition loop. // // ws is bounded for the same reason and it is the more expensive of the two. // `*` let the model open the object and then satisfy ws with whitespace until // max_tokens, which is 512 here: both interactive turns measured on 2026-08-04 // decoded exactly 512 tokens and spent 24-30 seconds doing it, all of it // whitespace (Vikunja #531). Nothing on this path sends a repeat penalty — // chatReq had no field for one — so the sampler never broke the loop. {0,4} // was measured: three runs, three clean stops at 33 tokens, no penalty needed. const responseGrammar = ` root ::= "{" ws "\"response\"" ws ":" ws string ws "," ws "\"mood\"" ws ":" ws mood ws "}" mood ::= "\"neutral\"" | "\"happy\"" | "\"thinking\"" | "\"tired\"" | "\"confused\"" string ::= "\"" ([^"\\\x00-\x1F] | "\\" ["\\/bfnrt] | "\\u" [0-9a-fA-F]{4}){0,1000} "\"" ws ::= [ \t\n]{0,4} ` // ResponseGrammar exposes responseGrammar to the other callers that emit the // same {"response","mood"} contract — cmd/mavend's reactive replier, which is // parsed by the same two fields. One definition, so the two cannot drift. const ResponseGrammar = responseGrammar // grammar returns the GBNF to attach to a phrasing request, or "" when the // operator turned it off. func (p *LLMPhraser) grammar() string { if p.cfg.NoGrammar { return "" } return responseGrammar } type chatResp struct { Choices []struct { Message struct { Content string `json:"content"` Reasoning string `json:"reasoning"` ReasoningContent string `json:"reasoning_content"` } `json:"message"` // FinishReason — "stop" when the model chose to end, "length" when the // token cap cut it off. Parsed since #531, where two turns ran to the // 512-token cap and both happened to parse anyway: the grammar had // already closed the JSON, so a truncated generation was indistinguishable // from a good one at every layer above this struct. FinishReason string `json:"finish_reason"` } `json:"choices"` } // logIfTruncated says so when a generation stopped at the token cap. // // A cap hit is never routine. Either the model was looping, which is the #531 // shape, or the reply was genuinely longer than maxTokens, which means she cut // herself off mid-sentence. Both are worth a line, and neither produced one // before: the caller sees a parsed string and cannot tell. func logIfTruncated(where, reason string, maxTokens int) { if reason == "length" { log.Printf("phraser: %s hit the %d-token cap (finish_reason=length) — the reply is truncated, or the model was looping", where, maxTokens) } } // postChat sends one message array to the resident llama-server and returns // what the model wrote, think block stripped. // // The only POST in the package. chatWithSystem and chatWithMessages each // carried their own copy of these forty lines, identical down to the error // strings, and the copies had already drifted twice: one logged the raw // content and the other did not, and one labelled a truncation "chat" where // the other said "chatWithSystem". Two transports mean two chances to // configure the sampler differently, which is exactly how #531 happened. // // where names the calling path, for the truncation line and nothing else. func (p *LLMPhraser) postChat(ctx context.Context, where string, msgs []chatMsg, maxTokens int) (string, error) { base, release, err := p.acquire() if err != nil { return "", err } defer release() body, err := json.Marshal(chatReq{ Messages: msgs, Temperature: p.temperature(), MaxTokens: maxTokens, Grammar: p.grammar(), RepeatPenalty: phraseRepeatPenalty, }) if err != nil { return "", fmt.Errorf("llm: marshal: %w", err) } httpReq, err := http.NewRequestWithContext(ctx, "POST", base+"/v1/chat/completions", bytes.NewReader(body)) if err != nil { return "", fmt.Errorf("llm: request: %w", err) } httpReq.Header.Set("Content-Type", "application/json") resp, err := p.client.Do(httpReq) if err != nil { return "", fmt.Errorf("llm: post: %w", err) } defer resp.Body.Close() raw, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("llm: read: %w", err) } if resp.StatusCode != 200 { return "", fmt.Errorf("llm: status %d: %s", resp.StatusCode, strings.TrimSpace(string(raw))) } var cr chatResp if err := json.Unmarshal(raw, &cr); err != nil { return "", fmt.Errorf("llm: parse: %w", err) } if len(cr.Choices) == 0 { return "", fmt.Errorf("llm: no choices in response") } logIfTruncated(where, cr.Choices[0].FinishReason, maxTokens) content := cr.Choices[0].Message.Content if content == "" { content = cr.Choices[0].Message.ReasoningContent } // Every phrasing path logs its raw generation now. Only chatWithMessages // did, so a nudge or a query that came back unparseable left nothing in the // log to read (V-397). log.Printf("phraser: %s raw content: %q", where, content) return stripThink(content), nil } // chatWithSystem is the common shape: one system turn, one user turn. // // The workstation model first when it will take work, and silently: every // caller of this helper is on the silent half of the degradation rule. It // answering is not news, and it being asleep is not news either. func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, maxTokens int) (string, error) { if out, ok := p.remoteChat(ctx, system, user, maxTokens); ok { return out, nil } return p.postChat(ctx, "chatWithSystem", []chatMsg{ {Role: "system", Content: system}, {Role: "user", Content: user}, }, maxTokens) } // chatWithMessages sends a full message array (system + history + current) to // the LLM completion endpoint. Like chatWithSystem but for an arbitrary message // slice — the caller owns the system prompt placement. func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTokens int) (string, error) { // Same silent preference as chatWithSystem, when the array is the shape // llm.Req can carry: one system turn and one user turn. PhraseChat already // folds the history into a single user message (some chat templates reject // consecutive user turns), so today that is every call. A longer array goes // to the resident model rather than get flattened here, because flattening a // conversation is a decision its owner should make. if len(msgs) == 2 && msgs[0].Role == "system" && msgs[1].Role == "user" { if out, ok := p.remoteChat(ctx, msgs[0].Content, msgs[1].Content, maxTokens); ok { return out, nil } } return p.postChat(ctx, "chat", msgs, maxTokens) }