package phraser import ( "context" "errors" "fmt" "log" "net/http" "strings" "sync" "time" "github.com/kami/maven/internal/delivery" "github.com/kami/maven/internal/dialogue" "github.com/kami/maven/internal/loop" ) // errEmptyResponse — the server answered and said nothing. Separate from a // transport failure: the model is up and produced no tokens, which is still not // an answer and must not score as one. var errEmptyResponse = errors.New("phraser: empty response from the model") type LLMPhraser struct { cfg Config client *http.Client // tmpl — the hand-written Russian nudges. Default path for nudges; see // Config.LLMNudges. nil only if the template file failed to load. tmpl *NudgeTemplates // spawnCtx — the parent of every llama-server this phraser starts, i.e. the // daemon's own context. Deliberately NOT the per-request context of the call // that asked for a model swap: that one is cancelled the moment the request // returns, which would kill the model it had just loaded. spawnCtx context.Context cancel context.CancelFunc // launch / probe — the two side effects of a swap, injectable so the swap // logic is testable without a real llama-server and a real model file. // launch is nil when this phraser does not own its server (NewLLMPhraserAt), // which is also what makes Swap refuse there. launch func(ctx context.Context, cfg Config) (backend, error) probe func(ctx context.Context, base string) (string, error) // remote — the workstation model, when one is configured. Set once at wiring // time by UseRemote and read on every phrasing call. nil ⇒ every call goes to // the resident llama-server this phraser owns, which is the whole deploy // before a `workstation` block exists. See world.go. remote Remote // swapMu — single-flight around Swap. Held for the whole swap, including the // model load, so two concurrent swap requests can never both be loading. swapMu sync.Mutex // mu guards everything below: the live backend, the swap gate and the // in-flight request count. See acquire/quiesce in swap.go. mu sync.Mutex be backend live liveModel swapping bool inflight int observers []func(baseURL string) } // liveModel — what is actually loaded right now. Distinct from Config, which // stays immutable after construction: a swap changes these three fields and // nothing else, so no reader of cfg (prompts, grammar, timeouts) races a swap. type liveModel struct { ModelPath string NGpuLayers int NCtx int } type Config struct { ModelPath string BinPath string Listen string NGpuLayers int NCtx int Timeout time.Duration // StartupTimeout bounds the wait for llama-server to print the address it // listens on. A config field and not a constant because the box may // legitimately need longer: a cold 1.7B loading off a spinning disk can // outrun a minute, and until this existed that returned "server did not // start within 60s" with no way to raise it. // // 0 ⇒ defaultStartupTimeout. StartupTimeout time.Duration // CacheRAMMiB bounds llama-server's prompt cache, which is what actually ate // this box. Measured on homesrv 2026-08-03: the server's own default limit is // 8192 MiB, it stores the full KV state of every idle slot it evicts (112 kiB // per token, so 166 MiB for one 1521-token prompt), and RSS climbed by that // much per distinct prompt until it hit 7.9 GB and half a gigabyte went to // swap. Weights are only 1.1 GB and mmapped, and -ngl 99 costs almost no RSS // because RADV keeps device memory outside the process. // // 0 ⇒ the flag is not passed and the server's own 8 GiB default applies. That // is the escape hatch for a llama-server too old to know --cache-ram, not a // recommendation. See docs/evals/2026-08-03-llama-prompt-cache.md. CacheRAMMiB int // ContextBlock renders the shared context block (who he is, how to // address him, the time) fresh for each turn. See internal/persona. // nil ⇒ no block, the prompts stand alone. ContextBlock func() string // LLMNudges puts the model back in charge of nudge wording. // // Off by default, and that is a deliberate deprecation of LLM-phrased // nudges: hand-written templates (nudges_ru_v1.json) word every nudge now. // A nudge has nothing to be creative about, and measured over many runs the // 0.8B broke the persona (formal "вы", plural imperatives, masculine // self-reference) and invented facts and units. Templates score 15/15 on the // nudge fixture, the model 11-13/15. // // The LLM path is kept, not deleted: flip this on to get it back. Chat, // query and reminder phrasing are untouched and still go through the model. LLMNudges bool // Temperature — what every phrasing call samples at. 0 ⇒ 0.7, which is // what this transport has always sent. // // A field rather than a constant so the talk fixture can sweep it // (Vikunja #402). Sampling is a dial, and a dial nobody can turn from // outside the package cannot be measured, only argued about. Temperature float64 // NoGrammar turns the GBNF constraint off (zero value ⇒ grammar ON). // The escape hatch exists because the target resident model — the // locally CPT'd Qwen3-1.7B — does not exist yet: if its chat template // ever fights the grammar, the fix should be a config flip on the // deploy box, not a code change and a rebuild. NoGrammar bool } func DefaultConfig(modelPath string) Config { return Config{ ModelPath: modelPath, BinPath: "llama-server", Listen: "127.0.0.1:0", NGpuLayers: -1, NCtx: 2048, // 512 MiB caps total RSS near 1 GB and still holds several recent prompts. CacheRAMMiB: 512, Timeout: 30 * time.Second, StartupTimeout: defaultStartupTimeout, } } func NewLLMPhraser(ctx context.Context, cfg Config) (*LLMPhraser, error) { ctx, cancel := context.WithCancel(ctx) p := &LLMPhraser{ cfg: cfg, client: &http.Client{Timeout: cfg.Timeout}, tmpl: loadNudgeTemplates(), spawnCtx: ctx, cancel: cancel, launch: spawnLlamaServer, probe: defaultProbe, live: liveModel{ModelPath: cfg.ModelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx}, } be, err := p.launch(ctx, cfg) if err != nil { cancel() return nil, err } p.be = be return p, nil } // NewLLMPhraserAt wires a phraser to a llama-server that someone else started // and owns. It spawns nothing, so Close does not kill anything. // // This exists for the phrasing scorer (internal/phraser/eval), which must // measure the phrasing against a shared llama-server without taking the model // load hit per run or killing a server another process depends on. The daemon // still uses NewLLMPhraser and still owns its own child process. func NewLLMPhraserAt(baseURL string, cfg Config) *LLMPhraser { return &LLMPhraser{ cfg: cfg, client: &http.Client{Timeout: cfg.Timeout}, tmpl: loadNudgeTemplates(), spawnCtx: context.Background(), cancel: func() {}, probe: defaultProbe, // launch stays nil: we did not start this server, so we must not stop it. // Swap therefore refuses here (ErrSwapNotOwned) instead of killing a // server another process depends on. be: borrowedBackend(strings.TrimSuffix(baseURL, "/")), live: liveModel{ModelPath: cfg.ModelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx}, } } // loadNudgeTemplates loads the Russian nudge templates. A broken template file // must not stop the daemon booting, so a failure logs and leaves the LLM path // in charge of nudges. func loadNudgeTemplates() *NudgeTemplates { nt, err := NewNudgeTemplates(nil) if err != nil { log.Printf("phraser: nudge templates unavailable, using the model: %v", err) return nil } return nt } // BaseURL is the llama-server this phraser talks to right now. It changes when // the model is swapped, so callers that cache it must register an observer // (OnSwap) rather than keeping the string forever. func (p *LLMPhraser) BaseURL() string { p.mu.Lock() defer p.mu.Unlock() if p.be == nil { return "" } return p.be.BaseURL() } func (p *LLMPhraser) Close() error { p.cancel() p.mu.Lock() be := p.be p.be = nil p.mu.Unlock() if be != nil { return be.Close() } return nil } func (p *LLMPhraser) PhraseNudge(ctx context.Context, c loop.Candidate) (delivery.PhrasedNudge, error) { // Templates first — see Config.LLMNudges for why this is the default. if !p.cfg.LLMNudges && p.tmpl != nil { return p.tmpl.PhraseNudge(ctx, c) } prompt := buildNudgePrompt(c) resp, err := p.chat(ctx, prompt) if err != nil { return delivery.PhrasedNudge{}, err } body, mood, perr := parseResponseMood(resp) if perr != nil { // Truncated JSON. Not a nudge — use the plain Russian fallback. log.Printf("phraser: PhraseNudge: %v", perr) body, mood = "", "" } if body == "" { // The model said nothing usable. Say it in Russian anyway — this text // goes straight to a Russian piper voice, so the old "water — care" // fallback was unspeakable. // // It says so in the log now (V-397). A clean parse of {"response":""} // left no trace at all: the nudge went out in template Russian and // nothing recorded that the model had been asked and had answered with // an empty field. log.Printf("phraser: PhraseNudge: empty response for rule %q, using the plain Russian fallback", c.Rule.Name) body = fallbackNudge(c) } if mood == "" { mood = "neutral" } return delivery.PhrasedNudge{Candidate: c, Body: body, Summary: body, Mood: mood}, nil } // PhraseQuery prompts the LLM with the user's utterance and matching notes to // compose a natural answer. On any LLM error it returns the fallback text — // "вот что я нашла: ", or "не знаю." with no notes — and the error // together. The daemon uses the text and keeps the turn alive; a caller that is // measuring counts the failure. Until Vikunja #397 the error was dropped, so a // dead server scored as bad phrasing. func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) { // Blank sources are no sources. A caller that hands over one empty string — // a page that fetched to nothing, a snippet trimmed away — used to take the // evidence branch and be told to answer from an empty list, which is the one // prompt guaranteed to make a small model fill the gap from memory. notes = nonEmpty(notes) if len(notes) == 0 { sys, prompt := p.knowledgePrompt(utterance) text, raw, err := p.generate(ctx, "phrase query (knowledge)", sys, prompt, 768) if err != nil { return UnknownFallback(), err } if raw == "" { return UnknownFallback(), errEmptyResponse } if text != "" { return text, nil } return raw, nil } sys, prompt := p.evidencePrompt(utterance, notes) text, raw, err := p.generate(ctx, "phrase query (evidence)", sys, prompt, 768) if err != nil { // Read the notes out rather than ship a broken fragment. return SourcesFallback(strings.Join(notes, "; ")), err } if text != "" { return text, nil } if raw == "" { // Same guard the knowledge branch above has had since it was written, // and this branch did not: the server answered and the model wrote // nothing, which returned ("", nil) — an empty answer reported as a // successful phrasing. The daemon's callers happen to check for the // empty string, so it read as a silent fallback there; the eval scored // it as bad phrasing rather than as the failure it is, and nothing on // either path logged that the model had produced no tokens. return SourcesFallback(strings.Join(notes, "; ")), errEmptyResponse } return raw, nil } // generate asks the model one question and reads the reply contract off the // answer. text is what she said, raw is what the model actually wrote — the // callers that ship bare prose when the model skipped the JSON need both. // // where names the path for the error, which every caller wraps identically. // An error here always means the caller must use its fallback: a transport // failure, or a generation that opened a JSON object and never closed it. func (p *LLMPhraser) generate(ctx context.Context, where, sys, user string, maxTokens int) (text, raw string, err error) { raw, err = p.chatWithSystem(ctx, sys, user, maxTokens) if err != nil { return "", raw, fmt.Errorf("%s: %w", where, err) } text, _, perr := parseResponseMood(raw) if perr != nil { return "", raw, fmt.Errorf("%s: %w", where, perr) } return text, raw, nil } // PhraseChat uses the LLM to respond conversationally, building a multi-turn // message array from dialogue history + the current user utterance. On any LLM // error it returns both ChatFallback and the error, on the same rule as // PhraseQuery: the fallback keeps the turn alive, the error stays visible. // chatUserMessage folds the prior turns and the current one into a single user // message, because some chat templates (Ministral and others) reject two user // turns in a row. That constraint is real; what was wrong is how it was met. // // The turns used to be joined with newlines and nothing else, so the model was // handed four unlabelled lines and no way to tell which one it was answering // (Vikunja #554). It answered an earlier one, or answered all of them at once: // asked "как дела" after a question about the telephone, she carried on about // the telephone. Four turns live for fifteen minutes, so the wrong line was // often several minutes old. // // The history holds only his own utterances, never her replies, so the label // says so and stays in the second person the persona requires. With no history // the message is the utterance alone, which is the common case and unchanged. func chatUserMessage(utterance string, history []dialogue.Turn) string { prior := make([]string, 0, len(history)) for _, t := range history { if s := strings.TrimSpace(t.Text); s != "" { prior = append(prior, "- "+s) } } if len(prior) == 0 { return strings.TrimSpace(utterance) } return "Раньше ты говорил:\n" + strings.Join(prior, "\n") + "\n\nОтветь только на то, что ты говоришь сейчас: " + strings.TrimSpace(utterance) } func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error) { sys := chatSystemPrompt(p.cfg.ContextBlock) msgs := []chatMsg{ {Role: "system", Content: sys}, } msgs = append(msgs, chatMsg{Role: "user", Content: chatUserMessage(utterance, history)}) resp, err := p.chatWithMessages(ctx, msgs, 768) if err != nil { return ChatFallback(), fmt.Errorf("phrase chat: %w", err) } text, _, perr := parseResponseMood(resp) if perr != nil { return ChatFallback(), fmt.Errorf("phrase chat: %w", perr) } if text != "" { return text, nil } // fallback: plain text without JSON if i := strings.IndexByte(resp, '\n'); i >= 0 { resp = resp[:i] } if resp = strings.TrimSpace(resp); resp == "" { // The model was up and wrote nothing. Same rule as PhraseQuery: the // fallback keeps the turn alive and the failure stays visible, rather // than ("", nil) telling the caller the chat path succeeded. return ChatFallback(), fmt.Errorf("phrase chat: %w", errEmptyResponse) } return resp, nil } func (p *LLMPhraser) PhraseReminder(ctx context.Context, d loop.ReminderDecision) (delivery.PhrasedReminder, error) { text := extractReminderText(d.Reminder.Payload) if text == "" { text = "reminder" } // Russian, like the other two prompts (Vikunja #404). Asking a model for a // Russian reply in English is asking it to switch languages mid-prompt, // and a 1.7B sometimes answers in the language it was asked in. The // persona rules and the JSON contract are not repeated here: this call // goes through chat(), so nudgeSystem already states both, and a second // statement of the same contract is one more thing that can drift. prompt := fmt.Sprintf( `Он поставил напоминание: "%s". Скажи это своими словами, коротко и мягко — одно предложение.`, text, ) resp, err := p.chat(ctx, prompt) if err != nil { return delivery.PhrasedReminder{}, err } body, mood, perr := parseResponseMood(resp) if perr != nil { // Truncated JSON. Fall through to the reminder's own text. log.Printf("phraser: PhraseReminder: %v", perr) body, mood = "", "" } if body == "" { // Same silence as PhraseNudge had (V-397): she reads the reminder's own // text out, which is fine, but nothing said the model had produced // nothing. log.Printf("phraser: PhraseReminder: empty response, reading the reminder text out instead") body = text } if mood == "" { mood = "neutral" } return delivery.PhrasedReminder{ Decision: d, Body: body, Summary: reminderSummary(body), Mood: mood, }, nil } func (p *LLMPhraser) chat(ctx context.Context, userPrompt string) (string, error) { return p.chatWithSystem(ctx, p.systemPrompt(), userPrompt, 512) } // PhraseSelf answers a question about her from her own description. Same // discipline as the evidence branch — say only what the text says — and a // different opener, because "вот что я нашла: я — твоя помощница" says she // looked herself up (Vikunja #555). She did not; this is the one subject she // does not have to read about. // // On any error it reads the description out rather than ship a fragment. That // is already a readable answer, which is why this needs no separate fallback. func (p *LLMPhraser) PhraseSelf(ctx context.Context, utterance, description string) (string, error) { sys, prompt := p.selfPrompt(utterance, description) text, _, err := p.generate(ctx, "phrase self", sys, prompt, 768) if err != nil { return description, err } if text != "" { return text, nil } return description, nil }