Maven/internal/phraser/llmphraser.go

package phraser

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/kami/maven/internal/delivery"
	"github.com/kami/maven/internal/dialogue"
	"github.com/kami/maven/internal/loop"
)

// errEmptyResponse — the server answered and said nothing. Separate from a
// transport failure: the model is up and produced no tokens, which is still not
// an answer and must not score as one.
var errEmptyResponse = errors.New("phraser: empty response from the model")

type LLMPhraser struct {
	cfg    Config
	client *http.Client

	// tmpl — the hand-written Russian nudges. Default path for nudges; see
	// Config.LLMNudges. nil only if the template file failed to load.
	tmpl *NudgeTemplates

	// spawnCtx — the parent of every llama-server this phraser starts, i.e. the
	// daemon's own context. Deliberately NOT the per-request context of the call
	// that asked for a model swap: that one is cancelled the moment the request
	// returns, which would kill the model it had just loaded.
	spawnCtx context.Context
	cancel   context.CancelFunc

	// launch / probe — the two side effects of a swap, injectable so the swap
	// logic is testable without a real llama-server and a real model file.
	// launch is nil when this phraser does not own its server (NewLLMPhraserAt),
	// which is also what makes Swap refuse there.
	launch func(ctx context.Context, cfg Config) (backend, error)
	probe  func(ctx context.Context, base string) (string, error)

	// remote — the workstation model, when one is configured. Set once at wiring
	// time by UseRemote and read on every phrasing call. nil ⇒ every call goes to
	// the resident llama-server this phraser owns, which is the whole deploy
	// before a `workstation` block exists. See world.go.
	remote Remote

	// swapMu — single-flight around Swap. Held for the whole swap, including the
	// model load, so two concurrent swap requests can never both be loading.
	swapMu sync.Mutex

	// mu guards everything below: the live backend, the swap gate and the
	// in-flight request count. See acquire/quiesce in swap.go.
	mu        sync.Mutex
	be        backend
	live      liveModel
	swapping  bool
	inflight  int
	observers []func(baseURL string)
}

// liveModel — what is actually loaded right now. Distinct from Config, which
// stays immutable after construction: a swap changes these three fields and
// nothing else, so no reader of cfg (prompts, grammar, timeouts) races a swap.
type liveModel struct {
	ModelPath  string
	NGpuLayers int
	NCtx       int
}

type Config struct {
	ModelPath  string
	BinPath    string
	Listen     string
	NGpuLayers int
	NCtx       int
	Timeout    time.Duration

	// StartupTimeout bounds the wait for llama-server to print the address it
	// listens on. A config field and not a constant because the box may
	// legitimately need longer: a cold 1.7B loading off a spinning disk can
	// outrun a minute, and until this existed that returned "server did not
	// start within 60s" with no way to raise it.
	//
	// 0 ⇒ defaultStartupTimeout.
	StartupTimeout time.Duration

	// CacheRAMMiB bounds llama-server's prompt cache, which is what actually ate
	// this box. Measured on homesrv 2026-08-03: the server's own default limit is
	// 8192 MiB, it stores the full KV state of every idle slot it evicts (112 kiB
	// per token, so 166 MiB for one 1521-token prompt), and RSS climbed by that
	// much per distinct prompt until it hit 7.9 GB and half a gigabyte went to
	// swap. Weights are only 1.1 GB and mmapped, and -ngl 99 costs almost no RSS
	// because RADV keeps device memory outside the process.
	//
	// 0 ⇒ the flag is not passed and the server's own 8 GiB default applies. That
	// is the escape hatch for a llama-server too old to know --cache-ram, not a
	// recommendation. See docs/evals/2026-08-03-llama-prompt-cache.md.
	CacheRAMMiB int

	// ContextBlock renders the shared context block (who he is, how to
	// address him, the time) fresh for each turn. See internal/persona.
	// nil ⇒ no block, the prompts stand alone.
	ContextBlock func() string

	// LLMNudges puts the model back in charge of nudge wording.
	//
	// Off by default, and that is a deliberate deprecation of LLM-phrased
	// nudges: hand-written templates (nudges_ru_v1.json) word every nudge now.
	// A nudge has nothing to be creative about, and measured over many runs the
	// 0.8B broke the persona (formal "вы", plural imperatives, masculine
	// self-reference) and invented facts and units. Templates score 15/15 on the
	// nudge fixture, the model 11-13/15.
	//
	// The LLM path is kept, not deleted: flip this on to get it back. Chat,
	// query and reminder phrasing are untouched and still go through the model.
	LLMNudges bool

	// Temperature — what every phrasing call samples at. 0 ⇒ 0.7, which is
	// what this transport has always sent.
	//
	// A field rather than a constant so the talk fixture can sweep it
	// (Vikunja #402). Sampling is a dial, and a dial nobody can turn from
	// outside the package cannot be measured, only argued about.
	Temperature float64

	// NoGrammar turns the GBNF constraint off (zero value ⇒ grammar ON).
	// The escape hatch exists because the target resident model — the
	// locally CPT'd Qwen3-1.7B — does not exist yet: if its chat template
	// ever fights the grammar, the fix should be a config flip on the
	// deploy box, not a code change and a rebuild.
	NoGrammar bool
}

func DefaultConfig(modelPath string) Config {
	return Config{
		ModelPath:  modelPath,
		BinPath:    "llama-server",
		Listen:     "127.0.0.1:0",
		NGpuLayers: -1,
		NCtx:       2048,
		// 512 MiB caps total RSS near 1 GB and still holds several recent prompts.
		CacheRAMMiB: 512,
		Timeout:     30 * time.Second,

		StartupTimeout: defaultStartupTimeout,
	}
}

func NewLLMPhraser(ctx context.Context, cfg Config) (*LLMPhraser, error) {
	ctx, cancel := context.WithCancel(ctx)
	p := &LLMPhraser{
		cfg:      cfg,
		client:   &http.Client{Timeout: cfg.Timeout},
		tmpl:     loadNudgeTemplates(),
		spawnCtx: ctx,
		cancel:   cancel,
		launch:   spawnLlamaServer,
		probe:    defaultProbe,
		live:     liveModel{ModelPath: cfg.ModelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx},
	}
	be, err := p.launch(ctx, cfg)
	if err != nil {
		cancel()
		return nil, err
	}
	p.be = be
	return p, nil
}

// NewLLMPhraserAt wires a phraser to a llama-server that someone else started
// and owns. It spawns nothing, so Close does not kill anything.
//
// This exists for the phrasing scorer (internal/phraser/eval), which must
// measure the phrasing against a shared llama-server without taking the model
// load hit per run or killing a server another process depends on. The daemon
// still uses NewLLMPhraser and still owns its own child process.
func NewLLMPhraserAt(baseURL string, cfg Config) *LLMPhraser {
	return &LLMPhraser{
		cfg:      cfg,
		client:   &http.Client{Timeout: cfg.Timeout},
		tmpl:     loadNudgeTemplates(),
		spawnCtx: context.Background(),
		cancel:   func() {},
		probe:    defaultProbe,
		// launch stays nil: we did not start this server, so we must not stop it.
		// Swap therefore refuses here (ErrSwapNotOwned) instead of killing a
		// server another process depends on.
		be:   borrowedBackend(strings.TrimSuffix(baseURL, "/")),
		live: liveModel{ModelPath: cfg.ModelPath, NGpuLayers: cfg.NGpuLayers, NCtx: cfg.NCtx},
	}
}

// loadNudgeTemplates loads the Russian nudge templates. A broken template file
// must not stop the daemon booting, so a failure logs and leaves the LLM path
// in charge of nudges.
func loadNudgeTemplates() *NudgeTemplates {
	nt, err := NewNudgeTemplates(nil)
	if err != nil {
		log.Printf("phraser: nudge templates unavailable, using the model: %v", err)
		return nil
	}
	return nt
}

// BaseURL is the llama-server this phraser talks to right now. It changes when
// the model is swapped, so callers that cache it must register an observer
// (OnSwap) rather than keeping the string forever.
func (p *LLMPhraser) BaseURL() string {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.be == nil {
		return ""
	}
	return p.be.BaseURL()
}

func (p *LLMPhraser) Close() error {
	p.cancel()
	p.mu.Lock()
	be := p.be
	p.be = nil
	p.mu.Unlock()
	if be != nil {
		return be.Close()
	}
	return nil
}

func (p *LLMPhraser) PhraseNudge(ctx context.Context, c loop.Candidate) (delivery.PhrasedNudge, error) {
	// Templates first — see Config.LLMNudges for why this is the default.
	if !p.cfg.LLMNudges && p.tmpl != nil {
		return p.tmpl.PhraseNudge(ctx, c)
	}
	prompt := buildNudgePrompt(c)
	resp, err := p.chat(ctx, prompt)
	if err != nil {
		return delivery.PhrasedNudge{}, err
	}
	body, mood, perr := parseResponseMood(resp)
	if perr != nil {
		// Truncated JSON. Not a nudge — use the plain Russian fallback.
		log.Printf("phraser: PhraseNudge: %v", perr)
		body, mood = "", ""
	}
	if body == "" {
		// The model said nothing usable. Say it in Russian anyway — this text
		// goes straight to a Russian piper voice, so the old "water — care"
		// fallback was unspeakable.
		//
		// It says so in the log now (V-397). A clean parse of {"response":""}
		// left no trace at all: the nudge went out in template Russian and
		// nothing recorded that the model had been asked and had answered with
		// an empty field.
		log.Printf("phraser: PhraseNudge: empty response for rule %q, using the plain Russian fallback", c.Rule.Name)
		body = fallbackNudge(c)
	}
	if mood == "" {
		mood = "neutral"
	}
	return delivery.PhrasedNudge{Candidate: c, Body: body, Summary: body, Mood: mood}, nil
}

// PhraseQuery prompts the LLM with the user's utterance and matching notes to
// compose a natural answer. On any LLM error it returns the fallback text —
// "вот что я нашла: <notes>", or "не знаю." with no notes — and the error
// together. The daemon uses the text and keeps the turn alive; a caller that is
// measuring counts the failure. Until Vikunja #397 the error was dropped, so a
// dead server scored as bad phrasing.
func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) {
	// Blank sources are no sources. A caller that hands over one empty string —
	// a page that fetched to nothing, a snippet trimmed away — used to take the
	// evidence branch and be told to answer from an empty list, which is the one
	// prompt guaranteed to make a small model fill the gap from memory.
	notes = nonEmpty(notes)
	if len(notes) == 0 {
		sys, prompt := p.knowledgePrompt(utterance)
		text, raw, err := p.generate(ctx, "phrase query (knowledge)", sys, prompt, 768)
		if err != nil {
			return UnknownFallback(), err
		}
		if raw == "" {
			return UnknownFallback(), errEmptyResponse
		}
		if text != "" {
			return text, nil
		}
		return raw, nil
	}
	sys, prompt := p.evidencePrompt(utterance, notes)
	text, raw, err := p.generate(ctx, "phrase query (evidence)", sys, prompt, 768)
	if err != nil {
		// Read the notes out rather than ship a broken fragment.
		return SourcesFallback(strings.Join(notes, "; ")), err
	}
	if text != "" {
		return text, nil
	}
	if raw == "" {
		// Same guard the knowledge branch above has had since it was written,
		// and this branch did not: the server answered and the model wrote
		// nothing, which returned ("", nil) — an empty answer reported as a
		// successful phrasing. The daemon's callers happen to check for the
		// empty string, so it read as a silent fallback there; the eval scored
		// it as bad phrasing rather than as the failure it is, and nothing on
		// either path logged that the model had produced no tokens.
		return SourcesFallback(strings.Join(notes, "; ")), errEmptyResponse
	}
	return raw, nil
}

// generate asks the model one question and reads the reply contract off the
// answer. text is what she said, raw is what the model actually wrote — the
// callers that ship bare prose when the model skipped the JSON need both.
//
// where names the path for the error, which every caller wraps identically.
// An error here always means the caller must use its fallback: a transport
// failure, or a generation that opened a JSON object and never closed it.
func (p *LLMPhraser) generate(ctx context.Context, where, sys, user string, maxTokens int) (text, raw string, err error) {
	raw, err = p.chatWithSystem(ctx, sys, user, maxTokens)
	if err != nil {
		return "", raw, fmt.Errorf("%s: %w", where, err)
	}
	text, _, perr := parseResponseMood(raw)
	if perr != nil {
		return "", raw, fmt.Errorf("%s: %w", where, perr)
	}
	return text, raw, nil
}

// PhraseChat uses the LLM to respond conversationally, building a multi-turn
// message array from dialogue history + the current user utterance. On any LLM
// error it returns both ChatFallback and the error, on the same rule as
// PhraseQuery: the fallback keeps the turn alive, the error stays visible.
// chatUserMessage folds the prior turns and the current one into a single user
// message, because some chat templates (Ministral and others) reject two user
// turns in a row. That constraint is real; what was wrong is how it was met.
//
// The turns used to be joined with newlines and nothing else, so the model was
// handed four unlabelled lines and no way to tell which one it was answering
// (Vikunja #554). It answered an earlier one, or answered all of them at once:
// asked "как дела" after a question about the telephone, she carried on about
// the telephone. Four turns live for fifteen minutes, so the wrong line was
// often several minutes old.
//
// The history holds only his own utterances, never her replies, so the label
// says so and stays in the second person the persona requires. With no history
// the message is the utterance alone, which is the common case and unchanged.
func chatUserMessage(utterance string, history []dialogue.Turn) string {
	prior := make([]string, 0, len(history))
	for _, t := range history {
		if s := strings.TrimSpace(t.Text); s != "" {
			prior = append(prior, "- "+s)
		}
	}
	if len(prior) == 0 {
		return strings.TrimSpace(utterance)
	}
	return "Раньше ты говорил:\n" + strings.Join(prior, "\n") +
		"\n\nОтветь только на то, что ты говоришь сейчас: " + strings.TrimSpace(utterance)
}

func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error) {
	sys := chatSystemPrompt(p.cfg.ContextBlock)
	msgs := []chatMsg{
		{Role: "system", Content: sys},
	}
	msgs = append(msgs, chatMsg{Role: "user", Content: chatUserMessage(utterance, history)})

	resp, err := p.chatWithMessages(ctx, msgs, 768)
	if err != nil {
		return ChatFallback(), fmt.Errorf("phrase chat: %w", err)
	}
	text, _, perr := parseResponseMood(resp)
	if perr != nil {
		return ChatFallback(), fmt.Errorf("phrase chat: %w", perr)
	}
	if text != "" {
		return text, nil
	}
	// fallback: plain text without JSON
	if i := strings.IndexByte(resp, '\n'); i >= 0 {
		resp = resp[:i]
	}
	if resp = strings.TrimSpace(resp); resp == "" {
		// The model was up and wrote nothing. Same rule as PhraseQuery: the
		// fallback keeps the turn alive and the failure stays visible, rather
		// than ("", nil) telling the caller the chat path succeeded.
		return ChatFallback(), fmt.Errorf("phrase chat: %w", errEmptyResponse)
	}
	return resp, nil
}

func (p *LLMPhraser) PhraseReminder(ctx context.Context, d loop.ReminderDecision) (delivery.PhrasedReminder, error) {
	text := extractReminderText(d.Reminder.Payload)
	if text == "" {
		text = "reminder"
	}

	// Russian, like the other two prompts (Vikunja #404). Asking a model for a
	// Russian reply in English is asking it to switch languages mid-prompt,
	// and a 1.7B sometimes answers in the language it was asked in. The
	// persona rules and the JSON contract are not repeated here: this call
	// goes through chat(), so nudgeSystem already states both, and a second
	// statement of the same contract is one more thing that can drift.
	prompt := fmt.Sprintf(
		`Он поставил напоминание: "%s". Скажи это своими словами, коротко и мягко — одно предложение.`,
		text,
	)
	resp, err := p.chat(ctx, prompt)
	if err != nil {
		return delivery.PhrasedReminder{}, err
	}
	body, mood, perr := parseResponseMood(resp)
	if perr != nil {
		// Truncated JSON. Fall through to the reminder's own text.
		log.Printf("phraser: PhraseReminder: %v", perr)
		body, mood = "", ""
	}
	if body == "" {
		// Same silence as PhraseNudge had (V-397): she reads the reminder's own
		// text out, which is fine, but nothing said the model had produced
		// nothing.
		log.Printf("phraser: PhraseReminder: empty response, reading the reminder text out instead")
		body = text
	}
	if mood == "" {
		mood = "neutral"
	}
	return delivery.PhrasedReminder{
		Decision: d, Body: body, Summary: reminderSummary(body), Mood: mood,
	}, nil
}

func (p *LLMPhraser) chat(ctx context.Context, userPrompt string) (string, error) {
	return p.chatWithSystem(ctx, p.systemPrompt(), userPrompt, 512)
}

// PhraseSelf answers a question about her from her own description. Same
// discipline as the evidence branch — say only what the text says — and a
// different opener, because "вот что я нашла: я — твоя помощница" says she
// looked herself up (Vikunja #555). She did not; this is the one subject she
// does not have to read about.
//
// On any error it reads the description out rather than ship a fragment. That
// is already a readable answer, which is why this needs no separate fallback.
func (p *LLMPhraser) PhraseSelf(ctx context.Context, utterance, description string) (string, error) {
	sys, prompt := p.selfPrompt(utterance, description)
	text, _, err := p.generate(ctx, "phrase self", sys, prompt, 768)
	if err != nil {
		return description, err
	}
	if text != "" {
		return text, nil
	}
	return description, nil
}