diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index b1a3e16..67970c9 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -46,6 +46,12 @@ type LLMPhraser struct { launch func(ctx context.Context, cfg Config) (backend, error) probe func(ctx context.Context, base string) (string, error) + // remote — the workstation model, when one is configured. Set once at wiring + // time by UseRemote and read on every phrasing call. nil ⇒ every call goes to + // the resident llama-server this phraser owns, which is the whole deploy + // before a `workstation` block exists. See world.go. + remote Remote + // swapMu — single-flight around Swap. Held for the whole swap, including the // model load, so two concurrent swap requests can never both be loading. swapMu sync.Mutex @@ -363,10 +369,7 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes [] // prompt guaranteed to make a small model fill the gap from memory. notes = nonEmpty(notes) if len(notes) == 0 { - // General knowledge — no notes to ground the answer. The system - // prompt is the single tested source in router.KnowledgePrompt. - sys := persona.Prepend(p.cfg.ContextBlock, router.KnowledgePrompt()) - prompt := fmt.Sprintf("Пользователь спрашивает: \"%s\".", utterance) + sys, prompt := p.knowledgePrompt(utterance) resp, err := p.chatWithSystem(ctx, sys, prompt, 768) if err != nil || resp == "" { return "не знаю.", nil @@ -381,11 +384,7 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes [] } return resp, nil } - sys := p.querySystemPrompt() - prompt := fmt.Sprintf( - "Он спрашивает: \"%s\"\n\nИсточники:\n%s\nОтветь ему коротко и своими словами, опираясь только на эти источники. Если ответа в них нет — так и скажи.", - utterance, evidenceBlock(notes), - ) + sys, prompt := p.evidencePrompt(utterance, notes) resp, err := p.chatWithSystem(ctx, sys, prompt, 768) text, _, perr := parseResponseMood(resp) if err != nil || perr != nil { @@ -481,6 +480,17 @@ func chatSystemPrompt(block func() string) string { // the LLM completion endpoint. Like chatWithSystem but for an arbitrary message // slice — the caller owns the system prompt placement. func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTokens int) (string, error) { + // Same silent preference as chatWithSystem, when the array is the shape + // llm.Req can carry: one system turn and one user turn. PhraseChat already + // folds the history into a single user message (some chat templates reject + // consecutive user turns), so today that is every call. A longer array goes + // to the resident model rather than get flattened here, because flattening a + // conversation is a decision its owner should make. + if len(msgs) == 2 && msgs[0].Role == "system" && msgs[1].Role == "user" { + if out, ok := p.remoteChat(ctx, msgs[0].Content, msgs[1].Content, maxTokens); ok { + return out, nil + } + } base, release, err := p.acquire() if err != nil { return "", err @@ -640,6 +650,12 @@ func (p *LLMPhraser) chat(ctx context.Context, userPrompt string) (string, error } func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, maxTokens int) (string, error) { + // The workstation model first when it will take work, and silently: every + // caller of this helper is on the silent half of the degradation rule. It + // answering is not news, and it being asleep is not news either. + if out, ok := p.remoteChat(ctx, system, user, maxTokens); ok { + return out, nil + } base, release, err := p.acquire() if err != nil { return "", err @@ -733,6 +749,27 @@ func (p *LLMPhraser) systemPrompt() string { return persona.Prepend(p.cfg.ContextBlock, nudgeSystem) } +// knowledgePrompt — the no-sources branch: a world question, answered from +// weights alone. The system prompt is the single tested source in +// router.KnowledgePrompt. +// +// Split out of PhraseQuery so PhraseWorld sends the workstation model the same +// bytes the resident model gets. Prompt parity across two models is a stated +// constraint (CLAUDE.md), and two copies of a prompt is how it stops holding. +func (p *LLMPhraser) knowledgePrompt(utterance string) (sys, user string) { + return persona.Prepend(p.cfg.ContextBlock, router.KnowledgePrompt()), + fmt.Sprintf("Пользователь спрашивает: \"%s\".", utterance) +} + +// evidencePrompt — the sources branch: read these, add nothing. Shared with +// PhraseWorld for the same reason as knowledgePrompt. +func (p *LLMPhraser) evidencePrompt(utterance string, notes []string) (sys, user string) { + return p.querySystemPrompt(), fmt.Sprintf( + "Он спрашивает: \"%s\"\n\nИсточники:\n%s\nОтветь ему коротко и своими словами, опираясь только на эти источники. Если ответа в них нет — так и скажи.", + utterance, evidenceBlock(notes), + ) +} + // querySystemPrompt returns the system prompt for the evidence branch of // PhraseQuery. Prepends the configured persona when set. // diff --git a/internal/phraser/world.go b/internal/phraser/world.go new file mode 100644 index 0000000..a021a4b --- /dev/null +++ b/internal/phraser/world.go @@ -0,0 +1,137 @@ +package phraser + +import ( + "context" + "errors" + "log" + + "github.com/kami/maven/internal/llm" +) + +// Remote — the workstation model, seen from the phraser. `*llm.Pair` satisfies +// it, and a test fake satisfies it in three lines. +// +// Only the refusing half of Pair is here on purpose. Pair.Complete falls back to +// its own floor client, and the phraser already owns a floor: the llama-server it +// spawned. Two floors under one call is one too many, so the phraser asks whether +// the remote will take work, uses it when it will, and otherwise does exactly +// what it did before this file existed. +type Remote interface { + // Available is an atomic read of a cached probe, so it is free to call per + // turn. See llm.Pair. + Available() bool + // CompleteRemote runs on the workstation or returns ErrRemoteUnavailable. It + // never falls back. + CompleteRemote(ctx context.Context, r llm.Req) (string, error) +} + +// ErrNoWorldModel — a world question was asked, a workstation model is +// configured to answer it, and that machine is not answering. The caller turns +// this into a gap he is told about ("не могу сейчас"), never into an answer from +// the resident model. +// +// This is the naming half of the degradation rule in docs/offload.md. The +// resident Qwen3-1.7B does not answer a world question worse than the 12B, it +// invents: measured, the workstation model scores knowledge 9/9 on the talk +// fixture against the resident model's confabulations +// (docs/evals/2026-08-02-workstation-gemma4-12b.md). +var ErrNoWorldModel = errors.New("phraser: no world model available") + +// chatTemperature — what the phraser's own transport has always sampled at. +// Named so the remote path cannot drift from it silently. Whether 0.7 is right +// at all is Vikunja #402, and answering that here would hide a phrasing change +// inside a routing change. +const chatTemperature = 0.7 + +// UseRemote points the phraser at the workstation model. Wiring time only, once, +// before anything phrases: the field is read without a lock on every call +// because a per-turn lock to answer a question that changes at deploy time is +// not worth paying for. +// +// A nil remote is the normal state of a box with no `workstation` block, and it +// must behave exactly as the box behaved before this seam existed. +func (p *LLMPhraser) UseRemote(r Remote) { + p.remote = r +} + +// PhraseWorld answers a question about the world — either from the model's own +// knowledge (no sources) or from a passage someone fetched (a live search, a ZIM +// article, a page he named). Three outcomes, and the middle one is the point: +// +// - No workstation configured. The resident model answers, exactly as it does +// today. Naming a gap needs a gap: on a box that never had a second model, +// refusing every world question would remove a capability he has now. +// - Workstation configured and taking work. It answers. +// - Workstation configured and down. ErrNoWorldModel, and the caller says so. +// +// The prompts are the ones PhraseQuery uses, built by the same two functions, so +// the two models are asked the same question in the same words. +func (p *LLMPhraser) PhraseWorld(ctx context.Context, utterance string, sources []string) (string, error) { + sources = nonEmpty(sources) + if p.remote == nil { + return p.PhraseQuery(ctx, utterance, sources) + } + var sys, user string + if len(sources) == 0 { + sys, user = p.knowledgePrompt(utterance) + } else { + sys, user = p.evidencePrompt(utterance, sources) + } + if !p.remote.Available() { + return "", ErrNoWorldModel + } + resp, err := p.remote.CompleteRemote(ctx, llm.Req{ + System: sys, + User: user, + Grammar: p.grammar(), + MaxTokens: 768, + Temperature: chatTemperature, + }) + if err != nil { + // The cached probe was one interval stale, or the card went away + // mid-request. Either way this is the gap, not an error to log and + // paper over with the smaller model. + log.Printf("phraser: world model: %v", err) + return "", errors.Join(ErrNoWorldModel, err) + } + resp = stripThink(resp) + text, _, perr := parseResponseMood(resp) + if perr != nil { + log.Printf("phraser: PhraseWorld: %v", perr) + return "", errors.Join(ErrNoWorldModel, perr) + } + if text != "" { + return text, nil + } + if resp == "" { + return "", ErrNoWorldModel + } + return resp, nil +} + +// remoteChat is the silent half, for the phrasing paths where the workstation +// model is only better: a nudge, a reminder, a reply, a question answered from +// his own notes. It reports whether it answered; it never reports why not, +// because the caller's next move is the resident model either way. +// +// He is not told which of the two models phrased his reply. That is the rule. +func (p *LLMPhraser) remoteChat(ctx context.Context, system, user string, maxTokens int) (string, bool) { + if p.remote == nil || !p.remote.Available() { + return "", false + } + out, err := p.remote.CompleteRemote(ctx, llm.Req{ + System: system, + User: user, + Grammar: p.grammar(), + MaxTokens: maxTokens, + Temperature: chatTemperature, + }) + if err != nil { + log.Printf("phraser: workstation model declined, phrasing here instead: %v", err) + return "", false + } + if out = stripThink(out); out == "" { + return "", false + } + return out, true +}