Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c47881106e | |||
| 9a70f7378b | |||
| b18f608594 | |||
| d1f8a734c5 | |||
| 71041029e2 | |||
| 35018226ef | |||
| 8833a9c76b | |||
| 6c07409452 | |||
| 1c2541f7d6 | |||
| 9e25f18a3e |
@@ -40,6 +40,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"log"
|
"log"
|
||||||
|
|
||||||
|
"github.com/kami/maven/internal/phraser"
|
||||||
"github.com/kami/maven/internal/router"
|
"github.com/kami/maven/internal/router"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -58,10 +59,14 @@ func (h *reactiveHandler) actionChat(ctx context.Context, dec router.Decision) s
|
|||||||
// Conversational: build history from dialogue session (prior user turns)
|
// Conversational: build history from dialogue session (prior user turns)
|
||||||
// and let the LLM respond from general knowledge + context.
|
// and let the LLM respond from general knowledge + context.
|
||||||
history := h.chatHistory()
|
history := h.chatHistory()
|
||||||
|
// The phraser hands back its own fallback text alongside the error, so the
|
||||||
|
// turn survives a dead server and the failure still reaches the log.
|
||||||
reply, err := h.phraser.PhraseChat(ctx, dec.Utterance, history)
|
reply, err := h.phraser.PhraseChat(ctx, dec.Utterance, history)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("voice: chat: %v", err)
|
log.Printf("voice: chat: %v", err)
|
||||||
return "поговорили."
|
}
|
||||||
|
if reply == "" {
|
||||||
|
return phraser.ChatFallback
|
||||||
}
|
}
|
||||||
return reply
|
return reply
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -445,7 +445,13 @@ func (h *reactiveHandler) queryMemory(ctx context.Context, t *queryTurn) (string
|
|||||||
// A note is phrased in Maven's voice; a fact is read back as it was
|
// A note is phrased in Maven's voice; a fact is read back as it was
|
||||||
// stored.
|
// stored.
|
||||||
if hit.Meta["type"] == "note" {
|
if hit.Meta["type"] == "note" {
|
||||||
if reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{text}); perr == nil && reply != "" {
|
reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{text})
|
||||||
|
switch {
|
||||||
|
case perr != nil:
|
||||||
|
// Reading the note back verbatim beats the phraser's own fallback,
|
||||||
|
// which only wraps the same text in "вот что я нашла:".
|
||||||
|
log.Printf("voice: recall phrase: %v", perr)
|
||||||
|
case reply != "":
|
||||||
return reply, true
|
return reply, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -100,7 +100,7 @@ func (l llmCompleter) Complete(ctx context.Context, system, user string) (string
|
|||||||
// grammar, or a llama-server too old to honour one, gets the plain text it used
|
// grammar, or a llama-server too old to honour one, gets the plain text it used
|
||||||
// to get rather than an empty meeting summary.
|
// to get rather than an empty meeting summary.
|
||||||
func unwrapSummary(raw string) string {
|
func unwrapSummary(raw string) string {
|
||||||
s := stripThink(strings.TrimSpace(raw))
|
s := phraser.StripThink(strings.TrimSpace(raw))
|
||||||
start := strings.Index(s, "{")
|
start := strings.Index(s, "{")
|
||||||
end := strings.LastIndex(s, "}")
|
end := strings.LastIndex(s, "}")
|
||||||
if start < 0 || end <= start {
|
if start < 0 || end <= start {
|
||||||
|
|||||||
+11
-100
@@ -2,122 +2,33 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/kami/maven/internal/llm"
|
|
||||||
"github.com/kami/maven/internal/persona"
|
|
||||||
"github.com/kami/maven/internal/phraser"
|
"github.com/kami/maven/internal/phraser"
|
||||||
"github.com/kami/maven/internal/router"
|
"github.com/kami/maven/internal/router"
|
||||||
"github.com/kami/maven/internal/voice"
|
"github.com/kami/maven/internal/voice"
|
||||||
)
|
)
|
||||||
|
|
||||||
// completer is the LLM seam for the replier (subset of router.Completer).
|
// llmReplier is the daemon-side wiring around phraser.Replier: it owns the
|
||||||
// *llm.Client satisfies it.
|
// deterministic floor, and nothing else. The phrasing itself, the prompt and the
|
||||||
type completer interface {
|
// output parsing live in internal/phraser so the eval can score them (#396).
|
||||||
Complete(ctx context.Context, r llm.Req) (string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// llmReplier phrases reactive confirmations with the resident model
|
|
||||||
// (Qwen3-1.7B). Stub is the
|
|
||||||
// floor on any error (offline-safe). Maven speaks as "she", feminine RU.
|
|
||||||
type llmReplier struct {
|
type llmReplier struct {
|
||||||
c completer
|
p *phraser.Replier
|
||||||
stub *voice.StubReplier
|
stub *voice.StubReplier
|
||||||
|
|
||||||
// block renders the shared context block per turn (who he is, the time).
|
|
||||||
// nil ⇒ the prompt stands alone.
|
|
||||||
block func() string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newLLMReplier(c completer, block func() string) *llmReplier {
|
func newLLMReplier(c phraser.Completer, block func() string) *llmReplier {
|
||||||
return &llmReplier{c: c, stub: voice.NewStubReplier(), block: block}
|
return &llmReplier{p: phraser.NewReplier(c, block), stub: voice.NewStubReplier()}
|
||||||
}
|
}
|
||||||
|
|
||||||
const replySystem = `Ты — Maven, домашняя ассистентка (о себе — в женском роде). Владелец — мужчина, говоришь с ним на "ты", в единственном числе; никогда не "вы"/"ваш" и не "он"/"его". Подтверди действие РОВНО ОДНИМ коротким предложением (≤120 символов), по-русски, спокойно и без официальных формулировок. Не задавай вопросов, не повторяй слова, не добавляй ничего после точки. Отвечай ТОЛЬКО одним объектом JSON с полями "response" (текст) и "mood" (ровно одно из: neutral, happy, thinking, tired, confused).
|
// Reply never fails: a clarify, a model error and an unusable generation all
|
||||||
Пример: {"response": "Записала, что ты выпил стакан воды.", "mood": "neutral"}
|
// answer from the stub, which is what keeps a turn from breaking on the model.
|
||||||
Никогда не пиши "..." в поле response.`
|
|
||||||
|
|
||||||
func (r *llmReplier) Reply(d router.Decision) string {
|
func (r *llmReplier) Reply(d router.Decision) string {
|
||||||
if d.Clarify {
|
if d.Clarify {
|
||||||
return r.stub.Reply(d)
|
return r.stub.Reply(d)
|
||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
out, err := r.p.PhraseReply(context.Background(), d)
|
||||||
defer cancel()
|
if err != nil || out == "" {
|
||||||
out, err := r.c.Complete(ctx, llm.Req{
|
|
||||||
System: persona.Prepend(r.block, replySystem),
|
|
||||||
User: replyContext(d),
|
|
||||||
Grammar: phraser.ResponseGrammar,
|
|
||||||
MaxTokens: 512,
|
|
||||||
RepeatPenalty: 1.3,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return r.stub.Reply(d)
|
return r.stub.Reply(d)
|
||||||
}
|
}
|
||||||
out = stripThink(out)
|
return out
|
||||||
if response, _ := parseResponseMood(out); response != "" {
|
|
||||||
return response
|
|
||||||
}
|
|
||||||
// fallback: try plain-text parsing
|
|
||||||
if out = firstSentence(out); out != "" {
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
return r.stub.Reply(d)
|
|
||||||
}
|
|
||||||
|
|
||||||
// firstSentence trims the model's output to a single clean confirmation: first
|
|
||||||
// line, first sentence, whitespace-normalized — the last-line defense against a
|
|
||||||
// small model that rambles past the first period despite the prompt + stop.
|
|
||||||
// stripThink removes the <think> block that Thinking-variant models emit.
|
|
||||||
func stripThink(s string) string {
|
|
||||||
if i := strings.LastIndex(s, "</think>"); i >= 0 {
|
|
||||||
s = strings.TrimSpace(s[i+8:])
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
func firstSentence(s string) string {
|
|
||||||
s = strings.TrimSpace(s)
|
|
||||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
|
||||||
s = s[:i]
|
|
||||||
}
|
|
||||||
// keep up to and including the first sentence-ending punctuation.
|
|
||||||
if i := strings.IndexAny(s, ".!?"); i >= 0 {
|
|
||||||
s = s[:i+1]
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(s)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseResponseMood extracts {"response","mood"} from LLM output, tolerant
|
|
||||||
// of thinking tokens and extra text before/after the JSON block.
|
|
||||||
func parseResponseMood(raw string) (response, mood string) {
|
|
||||||
cleaned := strings.TrimSpace(raw)
|
|
||||||
start := strings.Index(cleaned, "{")
|
|
||||||
end := strings.LastIndex(cleaned, "}")
|
|
||||||
if start < 0 || end < 0 || end <= start {
|
|
||||||
return "", ""
|
|
||||||
}
|
|
||||||
var parsed struct {
|
|
||||||
Response string `json:"response"`
|
|
||||||
Mood string `json:"mood"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal([]byte(cleaned[start:end+1]), &parsed); err != nil {
|
|
||||||
return "", ""
|
|
||||||
}
|
|
||||||
return parsed.Response, parsed.Mood
|
|
||||||
}
|
|
||||||
|
|
||||||
// replyContext renders the decision into a compact RU description for the model.
|
|
||||||
func replyContext(d router.Decision) string {
|
|
||||||
switch d.Intent {
|
|
||||||
case router.IntentFact:
|
|
||||||
return "записала факт: " + d.Slots.Key + " " + d.Slots.Value
|
|
||||||
case router.IntentNote:
|
|
||||||
return "сохранила заметку: " + d.Slots.Text
|
|
||||||
case router.IntentReminder:
|
|
||||||
return "поставила напоминание: " + d.Slots.Text
|
|
||||||
default:
|
|
||||||
return string(d.Intent) + ": " + d.Slots.Text
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,28 +5,22 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/kami/maven/internal/llm"
|
"github.com/kami/maven/internal/llm"
|
||||||
"github.com/kami/maven/internal/phraser"
|
|
||||||
"github.com/kami/maven/internal/router"
|
"github.com/kami/maven/internal/router"
|
||||||
"github.com/kami/maven/internal/voice"
|
"github.com/kami/maven/internal/voice"
|
||||||
)
|
)
|
||||||
|
|
||||||
type mockCompleter struct {
|
// The phrasing itself is tested in internal/phraser. What is left here is the
|
||||||
|
// only thing the daemon adds: the stub floor, on the three ways a reply can
|
||||||
|
// fail to arrive.
|
||||||
|
type stubCompleter struct {
|
||||||
out string
|
out string
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m mockCompleter) Complete(_ context.Context, _ llm.Req) (string, error) { return m.out, m.err }
|
func (s stubCompleter) Complete(_ context.Context, _ llm.Req) (string, error) { return s.out, s.err }
|
||||||
|
|
||||||
func TestLLMReplierReturnsLLMReply(t *testing.T) {
|
func TestLLMReplierPassesTheModelReplyThrough(t *testing.T) {
|
||||||
r := newLLMReplier(mockCompleter{out: `{"response":"записала, кофе закончился","mood":"neutral"}`}, nil)
|
r := newLLMReplier(stubCompleter{out: `{"response":"записала, кофе закончился","mood":"neutral"}`}, nil)
|
||||||
got := r.Reply(router.Decision{Intent: router.IntentNote, Slots: router.Slots{Text: "кофе закончился"}})
|
|
||||||
if got != "записала, кофе закончился" {
|
|
||||||
t.Errorf("got %q, want %q", got, "записала, кофе закончился")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLLMReplierFallsBackToPlainText(t *testing.T) {
|
|
||||||
r := newLLMReplier(mockCompleter{out: "записала, кофе закончился"}, nil)
|
|
||||||
got := r.Reply(router.Decision{Intent: router.IntentNote, Slots: router.Slots{Text: "кофе закончился"}})
|
got := r.Reply(router.Decision{Intent: router.IntentNote, Slots: router.Slots{Text: "кофе закончился"}})
|
||||||
if got != "записала, кофе закончился" {
|
if got != "записала, кофе закончился" {
|
||||||
t.Errorf("got %q, want %q", got, "записала, кофе закончился")
|
t.Errorf("got %q, want %q", got, "записала, кофе закончился")
|
||||||
@@ -34,54 +28,30 @@ func TestLLMReplierFallsBackToPlainText(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestLLMReplierFallsBackToStubOnError(t *testing.T) {
|
func TestLLMReplierFallsBackToStubOnError(t *testing.T) {
|
||||||
r := newLLMReplier(mockCompleter{err: errTestLLMDown}, nil)
|
r := newLLMReplier(stubCompleter{err: errReplierTest}, nil)
|
||||||
noteDec := router.Decision{Intent: router.IntentNote}
|
assertStub(t, r, router.Decision{Intent: router.IntentNote}, "llm error")
|
||||||
got := r.Reply(noteDec)
|
|
||||||
want := voice.NewStubReplier().Reply(noteDec)
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("on llm error: got %q, want stub %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLLMReplierFallsBackToStubOnEmpty(t *testing.T) {
|
func TestLLMReplierFallsBackToStubOnEmpty(t *testing.T) {
|
||||||
r := newLLMReplier(mockCompleter{out: ""}, nil)
|
r := newLLMReplier(stubCompleter{out: ""}, nil)
|
||||||
noteDec := router.Decision{Intent: router.IntentNote}
|
assertStub(t, r, router.Decision{Intent: router.IntentNote}, "empty llm")
|
||||||
got := r.Reply(noteDec)
|
|
||||||
want := voice.NewStubReplier().Reply(noteDec)
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("on empty llm: got %q, want stub %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLLMReplierClarifyUsesStub(t *testing.T) {
|
func TestLLMReplierClarifyUsesStub(t *testing.T) {
|
||||||
r := newLLMReplier(mockCompleter{out: "я всё поняла"}, nil)
|
r := newLLMReplier(stubCompleter{out: "я всё поняла"}, nil)
|
||||||
clarifyDec := router.Decision{Clarify: true}
|
assertStub(t, r, router.Decision{Clarify: true}, "clarify")
|
||||||
got := r.Reply(clarifyDec)
|
}
|
||||||
want := voice.NewStubReplier().Reply(clarifyDec)
|
|
||||||
|
func assertStub(t *testing.T, r *llmReplier, d router.Decision, what string) {
|
||||||
|
t.Helper()
|
||||||
|
got, want := r.Reply(d), voice.NewStubReplier().Reply(d)
|
||||||
if got != want {
|
if got != want {
|
||||||
t.Errorf("on clarify: got %q, want stub %q", got, want)
|
t.Errorf("on %s: got %q, want stub %q", what, got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var errTestLLMDown = errTest("llm down")
|
var errReplierTest = errTest("llm down")
|
||||||
|
|
||||||
type errTest string
|
type errTest string
|
||||||
|
|
||||||
func (e errTest) Error() string { return string(e) }
|
func (e errTest) Error() string { return string(e) }
|
||||||
|
|
||||||
// grammarRecorder captures the request so the grammar can be asserted on.
|
|
||||||
type grammarRecorder struct{ req llm.Req }
|
|
||||||
|
|
||||||
func (g *grammarRecorder) Complete(_ context.Context, r llm.Req) (string, error) {
|
|
||||||
g.req = r
|
|
||||||
return `{"response":"записала","mood":"neutral"}`, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLLMReplierCarriesTheResponseGrammar(t *testing.T) {
|
|
||||||
rec := &grammarRecorder{}
|
|
||||||
r := newLLMReplier(rec, nil)
|
|
||||||
r.Reply(router.Decision{Intent: router.IntentNote, Slots: router.Slots{Text: "кофе закончился"}})
|
|
||||||
if rec.req.Grammar != phraser.ResponseGrammar {
|
|
||||||
t.Errorf("grammar = %q, want phraser.ResponseGrammar", rec.req.Grammar)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -54,7 +54,11 @@ func (h *reactiveHandler) phraseSource(ctx context.Context, name, utterance stri
|
|||||||
log.Printf("voice: %s: no world model, reading the source back instead", name)
|
log.Printf("voice: %s: no world model, reading the source back instead", name)
|
||||||
return ""
|
return ""
|
||||||
case err != nil:
|
case err != nil:
|
||||||
|
// The resident phraser answers this call with its fallback text and the
|
||||||
|
// error together. Drop the text: these callers hold the passage itself
|
||||||
|
// and read it back better than "вот что я нашла: <passage>" does.
|
||||||
log.Printf("voice: %s: phrase: %v", name, err)
|
log.Printf("voice: %s: phrase: %v", name, err)
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
return reply
|
return reply
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
# Recall topic veto, what it costs and what it buys, 2026-08-03
|
||||||
|
|
||||||
|
Vikunja #496. The task asked for a cross-language fix. Skip the topic veto in
|
||||||
|
`memory.RecallAllowed` when the question and the hit are in different scripts.
|
||||||
|
An English question would then stop losing a Russian note.
|
||||||
|
|
||||||
|
No such case exists. No fixture case puts the question and its wanted note in
|
||||||
|
different scripts. The case the task named is not one either.
|
||||||
|
|
||||||
|
en-hard-024
|
||||||
|
query "what fixed the screen problem"
|
||||||
|
note "the flicker went away once i swapped the display cable"
|
||||||
|
|
||||||
|
Both are English. It is a paraphrase failure, not a language failure. A script
|
||||||
|
test would not have changed a single case, and neither would a bilingual stem
|
||||||
|
map.
|
||||||
|
|
||||||
|
## What the veto is worth today
|
||||||
|
|
||||||
|
Measured with the real embedder, multilingual-e5-small int8, gate 0.55, margin
|
||||||
|
0.008. The first row is the veto as it ships. The second is `RecallAllowed`
|
||||||
|
forced to true.
|
||||||
|
|
||||||
|
| | cases passing | answered | false recall | silenced by gate |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| veto on | 22/32 | 17/27 | 0/5 | 2 |
|
||||||
|
| veto off | 22/32 | 18/27 | 1/5 | 1 |
|
||||||
|
|
||||||
|
The pass count does not move. The veto trades one true recall for one false one.
|
||||||
|
It costs `en-hard-024` and it buys `ru-silent-029`:
|
||||||
|
|
||||||
|
ru-silent-029
|
||||||
|
query "во сколько отходит поезд"
|
||||||
|
note "погулял вдоль реки" 0.835, margin 0.019
|
||||||
|
|
||||||
|
The second case counted as silenced by the gate is `ru-home-026` at margin
|
||||||
|
0.001, which the margin gate stops. The veto has nothing to do with it.
|
||||||
|
|
||||||
|
## Why no lexical rule separates the two
|
||||||
|
|
||||||
|
`en-hard-024` and `ru-silent-029` are in the same lexical class. Both questions
|
||||||
|
share zero content words with their hit, and neither carries a first-person
|
||||||
|
marker. The scores sit on top of each other, 0.826 against 0.835, and so do the
|
||||||
|
margins, 0.023 against 0.019. Only one thing separates them. A screen problem
|
||||||
|
and a swapped display cable are the same event. A train and a river walk are
|
||||||
|
not. The embedder scores that difference at nine thousandths.
|
||||||
|
|
||||||
|
So the signal is semantic and the gate is lexical. Any rule cheap enough to sit
|
||||||
|
in `RecallAllowed` and strong enough to recover `en-hard-024` also re-admits
|
||||||
|
`ru-silent-029`, which puts false recall back to 1/5.
|
||||||
|
|
||||||
|
One near-miss rule was tried on paper and rejected: let the veto pass when the
|
||||||
|
hit itself is first person. It works on these two, because the English note says
|
||||||
|
"i swapped" and the Russian note says only "погулял". It is backwards as a
|
||||||
|
principle. A first-person note is exactly the personal note the veto keeps away
|
||||||
|
from a world question. The rule would weaken the veto where it was designed to
|
||||||
|
bite. It survives here only because Russian drops the pronoun.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Accept the loss. `en-hard-024` stays silenced and false recall stays 0/5.
|
||||||
|
|
||||||
|
The way out is a reranker, not a longer word list. Recall@3 is 85.2% against
|
||||||
|
recall@1 at 70.4%, so the right note is usually in the returned set and ranked
|
||||||
|
wrong. That is where the remaining points are, and it is not this task.
|
||||||
@@ -60,6 +60,14 @@ var firstPerson = map[string]bool{
|
|||||||
// kill one false one. A question about his own life keeps the embedder alone
|
// kill one false one. A question about his own life keeps the embedder alone
|
||||||
// as its judge. A question about the world has to name something the memory
|
// as its judge. A question about the world has to name something the memory
|
||||||
// actually mentions.
|
// actually mentions.
|
||||||
|
//
|
||||||
|
// The veto's price was re-measured on 2026-08-03 (#496,
|
||||||
|
// docs/evals/2026-08-03-recall-topic-veto.md). It costs one true recall and
|
||||||
|
// buys one false one, and the fixture pass count is the same either way. The
|
||||||
|
// lost case is an English paraphrase, not the cross-language loss it was
|
||||||
|
// reported as, and the fixture has no cross-language case at all. Do not add a
|
||||||
|
// script test or a bilingual stem map for it — both are no-ops here. The
|
||||||
|
// separating signal is semantic and belongs in a reranker, not in this file.
|
||||||
func RecallAllowed(query, text string) bool {
|
func RecallAllowed(query, text string) bool {
|
||||||
if mentionsHim(query) {
|
if mentionsHim(query) {
|
||||||
return true
|
return true
|
||||||
|
|||||||
@@ -31,6 +31,22 @@ func TestRecallAllowed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The known cost of the veto and the thing that pays for it, both measured on
|
||||||
|
// the held-out fixture with the real embedder (#496,
|
||||||
|
// docs/evals/2026-08-03-recall-topic-veto.md). The two are one lexical class:
|
||||||
|
// zero shared content words, no first-person marker, scores 0.826 against 0.835
|
||||||
|
// and margins 0.023 against 0.019. Recovering the first re-admits the second,
|
||||||
|
// which puts false recall back to 1/5. Anyone loosening the veto has to move
|
||||||
|
// the first line without moving the second.
|
||||||
|
func TestRecallVetoTradeIsPinned(t *testing.T) {
|
||||||
|
if RecallAllowed("what fixed the screen problem", "the flicker went away once i swapped the display cable") {
|
||||||
|
t.Error("en-hard-024 is expected to stay vetoed — if this passes now, re-measure false recall before celebrating")
|
||||||
|
}
|
||||||
|
if RecallAllowed("во сколько отходит поезд", "погулял вдоль реки") {
|
||||||
|
t.Error("ru-silent-029 must stay vetoed — this is the false recall the veto exists to stop")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// A question made only of filler has no topic word to match on, and the score
|
// A question made only of filler has no topic word to match on, and the score
|
||||||
// gate is then the only judge it can have.
|
// gate is then the only judge it can have.
|
||||||
func TestRecallAllowedFallsBackWhenNothingToCompare(t *testing.T) {
|
func TestRecallAllowedFallsBackWhenNothingToCompare(t *testing.T) {
|
||||||
|
|||||||
@@ -26,20 +26,22 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/kami/maven/internal/dialogue"
|
"github.com/kami/maven/internal/dialogue"
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed talk_v1.json
|
//go:embed talk_v1.json
|
||||||
var talkFixtureJSON []byte
|
var talkFixtureJSON []byte
|
||||||
|
|
||||||
// The three phrasing paths under test. Values match the fixture's "path" field.
|
// The phrasing paths under test. Values match the fixture's "path" field.
|
||||||
const (
|
const (
|
||||||
PathChat = "chat" // PhraseChat
|
PathChat = "chat" // PhraseChat
|
||||||
PathQuery = "query" // PhraseQuery with notes
|
PathQuery = "query" // PhraseQuery with notes
|
||||||
PathKnowledge = "knowledge" // PhraseQuery with no notes
|
PathKnowledge = "knowledge" // PhraseQuery with no notes
|
||||||
|
PathReply = "reply" // PhraseReply, the reactive confirmation
|
||||||
)
|
)
|
||||||
|
|
||||||
// TalkPaths — report order.
|
// TalkPaths — report order.
|
||||||
var TalkPaths = []string{PathChat, PathQuery, PathKnowledge}
|
var TalkPaths = []string{PathChat, PathQuery, PathKnowledge, PathReply}
|
||||||
|
|
||||||
// TalkCheckNames — the checks that apply to a free-form reply, in report order.
|
// TalkCheckNames — the checks that apply to a free-form reply, in report order.
|
||||||
// Deliberately a subset of CheckNames: length, mood and "no questions" are nudge
|
// Deliberately a subset of CheckNames: length, mood and "no questions" are nudge
|
||||||
@@ -58,17 +60,30 @@ var TalkCheckNames = []string{
|
|||||||
// WantAny is the on-topic contract: at least one lowercased fragment must appear
|
// WantAny is the on-topic contract: at least one lowercased fragment must appear
|
||||||
// in the reply. Fragments are stems ("пароль" → "парол") so declension does not
|
// in the reply. Fragments are stems ("пароль" → "парол") so declension does not
|
||||||
// defeat them.
|
// defeat them.
|
||||||
|
//
|
||||||
|
// Intent, Key and Value carry the reply path's decision: that path is phrased
|
||||||
|
// from what the router already resolved, not from the raw utterance. Utterance
|
||||||
|
// stays filled anyway, because it is what a human reads in the report.
|
||||||
type TalkCase struct {
|
type TalkCase struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
Utterance string `json:"utterance"`
|
Utterance string `json:"utterance"`
|
||||||
History []string `json:"history,omitempty"`
|
History []string `json:"history,omitempty"`
|
||||||
Notes []string `json:"notes,omitempty"`
|
Notes []string `json:"notes,omitempty"`
|
||||||
|
Intent string `json:"intent,omitempty"`
|
||||||
|
Key string `json:"key,omitempty"`
|
||||||
|
Value string `json:"value,omitempty"`
|
||||||
WantAny []string `json:"want_any"`
|
WantAny []string `json:"want_any"`
|
||||||
Tags []string `json:"tags,omitempty"`
|
Tags []string `json:"tags,omitempty"`
|
||||||
Note string `json:"note,omitempty"`
|
Note string `json:"note,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TalkSchemaVersion — the version this loader understands. Separate from the
|
||||||
|
// nudge fixture's SchemaVersion: the two fixtures have different shapes and
|
||||||
|
// change on different days, and one shared constant would force a bump on the
|
||||||
|
// fixture that did not move.
|
||||||
|
const TalkSchemaVersion = 1
|
||||||
|
|
||||||
// TalkFixture — the versioned envelope, same gating as Fixture.
|
// TalkFixture — the versioned envelope, same gating as Fixture.
|
||||||
type TalkFixture struct {
|
type TalkFixture struct {
|
||||||
SchemaVersion int `json:"schema_version"`
|
SchemaVersion int `json:"schema_version"`
|
||||||
@@ -83,8 +98,8 @@ func LoadTalk() (TalkFixture, error) {
|
|||||||
if err := json.Unmarshal(talkFixtureJSON, &f); err != nil {
|
if err := json.Unmarshal(talkFixtureJSON, &f); err != nil {
|
||||||
return TalkFixture{}, fmt.Errorf("parse talk fixture: %w", err)
|
return TalkFixture{}, fmt.Errorf("parse talk fixture: %w", err)
|
||||||
}
|
}
|
||||||
if f.SchemaVersion != SchemaVersion {
|
if f.SchemaVersion != TalkSchemaVersion {
|
||||||
return TalkFixture{}, fmt.Errorf("talk fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion)
|
return TalkFixture{}, fmt.Errorf("talk fixture schema_version %d, want %d", f.SchemaVersion, TalkSchemaVersion)
|
||||||
}
|
}
|
||||||
if len(f.Cases) == 0 {
|
if len(f.Cases) == 0 {
|
||||||
return TalkFixture{}, fmt.Errorf("talk fixture has no cases")
|
return TalkFixture{}, fmt.Errorf("talk fixture has no cases")
|
||||||
@@ -92,13 +107,27 @@ func LoadTalk() (TalkFixture, error) {
|
|||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Talker — the two methods a conversational path must have to be scorable.
|
// Talker — the methods a conversational path must have to be scorable.
|
||||||
// *phraser.LLMPhraser satisfies it; same trick as Nudger.
|
// *phraser.LLMPhraser satisfies the first two; *phraser.Replier satisfies the
|
||||||
|
// third, so a run that scores all four paths passes a Pair.
|
||||||
type Talker interface {
|
type Talker interface {
|
||||||
PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error)
|
PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error)
|
||||||
PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error)
|
PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Confirmer — the reply path. *phraser.Replier satisfies it.
|
||||||
|
type Confirmer interface {
|
||||||
|
PhraseReply(ctx context.Context, d router.Decision) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pair joins the two objects the daemon wires separately — the phraser and the
|
||||||
|
// replier — so one ScoreTalk call covers every path Maven speaks through. A bare
|
||||||
|
// Talker still works; its reply cases score as errors, which is honest.
|
||||||
|
type Pair struct {
|
||||||
|
Talker
|
||||||
|
Confirmer
|
||||||
|
}
|
||||||
|
|
||||||
// TalkOutcome — one scored case.
|
// TalkOutcome — one scored case.
|
||||||
type TalkOutcome struct {
|
type TalkOutcome struct {
|
||||||
Case TalkCase
|
Case TalkCase
|
||||||
@@ -194,10 +223,30 @@ func (c TalkCase) run(ctx context.Context, t Talker) (string, error) {
|
|||||||
return t.PhraseQuery(ctx, c.Utterance, c.Notes)
|
return t.PhraseQuery(ctx, c.Utterance, c.Notes)
|
||||||
case PathKnowledge:
|
case PathKnowledge:
|
||||||
return t.PhraseQuery(ctx, c.Utterance, nil)
|
return t.PhraseQuery(ctx, c.Utterance, nil)
|
||||||
|
case PathReply:
|
||||||
|
conf, ok := t.(Confirmer)
|
||||||
|
if !ok {
|
||||||
|
return "", fmt.Errorf("target cannot phrase replies — pass a Pair")
|
||||||
|
}
|
||||||
|
return conf.PhraseReply(ctx, c.decision())
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("unknown path %q", c.Path)
|
return "", fmt.Errorf("unknown path %q", c.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// decision rebuilds what the router would have handed the replier. Text is the
|
||||||
|
// utterance for a note or a reminder, which is what the router puts there.
|
||||||
|
func (c TalkCase) decision() router.Decision {
|
||||||
|
return router.Decision{
|
||||||
|
Intent: router.Intent(c.Intent),
|
||||||
|
Slots: router.Slots{
|
||||||
|
Key: c.Key,
|
||||||
|
Value: c.Value,
|
||||||
|
Text: c.Utterance,
|
||||||
|
HasKey: c.Key != "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (c TalkCase) turns() []dialogue.Turn {
|
func (c TalkCase) turns() []dialogue.Turn {
|
||||||
turns := make([]dialogue.Turn, 0, len(c.History))
|
turns := make([]dialogue.Turn, 0, len(c.History))
|
||||||
for _, h := range c.History {
|
for _, h := range c.History {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/kami/maven/internal/llm"
|
"github.com/kami/maven/internal/llm"
|
||||||
"github.com/kami/maven/internal/persona"
|
"github.com/kami/maven/internal/persona"
|
||||||
"github.com/kami/maven/internal/phraser"
|
"github.com/kami/maven/internal/phraser"
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
)
|
)
|
||||||
|
|
||||||
// perPathMinimum — the resolution floor. A per-path score built on a handful of
|
// perPathMinimum — the resolution floor. A per-path score built on a handful of
|
||||||
@@ -36,6 +37,10 @@ func TestTalkFixture(t *testing.T) {
|
|||||||
|
|
||||||
switch c.Path {
|
switch c.Path {
|
||||||
case PathChat, PathQuery, PathKnowledge:
|
case PathChat, PathQuery, PathKnowledge:
|
||||||
|
case PathReply:
|
||||||
|
if c.Intent == "" {
|
||||||
|
t.Errorf("%s: reply case has no intent — the replier is phrased from the decision", c.ID)
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
t.Errorf("%s: unknown path %q", c.ID, c.Path)
|
t.Errorf("%s: unknown path %q", c.ID, c.Path)
|
||||||
}
|
}
|
||||||
@@ -69,10 +74,15 @@ type fakeTalker struct{ reply string }
|
|||||||
func (f fakeTalker) PhraseChat(context.Context, string, []dialogue.Turn) (string, error) {
|
func (f fakeTalker) PhraseChat(context.Context, string, []dialogue.Turn) (string, error) {
|
||||||
return f.reply, nil
|
return f.reply, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f fakeTalker) PhraseQuery(context.Context, string, []string) (string, error) {
|
func (f fakeTalker) PhraseQuery(context.Context, string, []string) (string, error) {
|
||||||
return f.reply, nil
|
return f.reply, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f fakeTalker) PhraseReply(context.Context, router.Decision) (string, error) {
|
||||||
|
return f.reply, nil
|
||||||
|
}
|
||||||
|
|
||||||
// TestScoreTalkCounts — a reply that fails on purpose must be counted on every
|
// TestScoreTalkCounts — a reply that fails on purpose must be counted on every
|
||||||
// path, so a real run cannot report a hidden zero.
|
// path, so a real run cannot report a hidden zero.
|
||||||
func TestScoreTalkCounts(t *testing.T) {
|
func TestScoreTalkCounts(t *testing.T) {
|
||||||
@@ -104,7 +114,7 @@ func TestScoreTalkCounts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLLMTalkBaseline — the resident model on the three conversational paths.
|
// TestLLMTalkBaseline — the resident model on all four phrasing paths.
|
||||||
// Opt-in exactly like TestLLMPhrasingBaseline: CI has no model and a run costs
|
// Opt-in exactly like TestLLMPhrasingBaseline: CI has no model and a run costs
|
||||||
// minutes on the CPU target.
|
// minutes on the CPU target.
|
||||||
//
|
//
|
||||||
@@ -132,32 +142,32 @@ func TestLLMTalkBaseline(t *testing.T) {
|
|||||||
p := phraser.NewLLMPhraserAt(base, cfg)
|
p := phraser.NewLLMPhraserAt(base, cfg)
|
||||||
defer p.Close()
|
defer p.Close()
|
||||||
|
|
||||||
// Unreachable server is fatal here, not a logged warning, and that differs
|
// The model id names the run in the report. Since Vikunja #397 every path
|
||||||
// from the nudge test on purpose. PhraseNudge returns its errors, so a dead
|
// returns its errors, so a server that dies mid-run shows up in the Errors
|
||||||
// server there shows up honestly in the Errors column. PhraseChat and
|
// column instead of scoring as bad phrasing — the before-and-after probe that
|
||||||
// PhraseQuery do NOT: they swallow every failure and return a canned string
|
// used to stand in for that is gone.
|
||||||
// ("поговорили.", "не знаю.", "вот что я нашла: …"). So on these three paths
|
|
||||||
// a dead server produces a full report with 0 errors and a terrible score —
|
|
||||||
// a number that looks like bad phrasing and is really no phrasing at all.
|
|
||||||
// Refusing to score without a confirmed model is the only guard available
|
|
||||||
// until the phraser reports its failures (Vikunja #397).
|
|
||||||
model, err := llm.ModelID(ctx, base)
|
model, err := llm.ModelID(ctx, base)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("no model at %s: %v — refusing to score, these paths hide their errors "+
|
t.Fatalf("no model at %s: %v", base, err)
|
||||||
"and would report a plausible-looking result off a dead server", base, err)
|
|
||||||
}
|
}
|
||||||
t.Logf("scoring model %s at %s", model, base)
|
t.Logf("scoring model %s at %s", model, base)
|
||||||
|
|
||||||
rep, err := ScoreTalk(ctx, "llm ("+model+", built-in persona)", p, f)
|
// The reply path is a separate object in the daemon too: the phraser owns its
|
||||||
|
// own llama-server, the replier is handed an llm.Client. Pair scores both.
|
||||||
|
block := func() string { return persona.Facts{}.Block(time.Now()) }
|
||||||
|
target := Pair{Talker: p, Confirmer: phraser.NewReplier(llm.New(base, cfg.Timeout), block)}
|
||||||
|
|
||||||
|
rep, err := ScoreTalk(ctx, "llm ("+model+", built-in persona)", target, f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ScoreTalk: %v", err)
|
t.Fatalf("ScoreTalk: %v", err)
|
||||||
}
|
}
|
||||||
t.Log("\n" + rep.String() + "\nreplies:\n" + rep.Replies() + "\nfailures:\n" + rep.Failures())
|
t.Log("\n" + rep.String() + "\nreplies:\n" + rep.Replies() + "\nfailures:\n" + rep.Failures())
|
||||||
|
|
||||||
// And again afterwards: the run takes minutes, and a server that died or got
|
// A run where nothing was phrased is not a low score, it is no measurement.
|
||||||
// OOM-killed halfway through would leave the first cases scored and the rest
|
if rep.Errors == rep.Total {
|
||||||
// silently canned. Checking only at the start would not catch that.
|
t.Fatalf("every case errored — nothing was measured, the score above is not a phrasing result")
|
||||||
if _, err := llm.ModelID(ctx, base); err != nil {
|
}
|
||||||
t.Fatalf("model at %s went away during the run: %v — the score above is not trustworthy", base, err)
|
if rep.Errors > 0 {
|
||||||
|
t.Logf("%d/%d cases errored — those are model failures, not phrasing failures", rep.Errors, rep.Total)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -222,6 +222,90 @@
|
|||||||
"utterance": "почему гром слышно позже молнии?",
|
"utterance": "почему гром слышно позже молнии?",
|
||||||
"want_any": ["звук", "све", "быстр", "гром", "молни"],
|
"want_any": ["звук", "све", "быстр", "гром", "молни"],
|
||||||
"tags": ["general"]
|
"tags": ["general"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reply-fact-coffee",
|
||||||
|
"path": "reply",
|
||||||
|
"intent": "fact",
|
||||||
|
"key": "кофе",
|
||||||
|
"value": "закончился",
|
||||||
|
"utterance": "кофе закончился",
|
||||||
|
"want_any": ["коф"],
|
||||||
|
"tags": ["fact"],
|
||||||
|
"note": "The plainest confirmation there is, and the sentence he hears most often."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reply-fact-weight",
|
||||||
|
"path": "reply",
|
||||||
|
"intent": "fact",
|
||||||
|
"key": "вес",
|
||||||
|
"value": "82",
|
||||||
|
"utterance": "мой вес 82",
|
||||||
|
"want_any": ["вес", "82"],
|
||||||
|
"tags": ["fact", "number"],
|
||||||
|
"note": "A number must survive into the confirmation; a paraphrase that drops it is useless."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reply-fact-pill",
|
||||||
|
"path": "reply",
|
||||||
|
"intent": "fact",
|
||||||
|
"key": "таблетки",
|
||||||
|
"value": "выпил",
|
||||||
|
"utterance": "таблетки выпил",
|
||||||
|
"want_any": ["таблетк"],
|
||||||
|
"tags": ["fact", "feminine"],
|
||||||
|
"note": "He says 'выпил', masculine and about himself. She must not copy the form onto herself."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reply-note-router",
|
||||||
|
"path": "reply",
|
||||||
|
"intent": "note",
|
||||||
|
"utterance": "роутер перезагружается сам по ночам",
|
||||||
|
"want_any": ["роутер"],
|
||||||
|
"tags": ["note"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reply-note-long",
|
||||||
|
"path": "reply",
|
||||||
|
"intent": "note",
|
||||||
|
"utterance": "если диск снова отвалится, посмотреть кабель, а не контроллер, в прошлый раз был кабель",
|
||||||
|
"want_any": ["диск", "кабел"],
|
||||||
|
"tags": ["note", "length"],
|
||||||
|
"note": "A long note baits a long confirmation. One sentence is the contract."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reply-reminder-evening",
|
||||||
|
"path": "reply",
|
||||||
|
"intent": "reminder",
|
||||||
|
"utterance": "напомни вечером полить цветы",
|
||||||
|
"want_any": ["цвет", "полит", "вечер"],
|
||||||
|
"tags": ["reminder"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reply-reminder-tomorrow",
|
||||||
|
"path": "reply",
|
||||||
|
"intent": "reminder",
|
||||||
|
"utterance": "напомни завтра позвонить в поликлинику",
|
||||||
|
"want_any": ["поликлиник", "позвон", "звон"],
|
||||||
|
"tags": ["reminder"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reply-formality-bait",
|
||||||
|
"path": "reply",
|
||||||
|
"intent": "note",
|
||||||
|
"utterance": "запишите пожалуйста что счётчики я сдал",
|
||||||
|
"want_any": ["счётчик", "счетчик"],
|
||||||
|
"tags": ["note", "persona-bait", "address"],
|
||||||
|
"note": "Polite plural in the input. The confirmation must still be на ты."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reply-question-bait",
|
||||||
|
"path": "reply",
|
||||||
|
"intent": "note",
|
||||||
|
"utterance": "надо купить фильтр для воды, не помню какой",
|
||||||
|
"want_any": ["фильтр"],
|
||||||
|
"tags": ["note", "no-question"],
|
||||||
|
"note": "An unresolved note invites her to ask which filter. A confirmation does not ask."
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package phraser
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A dead server must be distinguishable from bad phrasing. Both PhraseChat and
|
||||||
|
// PhraseQuery keep the turn alive with canned text — ChatFallback, "не знаю.",
|
||||||
|
// "вот что я нашла: …" — and every one of those is also a legitimate reply, so
|
||||||
|
// the text alone cannot say which happened. The error is the only signal, and
|
||||||
|
// before Vikunja #397 it was dropped: the talk scorer reported a full run with
|
||||||
|
// zero errors off a server that answered nothing.
|
||||||
|
func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Error(w, "model not loaded", http.StatusServiceUnavailable)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
p := NewLLMPhraserAt(srv.URL, Config{})
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
call func() (string, error)
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"chat", func() (string, error) {
|
||||||
|
return p.PhraseChat(context.Background(), "как дела", nil)
|
||||||
|
}, ChatFallback},
|
||||||
|
{"knowledge", func() (string, error) {
|
||||||
|
return p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
|
||||||
|
}, "не знаю."},
|
||||||
|
{"evidence", func() (string, error) {
|
||||||
|
return p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"два литра"})
|
||||||
|
}, "вот что я нашла: два литра"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
got, err := c.call()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("no error from a dead server; the scorer would count this as bad phrasing")
|
||||||
|
}
|
||||||
|
if got != c.want {
|
||||||
|
t.Errorf("fallback text = %q, want %q — the daemon still has to say something", got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An empty answer is a failure too: the server is up and produced no tokens,
|
||||||
|
// which is not an answer and must not score as one.
|
||||||
|
func TestEmptyKnowledgeAnswerIsAnError(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.Write([]byte(`{"choices":[{"message":{"content":""}}]}`))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
p := NewLLMPhraserAt(srv.URL, Config{})
|
||||||
|
|
||||||
|
got, err := p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("an empty response scored as an answer")
|
||||||
|
}
|
||||||
|
if got != "не знаю." {
|
||||||
|
t.Errorf("fallback text = %q, want \"не знаю.\"", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "empty") {
|
||||||
|
t.Errorf("error = %v; want it to name the empty response", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
@@ -26,6 +27,11 @@ import (
|
|||||||
|
|
||||||
var listenRE = regexp.MustCompile(`listening on (https?://\S+)`)
|
var listenRE = regexp.MustCompile(`listening on (https?://\S+)`)
|
||||||
|
|
||||||
|
// errEmptyResponse — the server answered and said nothing. Separate from a
|
||||||
|
// transport failure: the model is up and produced no tokens, which is still not
|
||||||
|
// an answer and must not score as one.
|
||||||
|
var errEmptyResponse = errors.New("phraser: empty response from the model")
|
||||||
|
|
||||||
type LLMPhraser struct {
|
type LLMPhraser struct {
|
||||||
cfg Config
|
cfg Config
|
||||||
client *http.Client
|
client *http.Client
|
||||||
@@ -428,8 +434,11 @@ func (p *LLMPhraser) PhraseNudge(ctx context.Context, c loop.Candidate) (deliver
|
|||||||
}
|
}
|
||||||
|
|
||||||
// PhraseQuery prompts the LLM with the user's utterance and matching notes to
|
// PhraseQuery prompts the LLM with the user's utterance and matching notes to
|
||||||
// compose a natural answer. Falls back to "вот что я нашла: <notes>" on any
|
// compose a natural answer. On any LLM error it returns the fallback text —
|
||||||
// LLM error — better to give the raw data than silence.
|
// "вот что я нашла: <notes>", or "не знаю." with no notes — and the error
|
||||||
|
// together. The daemon uses the text and keeps the turn alive; a caller that is
|
||||||
|
// measuring counts the failure. Until Vikunja #397 the error was dropped, so a
|
||||||
|
// dead server scored as bad phrasing.
|
||||||
func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) {
|
func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) {
|
||||||
// Blank sources are no sources. A caller that hands over one empty string —
|
// Blank sources are no sources. A caller that hands over one empty string —
|
||||||
// a page that fetched to nothing, a snippet trimmed away — used to take the
|
// a page that fetched to nothing, a snippet trimmed away — used to take the
|
||||||
@@ -439,13 +448,15 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
|
|||||||
if len(notes) == 0 {
|
if len(notes) == 0 {
|
||||||
sys, prompt := p.knowledgePrompt(utterance)
|
sys, prompt := p.knowledgePrompt(utterance)
|
||||||
resp, err := p.chatWithSystem(ctx, sys, prompt, 768)
|
resp, err := p.chatWithSystem(ctx, sys, prompt, 768)
|
||||||
if err != nil || resp == "" {
|
if err != nil {
|
||||||
return "не знаю.", nil
|
return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", err)
|
||||||
|
}
|
||||||
|
if resp == "" {
|
||||||
|
return "не знаю.", errEmptyResponse
|
||||||
}
|
}
|
||||||
text, _, perr := parseResponseMood(resp)
|
text, _, perr := parseResponseMood(resp)
|
||||||
if perr != nil {
|
if perr != nil {
|
||||||
log.Printf("phraser: PhraseQuery: %v", perr)
|
return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", perr)
|
||||||
return "не знаю.", nil
|
|
||||||
}
|
}
|
||||||
if text != "" {
|
if text != "" {
|
||||||
return text, nil
|
return text, nil
|
||||||
@@ -457,13 +468,12 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
|
|||||||
text, _, perr := parseResponseMood(resp)
|
text, _, perr := parseResponseMood(resp)
|
||||||
if err != nil || perr != nil {
|
if err != nil || perr != nil {
|
||||||
// Read the notes out rather than ship a broken fragment.
|
// Read the notes out rather than ship a broken fragment.
|
||||||
if perr != nil {
|
cause := err
|
||||||
log.Printf("phraser: PhraseQuery: %v", perr)
|
if cause == nil {
|
||||||
|
cause = perr
|
||||||
}
|
}
|
||||||
if len(notes) == 1 {
|
return "вот что я нашла: " + strings.Join(notes, "; "),
|
||||||
return "вот что я нашла: " + notes[0], nil
|
fmt.Errorf("phrase query (evidence): %w", cause)
|
||||||
}
|
|
||||||
return "вот что я нашла: " + strings.Join(notes, "; "), nil
|
|
||||||
}
|
}
|
||||||
if text != "" {
|
if text != "" {
|
||||||
return text, nil
|
return text, nil
|
||||||
@@ -472,8 +482,9 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
|
|||||||
}
|
}
|
||||||
|
|
||||||
// PhraseChat uses the LLM to respond conversationally, building a multi-turn
|
// PhraseChat uses the LLM to respond conversationally, building a multi-turn
|
||||||
// message array from dialogue history + the current user utterance. Falls back
|
// message array from dialogue history + the current user utterance. On any LLM
|
||||||
// to a simple greeting on any LLM error — better to say something than nothing.
|
// error it returns both ChatFallback and the error, on the same rule as
|
||||||
|
// PhraseQuery: the fallback keeps the turn alive, the error stays visible.
|
||||||
func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error) {
|
func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error) {
|
||||||
sys := chatSystemPrompt(p.cfg.ContextBlock)
|
sys := chatSystemPrompt(p.cfg.ContextBlock)
|
||||||
msgs := []chatMsg{
|
msgs := []chatMsg{
|
||||||
@@ -490,13 +501,11 @@ func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history [
|
|||||||
|
|
||||||
resp, err := p.chatWithMessages(ctx, msgs, 768)
|
resp, err := p.chatWithMessages(ctx, msgs, 768)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("phraser: PhraseChat: %v", err)
|
return ChatFallback, fmt.Errorf("phrase chat: %w", err)
|
||||||
return "поговорили.", nil
|
|
||||||
}
|
}
|
||||||
text, _, perr := parseResponseMood(resp)
|
text, _, perr := parseResponseMood(resp)
|
||||||
if perr != nil {
|
if perr != nil {
|
||||||
log.Printf("phraser: PhraseChat: %v", perr)
|
return ChatFallback, fmt.Errorf("phrase chat: %w", perr)
|
||||||
return "поговорили.", nil
|
|
||||||
}
|
}
|
||||||
if text != "" {
|
if text != "" {
|
||||||
return text, nil
|
return text, nil
|
||||||
|
|||||||
@@ -66,11 +66,17 @@ type Stub struct{}
|
|||||||
// NewStub builds the floor phraser. no config — the Stub is stateless.
|
// NewStub builds the floor phraser. no config — the Stub is stateless.
|
||||||
func NewStub() *Stub { return &Stub{} }
|
func NewStub() *Stub { return &Stub{} }
|
||||||
|
|
||||||
|
// ChatFallback — what she says on the chat path when the model gave her
|
||||||
|
// nothing to say. It replaced "поговорили.", which reads as a summary of a
|
||||||
|
// conversation that did not happen. Said out loud this one is an admission,
|
||||||
|
// which is what it is.
|
||||||
|
const ChatFallback = "даже не знаю, что сказать."
|
||||||
|
|
||||||
// PhraseChat returns a stub reply — the LLMPhraser replaces this with a
|
// PhraseChat returns a stub reply — the LLMPhraser replaces this with a
|
||||||
// prompted response from the model. The history parameter is accepted but
|
// prompted response from the model. The history parameter is accepted but
|
||||||
// ignored at the stub level (the production impl uses it for multi-turn).
|
// ignored at the stub level (the production impl uses it for multi-turn).
|
||||||
func (s *Stub) PhraseChat(_ context.Context, _ string, _ []dialogue.Turn) (string, error) {
|
func (s *Stub) PhraseChat(_ context.Context, _ string, _ []dialogue.Turn) (string, error) {
|
||||||
return "поговорили.", nil
|
return ChatFallback, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PhraseQuery returns a deterministic summary of the best matching notes.
|
// PhraseQuery returns a deterministic summary of the best matching notes.
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
// phraser/replier.go — reactive reply phrasing, the confirmation he hears
|
||||||
|
// after every fact, note and reminder.
|
||||||
|
//
|
||||||
|
// It lived in cmd/mavend as package main until Vikunja #396, which meant the
|
||||||
|
// most frequently heard sentence Maven says was the one path the phrasing eval
|
||||||
|
// could not import, let alone score. Nothing here talks to the daemon: the
|
||||||
|
// caller supplies the completer and the context block, and cmd/mavend keeps the
|
||||||
|
// stub fallback so a model error still answers.
|
||||||
|
package phraser
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/kami/maven/internal/llm"
|
||||||
|
"github.com/kami/maven/internal/persona"
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Completer is the model seam for the replier, a subset of router.Completer.
|
||||||
|
// *llm.Client satisfies it.
|
||||||
|
type Completer interface {
|
||||||
|
Complete(ctx context.Context, r llm.Req) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// replyTimeout bounds one reply. Generous because the resident model on the CPU
|
||||||
|
// floor is slow and the caller has a deterministic fallback anyway.
|
||||||
|
const replyTimeout = 60 * time.Second
|
||||||
|
|
||||||
|
// ReplySystemPrompt — the reactive confirmation contract: one short Russian
|
||||||
|
// sentence, feminine self-reference, informal address, no question.
|
||||||
|
const ReplySystemPrompt = `Ты — Maven, домашняя ассистентка (о себе — в женском роде). Владелец — мужчина, говоришь с ним на "ты", в единственном числе; никогда не "вы"/"ваш" и не "он"/"его". Подтверди действие РОВНО ОДНИМ коротким предложением (≤120 символов), по-русски, спокойно и без официальных формулировок. Не задавай вопросов, не повторяй слова, не добавляй ничего после точки. Отвечай ТОЛЬКО одним объектом JSON с полями "response" (текст) и "mood" (ровно одно из: neutral, happy, thinking, tired, confused).
|
||||||
|
Пример: {"response": "Записала, что ты выпил стакан воды.", "mood": "neutral"}
|
||||||
|
Никогда не пиши "..." в поле response.`
|
||||||
|
|
||||||
|
// Replier phrases reactive confirmations with the resident model. It has no
|
||||||
|
// fallback of its own: an error is returned, and the daemon answers from the
|
||||||
|
// deterministic stub. That is also what makes it scorable — a dead server shows
|
||||||
|
// up as an error rather than as bad phrasing.
|
||||||
|
type Replier struct {
|
||||||
|
c Completer
|
||||||
|
|
||||||
|
// block renders the shared context block per turn (who he is, the time).
|
||||||
|
// nil ⇒ the prompt stands alone.
|
||||||
|
block func() string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewReplier builds a replier over c. block may be nil.
|
||||||
|
func NewReplier(c Completer, block func() string) *Replier {
|
||||||
|
return &Replier{c: c, block: block}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PhraseReply returns the confirmation for one decision. An empty string with a
|
||||||
|
// nil error means the model produced nothing usable, which the caller must
|
||||||
|
// treat exactly like an error.
|
||||||
|
func (r *Replier) PhraseReply(ctx context.Context, d router.Decision) (string, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, replyTimeout)
|
||||||
|
defer cancel()
|
||||||
|
out, err := r.c.Complete(ctx, llm.Req{
|
||||||
|
System: persona.Prepend(r.block, ReplySystemPrompt),
|
||||||
|
User: replyContext(d),
|
||||||
|
Grammar: ResponseGrammar,
|
||||||
|
MaxTokens: 512,
|
||||||
|
RepeatPenalty: 1.3,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
out = stripThink(out)
|
||||||
|
if response, _, perr := parseResponseMood(out); perr != nil {
|
||||||
|
return "", perr
|
||||||
|
} else if response != "" {
|
||||||
|
return response, nil
|
||||||
|
}
|
||||||
|
// fallback: the model answered in bare prose, which is fine here.
|
||||||
|
return firstSentence(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// firstSentence trims the model's output to a single clean confirmation: first
|
||||||
|
// line, first sentence, whitespace-normalized — the last-line defense against a
|
||||||
|
// small model that rambles past the first period despite the prompt + stop.
|
||||||
|
func firstSentence(s string) string {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||||
|
s = s[:i]
|
||||||
|
}
|
||||||
|
// keep up to and including the first sentence-ending punctuation.
|
||||||
|
if i := strings.IndexAny(s, ".!?"); i >= 0 {
|
||||||
|
s = s[:i+1]
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// replyContext renders the decision into a compact RU description for the model.
|
||||||
|
func replyContext(d router.Decision) string {
|
||||||
|
switch d.Intent {
|
||||||
|
case router.IntentFact:
|
||||||
|
return "записала факт: " + d.Slots.Key + " " + d.Slots.Value
|
||||||
|
case router.IntentNote:
|
||||||
|
return "сохранила заметку: " + d.Slots.Text
|
||||||
|
case router.IntentReminder:
|
||||||
|
return "поставила напоминание: " + d.Slots.Text
|
||||||
|
default:
|
||||||
|
return string(d.Intent) + ": " + d.Slots.Text
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StripThink removes the <think> block a Thinking-variant model emits before its
|
||||||
|
// answer. Exported for the daemon's own model callers, which parse output that
|
||||||
|
// never passes through a phraser method.
|
||||||
|
func StripThink(s string) string { return stripThink(s) }
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package phraser
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/kami/maven/internal/llm"
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
|
)
|
||||||
|
|
||||||
|
type mockCompleter struct {
|
||||||
|
out string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m mockCompleter) Complete(_ context.Context, _ llm.Req) (string, error) { return m.out, m.err }
|
||||||
|
|
||||||
|
func TestReplierReturnsLLMReply(t *testing.T) {
|
||||||
|
r := NewReplier(mockCompleter{out: `{"response":"записала, кофе закончился","mood":"neutral"}`}, nil)
|
||||||
|
got, err := r.PhraseReply(context.Background(), noteDecision())
|
||||||
|
if err != nil || got != "записала, кофе закончился" {
|
||||||
|
t.Errorf("got %q, %v, want %q, nil", got, err, "записала, кофе закончился")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplierFallsBackToPlainText(t *testing.T) {
|
||||||
|
r := NewReplier(mockCompleter{out: "записала, кофе закончился"}, nil)
|
||||||
|
got, err := r.PhraseReply(context.Background(), noteDecision())
|
||||||
|
if err != nil || got != "записала, кофе закончился" {
|
||||||
|
t.Errorf("got %q, %v, want %q, nil", got, err, "записала, кофе закончился")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplierReportsTheModelError(t *testing.T) {
|
||||||
|
r := NewReplier(mockCompleter{err: errTestLLMDown}, nil)
|
||||||
|
got, err := r.PhraseReply(context.Background(), noteDecision())
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("got %q, nil error — a dead model must be reported, not phrased around", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A fragment the grammar left half-open is a failed generation. It must come
|
||||||
|
// back as an error so the daemon reaches its stub, not as a reply.
|
||||||
|
func TestReplierRejectsBrokenJSON(t *testing.T) {
|
||||||
|
r := NewReplier(mockCompleter{out: `{"response":"запис`}, nil)
|
||||||
|
got, err := r.PhraseReply(context.Background(), noteDecision())
|
||||||
|
if err == nil || got != "" {
|
||||||
|
t.Errorf("got %q, %v, want empty and an error", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplierEmptyOutputIsEmpty(t *testing.T) {
|
||||||
|
r := NewReplier(mockCompleter{out: ""}, nil)
|
||||||
|
got, err := r.PhraseReply(context.Background(), noteDecision())
|
||||||
|
if err != nil || got != "" {
|
||||||
|
t.Errorf("got %q, %v, want empty and no error", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// grammarRecorder captures the request so the grammar can be asserted on.
|
||||||
|
type grammarRecorder struct{ req llm.Req }
|
||||||
|
|
||||||
|
func (g *grammarRecorder) Complete(_ context.Context, r llm.Req) (string, error) {
|
||||||
|
g.req = r
|
||||||
|
return `{"response":"записала","mood":"neutral"}`, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReplierCarriesTheResponseGrammar(t *testing.T) {
|
||||||
|
rec := &grammarRecorder{}
|
||||||
|
r := NewReplier(rec, nil)
|
||||||
|
if _, err := r.PhraseReply(context.Background(), noteDecision()); err != nil {
|
||||||
|
t.Fatalf("PhraseReply: %v", err)
|
||||||
|
}
|
||||||
|
if rec.req.Grammar != ResponseGrammar {
|
||||||
|
t.Errorf("grammar = %q, want ResponseGrammar", rec.req.Grammar)
|
||||||
|
}
|
||||||
|
if rec.req.System != ReplySystemPrompt {
|
||||||
|
t.Errorf("system prompt = %q, want ReplySystemPrompt", rec.req.System)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func noteDecision() router.Decision {
|
||||||
|
return router.Decision{Intent: router.IntentNote, Slots: router.Slots{Text: "кофе закончился"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
var errTestLLMDown = errTest("llm down")
|
||||||
|
|
||||||
|
type errTest string
|
||||||
|
|
||||||
|
func (e errTest) Error() string { return string(e) }
|
||||||
@@ -215,10 +215,12 @@ func TestSwap_RollbackFailureLeavesNoBackendAndDegrades(t *testing.T) {
|
|||||||
if _, _, aerr := p.acquire(); !errors.Is(aerr, ErrNoBackend) {
|
if _, _, aerr := p.acquire(); !errors.Is(aerr, ErrNoBackend) {
|
||||||
t.Errorf("acquire error = %v; want ErrNoBackend", aerr)
|
t.Errorf("acquire error = %v; want ErrNoBackend", aerr)
|
||||||
}
|
}
|
||||||
// Phrasing degrades to its fallback instead of failing the turn.
|
// Phrasing degrades to its fallback instead of failing the turn, and since
|
||||||
|
// Vikunja #397 it reports the error next to that fallback so a measuring
|
||||||
|
// caller can tell "no model" from "bad phrasing".
|
||||||
got, err := p.PhraseChat(context.Background(), "привет", nil)
|
got, err := p.PhraseChat(context.Background(), "привет", nil)
|
||||||
if err != nil {
|
if !errors.Is(err, ErrNoBackend) {
|
||||||
t.Fatalf("PhraseChat after a total failure returned an error: %v", err)
|
t.Errorf("PhraseChat error = %v; want ErrNoBackend alongside the fallback", err)
|
||||||
}
|
}
|
||||||
if got == "" {
|
if got == "" {
|
||||||
t.Error("PhraseChat returned empty; the fallback must still say something")
|
t.Error("PhraseChat returned empty; the fallback must still say something")
|
||||||
|
|||||||
Reference in New Issue
Block a user