Compare commits

...

4 Commits

Author SHA1 Message Date
claude c47881106e phraser: say "даже не знаю, что сказать" when there is nothing to say (V-397)
Review of #108: "поговорили." reads as a summary of a conversation that did
not happen. One exported constant now, so the Stub, the LLMPhraser fallback
and the daemon all say the same thing.

internal/voice/replier.go keeps its own copy — that is the separate replier
seam, not this one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-04 00:50:46 +04:00
claude 9a70f7378b phraser: move errEmptyResponse next to its only caller (V-397)
It sat in world.go, which is about the workstation model; it is a phrasing
error and belongs in llmphraser.go. Also trims the PhraseQuery doc.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-04 00:47:31 +04:00
claude b18f608594 mavend, eval: use the phrasing errors the phraser now returns (V-397)
Call sites take the fallback text and log the error instead of treating a
canned string as success. phraseSource drops the text entirely — its callers
hold the passage and read it back better than "вот что я нашла: <passage>".

The talk scorer's before-and-after model probe (the #395 workaround) goes;
the run now fails only when every case errored, which is the honest
"nothing was measured" condition. TalkFixture gets its own schema version so
the two fixtures can be versioned apart.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-04 00:41:16 +04:00
claude d1f8a734c5 phraser: report the failure next to the fallback (V-397)
PhraseChat and PhraseQuery returned canned text with a nil error, so a dead
or OOM-killed server was indistinguishable from bad phrasing — "не знаю." is
also a legitimate answer.

Both now return the fallback text AND the error. The daemon keeps using the
text, so the turn still survives; a measuring caller counts a real failure.
An empty response is its own error: the model is up and said nothing.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-04 00:41:16 +04:00
9 changed files with 147 additions and 42 deletions
+6 -1
View File
@@ -40,6 +40,7 @@ import (
"context" "context"
"log" "log"
"github.com/kami/maven/internal/phraser"
"github.com/kami/maven/internal/router" "github.com/kami/maven/internal/router"
) )
@@ -58,10 +59,14 @@ func (h *reactiveHandler) actionChat(ctx context.Context, dec router.Decision) s
// Conversational: build history from dialogue session (prior user turns) // Conversational: build history from dialogue session (prior user turns)
// and let the LLM respond from general knowledge + context. // and let the LLM respond from general knowledge + context.
history := h.chatHistory() history := h.chatHistory()
// The phraser hands back its own fallback text alongside the error, so the
// turn survives a dead server and the failure still reaches the log.
reply, err := h.phraser.PhraseChat(ctx, dec.Utterance, history) reply, err := h.phraser.PhraseChat(ctx, dec.Utterance, history)
if err != nil { if err != nil {
log.Printf("voice: chat: %v", err) log.Printf("voice: chat: %v", err)
return "поговорили." }
if reply == "" {
return phraser.ChatFallback
} }
return reply return reply
} }
+7 -1
View File
@@ -445,7 +445,13 @@ func (h *reactiveHandler) queryMemory(ctx context.Context, t *queryTurn) (string
// A note is phrased in Maven's voice; a fact is read back as it was // A note is phrased in Maven's voice; a fact is read back as it was
// stored. // stored.
if hit.Meta["type"] == "note" { if hit.Meta["type"] == "note" {
if reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{text}); perr == nil && reply != "" { reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{text})
switch {
case perr != nil:
// Reading the note back verbatim beats the phraser's own fallback,
// which only wraps the same text in "вот что я нашла:".
log.Printf("voice: recall phrase: %v", perr)
case reply != "":
return reply, true return reply, true
} }
} }
+4
View File
@@ -54,7 +54,11 @@ func (h *reactiveHandler) phraseSource(ctx context.Context, name, utterance stri
log.Printf("voice: %s: no world model, reading the source back instead", name) log.Printf("voice: %s: no world model, reading the source back instead", name)
return "" return ""
case err != nil: case err != nil:
// The resident phraser answers this call with its fallback text and the
// error together. Drop the text: these callers hold the passage itself
// and read it back better than "вот что я нашла: <passage>" does.
log.Printf("voice: %s: phrase: %v", name, err) log.Printf("voice: %s: phrase: %v", name, err)
return ""
} }
return reply return reply
} }
+8 -2
View File
@@ -78,6 +78,12 @@ type TalkCase struct {
Note string `json:"note,omitempty"` Note string `json:"note,omitempty"`
} }
// TalkSchemaVersion — the version this loader understands. Separate from the
// nudge fixture's SchemaVersion: the two fixtures have different shapes and
// change on different days, and one shared constant would force a bump on the
// fixture that did not move.
const TalkSchemaVersion = 1
// TalkFixture — the versioned envelope, same gating as Fixture. // TalkFixture — the versioned envelope, same gating as Fixture.
type TalkFixture struct { type TalkFixture struct {
SchemaVersion int `json:"schema_version"` SchemaVersion int `json:"schema_version"`
@@ -92,8 +98,8 @@ func LoadTalk() (TalkFixture, error) {
if err := json.Unmarshal(talkFixtureJSON, &f); err != nil { if err := json.Unmarshal(talkFixtureJSON, &f); err != nil {
return TalkFixture{}, fmt.Errorf("parse talk fixture: %w", err) return TalkFixture{}, fmt.Errorf("parse talk fixture: %w", err)
} }
if f.SchemaVersion != SchemaVersion { if f.SchemaVersion != TalkSchemaVersion {
return TalkFixture{}, fmt.Errorf("talk fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion) return TalkFixture{}, fmt.Errorf("talk fixture schema_version %d, want %d", f.SchemaVersion, TalkSchemaVersion)
} }
if len(f.Cases) == 0 { if len(f.Cases) == 0 {
return TalkFixture{}, fmt.Errorf("talk fixture has no cases") return TalkFixture{}, fmt.Errorf("talk fixture has no cases")
+11 -16
View File
@@ -142,19 +142,13 @@ func TestLLMTalkBaseline(t *testing.T) {
p := phraser.NewLLMPhraserAt(base, cfg) p := phraser.NewLLMPhraserAt(base, cfg)
defer p.Close() defer p.Close()
// Unreachable server is fatal here, not a logged warning, and that differs // The model id names the run in the report. Since Vikunja #397 every path
// from the nudge test on purpose. PhraseNudge returns its errors, so a dead // returns its errors, so a server that dies mid-run shows up in the Errors
// server there shows up honestly in the Errors column. PhraseChat and // column instead of scoring as bad phrasing — the before-and-after probe that
// PhraseQuery do NOT: they swallow every failure and return a canned string // used to stand in for that is gone.
// ("поговорили.", "не знаю.", "вот что я нашла: …"). So on these three paths
// a dead server produces a full report with 0 errors and a terrible score —
// a number that looks like bad phrasing and is really no phrasing at all.
// Refusing to score without a confirmed model is the only guard available
// until the phraser reports its failures (Vikunja #397).
model, err := llm.ModelID(ctx, base) model, err := llm.ModelID(ctx, base)
if err != nil { if err != nil {
t.Fatalf("no model at %s: %v — refusing to score, these paths hide their errors "+ t.Fatalf("no model at %s: %v", base, err)
"and would report a plausible-looking result off a dead server", base, err)
} }
t.Logf("scoring model %s at %s", model, base) t.Logf("scoring model %s at %s", model, base)
@@ -169,10 +163,11 @@ func TestLLMTalkBaseline(t *testing.T) {
} }
t.Log("\n" + rep.String() + "\nreplies:\n" + rep.Replies() + "\nfailures:\n" + rep.Failures()) t.Log("\n" + rep.String() + "\nreplies:\n" + rep.Replies() + "\nfailures:\n" + rep.Failures())
// And again afterwards: the run takes minutes, and a server that died or got // A run where nothing was phrased is not a low score, it is no measurement.
// OOM-killed halfway through would leave the first cases scored and the rest if rep.Errors == rep.Total {
// silently canned. Checking only at the start would not catch that. t.Fatalf("every case errored — nothing was measured, the score above is not a phrasing result")
if _, err := llm.ModelID(ctx, base); err != nil { }
t.Fatalf("model at %s went away during the run: %v — the score above is not trustworthy", base, err) if rep.Errors > 0 {
t.Logf("%d/%d cases errored — those are model failures, not phrasing failures", rep.Errors, rep.Total)
} }
} }
+72
View File
@@ -0,0 +1,72 @@
package phraser
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// A dead server must be distinguishable from bad phrasing. Both PhraseChat and
// PhraseQuery keep the turn alive with canned text — ChatFallback, "не знаю.",
// "вот что я нашла: …" — and every one of those is also a legitimate reply, so
// the text alone cannot say which happened. The error is the only signal, and
// before Vikunja #397 it was dropped: the talk scorer reported a full run with
// zero errors off a server that answered nothing.
func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "model not loaded", http.StatusServiceUnavailable)
}))
t.Cleanup(srv.Close)
p := NewLLMPhraserAt(srv.URL, Config{})
cases := []struct {
name string
call func() (string, error)
want string
}{
{"chat", func() (string, error) {
return p.PhraseChat(context.Background(), "как дела", nil)
}, ChatFallback},
{"knowledge", func() (string, error) {
return p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
}, "не знаю."},
{"evidence", func() (string, error) {
return p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"два литра"})
}, "вот что я нашла: два литра"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := c.call()
if err == nil {
t.Fatalf("no error from a dead server; the scorer would count this as bad phrasing")
}
if got != c.want {
t.Errorf("fallback text = %q, want %q — the daemon still has to say something", got, c.want)
}
})
}
}
// An empty answer is a failure too: the server is up and produced no tokens,
// which is not an answer and must not score as one.
func TestEmptyKnowledgeAnswerIsAnError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"choices":[{"message":{"content":""}}]}`))
}))
t.Cleanup(srv.Close)
p := NewLLMPhraserAt(srv.URL, Config{})
got, err := p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
if err == nil {
t.Fatal("an empty response scored as an answer")
}
if got != "не знаю." {
t.Errorf("fallback text = %q, want \"не знаю.\"", got)
}
if !strings.Contains(err.Error(), "empty") {
t.Errorf("error = %v; want it to name the empty response", err)
}
}
+27 -18
View File
@@ -5,6 +5,7 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"log" "log"
@@ -26,6 +27,11 @@ import (
var listenRE = regexp.MustCompile(`listening on (https?://\S+)`) var listenRE = regexp.MustCompile(`listening on (https?://\S+)`)
// errEmptyResponse — the server answered and said nothing. Separate from a
// transport failure: the model is up and produced no tokens, which is still not
// an answer and must not score as one.
var errEmptyResponse = errors.New("phraser: empty response from the model")
type LLMPhraser struct { type LLMPhraser struct {
cfg Config cfg Config
client *http.Client client *http.Client
@@ -428,8 +434,11 @@ func (p *LLMPhraser) PhraseNudge(ctx context.Context, c loop.Candidate) (deliver
} }
// PhraseQuery prompts the LLM with the user's utterance and matching notes to // PhraseQuery prompts the LLM with the user's utterance and matching notes to
// compose a natural answer. Falls back to "вот что я нашла: <notes>" on any // compose a natural answer. On any LLM error it returns the fallback text —
// LLM error — better to give the raw data than silence. // "вот что я нашла: <notes>", or "не знаю." with no notes — and the error
// together. The daemon uses the text and keeps the turn alive; a caller that is
// measuring counts the failure. Until Vikunja #397 the error was dropped, so a
// dead server scored as bad phrasing.
func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) { func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) {
// Blank sources are no sources. A caller that hands over one empty string — // Blank sources are no sources. A caller that hands over one empty string —
// a page that fetched to nothing, a snippet trimmed away — used to take the // a page that fetched to nothing, a snippet trimmed away — used to take the
@@ -439,13 +448,15 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
if len(notes) == 0 { if len(notes) == 0 {
sys, prompt := p.knowledgePrompt(utterance) sys, prompt := p.knowledgePrompt(utterance)
resp, err := p.chatWithSystem(ctx, sys, prompt, 768) resp, err := p.chatWithSystem(ctx, sys, prompt, 768)
if err != nil || resp == "" { if err != nil {
return "не знаю.", nil return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", err)
}
if resp == "" {
return "не знаю.", errEmptyResponse
} }
text, _, perr := parseResponseMood(resp) text, _, perr := parseResponseMood(resp)
if perr != nil { if perr != nil {
log.Printf("phraser: PhraseQuery: %v", perr) return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", perr)
return "не знаю.", nil
} }
if text != "" { if text != "" {
return text, nil return text, nil
@@ -457,13 +468,12 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
text, _, perr := parseResponseMood(resp) text, _, perr := parseResponseMood(resp)
if err != nil || perr != nil { if err != nil || perr != nil {
// Read the notes out rather than ship a broken fragment. // Read the notes out rather than ship a broken fragment.
if perr != nil { cause := err
log.Printf("phraser: PhraseQuery: %v", perr) if cause == nil {
cause = perr
} }
if len(notes) == 1 { return "вот что я нашла: " + strings.Join(notes, "; "),
return "вот что я нашла: " + notes[0], nil fmt.Errorf("phrase query (evidence): %w", cause)
}
return "вот что я нашла: " + strings.Join(notes, "; "), nil
} }
if text != "" { if text != "" {
return text, nil return text, nil
@@ -472,8 +482,9 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
} }
// PhraseChat uses the LLM to respond conversationally, building a multi-turn // PhraseChat uses the LLM to respond conversationally, building a multi-turn
// message array from dialogue history + the current user utterance. Falls back // message array from dialogue history + the current user utterance. On any LLM
// to a simple greeting on any LLM error — better to say something than nothing. // error it returns both ChatFallback and the error, on the same rule as
// PhraseQuery: the fallback keeps the turn alive, the error stays visible.
func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error) { func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error) {
sys := chatSystemPrompt(p.cfg.ContextBlock) sys := chatSystemPrompt(p.cfg.ContextBlock)
msgs := []chatMsg{ msgs := []chatMsg{
@@ -490,13 +501,11 @@ func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history [
resp, err := p.chatWithMessages(ctx, msgs, 768) resp, err := p.chatWithMessages(ctx, msgs, 768)
if err != nil { if err != nil {
log.Printf("phraser: PhraseChat: %v", err) return ChatFallback, fmt.Errorf("phrase chat: %w", err)
return "поговорили.", nil
} }
text, _, perr := parseResponseMood(resp) text, _, perr := parseResponseMood(resp)
if perr != nil { if perr != nil {
log.Printf("phraser: PhraseChat: %v", perr) return ChatFallback, fmt.Errorf("phrase chat: %w", perr)
return "поговорили.", nil
} }
if text != "" { if text != "" {
return text, nil return text, nil
+7 -1
View File
@@ -66,11 +66,17 @@ type Stub struct{}
// NewStub builds the floor phraser. no config — the Stub is stateless. // NewStub builds the floor phraser. no config — the Stub is stateless.
func NewStub() *Stub { return &Stub{} } func NewStub() *Stub { return &Stub{} }
// ChatFallback — what she says on the chat path when the model gave her
// nothing to say. It replaced "поговорили.", which reads as a summary of a
// conversation that did not happen. Said out loud this one is an admission,
// which is what it is.
const ChatFallback = "даже не знаю, что сказать."
// PhraseChat returns a stub reply — the LLMPhraser replaces this with a // PhraseChat returns a stub reply — the LLMPhraser replaces this with a
// prompted response from the model. The history parameter is accepted but // prompted response from the model. The history parameter is accepted but
// ignored at the stub level (the production impl uses it for multi-turn). // ignored at the stub level (the production impl uses it for multi-turn).
func (s *Stub) PhraseChat(_ context.Context, _ string, _ []dialogue.Turn) (string, error) { func (s *Stub) PhraseChat(_ context.Context, _ string, _ []dialogue.Turn) (string, error) {
return "поговорили.", nil return ChatFallback, nil
} }
// PhraseQuery returns a deterministic summary of the best matching notes. // PhraseQuery returns a deterministic summary of the best matching notes.
+5 -3
View File
@@ -215,10 +215,12 @@ func TestSwap_RollbackFailureLeavesNoBackendAndDegrades(t *testing.T) {
if _, _, aerr := p.acquire(); !errors.Is(aerr, ErrNoBackend) { if _, _, aerr := p.acquire(); !errors.Is(aerr, ErrNoBackend) {
t.Errorf("acquire error = %v; want ErrNoBackend", aerr) t.Errorf("acquire error = %v; want ErrNoBackend", aerr)
} }
// Phrasing degrades to its fallback instead of failing the turn. // Phrasing degrades to its fallback instead of failing the turn, and since
// Vikunja #397 it reports the error next to that fallback so a measuring
// caller can tell "no model" from "bad phrasing".
got, err := p.PhraseChat(context.Background(), "привет", nil) got, err := p.PhraseChat(context.Background(), "привет", nil)
if err != nil { if !errors.Is(err, ErrNoBackend) {
t.Fatalf("PhraseChat after a total failure returned an error: %v", err) t.Errorf("PhraseChat error = %v; want ErrNoBackend alongside the fallback", err)
} }
if got == "" { if got == "" {
t.Error("PhraseChat returned empty; the fallback must still say something") t.Error("PhraseChat returned empty; the fallback must still say something")