mavend, eval: use the phrasing errors the phraser now returns (V-397)

Call sites take the fallback text and log the error instead of treating a
canned string as success. phraseSource drops the text entirely — its callers
hold the passage and read it back better than "вот что я нашла: <passage>".

The talk scorer's before-and-after model probe (the #395 workaround) goes;
the run now fails only when every case errored, which is the honest
"nothing was measured" condition. TalkFixture gets its own schema version so
the two fixtures can be versioned apart.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

cmd/mavend/actions.go

@@ -58,9 +58,13 @@ func (h *reactiveHandler) actionChat(ctx context.Context, dec router.Decision) s
 	// Conversational: build history from dialogue session (prior user turns)
 	// and let the LLM respond from general knowledge + context.
 	history := h.chatHistory()
+	// The phraser hands back its own fallback text alongside the error, so the
+	// turn survives a dead server and the failure still reaches the log.
 	reply, err := h.phraser.PhraseChat(ctx, dec.Utterance, history)
 	if err != nil {
 		log.Printf("voice: chat: %v", err)
+	}
+	if reply == "" {
 		return "поговорили."
 	}
 	return reply

cmd/mavend/actions_query.go

@@ -445,7 +445,13 @@ func (h *reactiveHandler) queryMemory(ctx context.Context, t *queryTurn) (string
 	// A note is phrased in Maven's voice; a fact is read back as it was
 	// stored.
 	if hit.Meta["type"] == "note" {
-		if reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{text}); perr == nil && reply != "" {
+		reply, perr := h.phraser.PhraseQuery(ctx, t.dec.Utterance, []string{text})
+		switch {
+		case perr != nil:
+			// Reading the note back verbatim beats the phraser's own fallback,
+			// which only wraps the same text in "вот что я нашла:".
+			log.Printf("voice: recall phrase: %v", perr)
+		case reply != "":
 			return reply, true
 		}
 	}

cmd/mavend/worldmodel.go

@@ -54,7 +54,11 @@ func (h *reactiveHandler) phraseSource(ctx context.Context, name, utterance stri
 		log.Printf("voice: %s: no world model, reading the source back instead", name)
 		return ""
 	case err != nil:
+		// The resident phraser answers this call with its fallback text and the
+		// error together. Drop the text: these callers hold the passage itself
+		// and read it back better than "вот что я нашла: <passage>" does.
 		log.Printf("voice: %s: phrase: %v", name, err)
+		return ""
 	}
 	return reply
 }

internal/phraser/eval/talk.go

@@ -78,6 +78,12 @@ type TalkCase struct {
 	Note      string   `json:"note,omitempty"`
 }
 
+// TalkSchemaVersion — the version this loader understands. Separate from the
+// nudge fixture's SchemaVersion: the two fixtures have different shapes and
+// change on different days, and one shared constant would force a bump on the
+// fixture that did not move.
+const TalkSchemaVersion = 1
+
 // TalkFixture — the versioned envelope, same gating as Fixture.
 type TalkFixture struct {
 	SchemaVersion int        `json:"schema_version"`
@@ -92,8 +98,8 @@ func LoadTalk() (TalkFixture, error) {
 	if err := json.Unmarshal(talkFixtureJSON, &f); err != nil {
 		return TalkFixture{}, fmt.Errorf("parse talk fixture: %w", err)
 	}
-	if f.SchemaVersion != SchemaVersion {
-		return TalkFixture{}, fmt.Errorf("talk fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion)
+	if f.SchemaVersion != TalkSchemaVersion {
+		return TalkFixture{}, fmt.Errorf("talk fixture schema_version %d, want %d", f.SchemaVersion, TalkSchemaVersion)
 	}
 	if len(f.Cases) == 0 {
 		return TalkFixture{}, fmt.Errorf("talk fixture has no cases")

internal/phraser/eval/talk_test.go

@@ -142,19 +142,13 @@ func TestLLMTalkBaseline(t *testing.T) {
 	p := phraser.NewLLMPhraserAt(base, cfg)
 	defer p.Close()
 
-	// Unreachable server is fatal here, not a logged warning, and that differs
-	// from the nudge test on purpose. PhraseNudge returns its errors, so a dead
-	// server there shows up honestly in the Errors column. PhraseChat and
-	// PhraseQuery do NOT: they swallow every failure and return a canned string
-	// ("поговорили.", "не знаю.", "вот что я нашла: …"). So on these three paths
-	// a dead server produces a full report with 0 errors and a terrible score —
-	// a number that looks like bad phrasing and is really no phrasing at all.
-	// Refusing to score without a confirmed model is the only guard available
-	// until the phraser reports its failures (Vikunja #397).
+	// The model id names the run in the report. Since Vikunja #397 every path
+	// returns its errors, so a server that dies mid-run shows up in the Errors
+	// column instead of scoring as bad phrasing — the before-and-after probe that
+	// used to stand in for that is gone.
 	model, err := llm.ModelID(ctx, base)
 	if err != nil {
-		t.Fatalf("no model at %s: %v — refusing to score, these paths hide their errors "+
-			"and would report a plausible-looking result off a dead server", base, err)
+		t.Fatalf("no model at %s: %v", base, err)
 	}
 	t.Logf("scoring model %s at %s", model, base)
 
@@ -169,10 +163,11 @@ func TestLLMTalkBaseline(t *testing.T) {
 	}
 	t.Log("\n" + rep.String() + "\nreplies:\n" + rep.Replies() + "\nfailures:\n" + rep.Failures())
 
-	// And again afterwards: the run takes minutes, and a server that died or got
-	// OOM-killed halfway through would leave the first cases scored and the rest
-	// silently canned. Checking only at the start would not catch that.
-	if _, err := llm.ModelID(ctx, base); err != nil {
-		t.Fatalf("model at %s went away during the run: %v — the score above is not trustworthy", base, err)
+	// A run where nothing was phrased is not a low score, it is no measurement.
+	if rep.Errors == rep.Total {
+		t.Fatalf("every case errored — nothing was measured, the score above is not a phrasing result")
+	}
+	if rep.Errors > 0 {
+		t.Logf("%d/%d cases errored — those are model failures, not phrasing failures", rep.Errors, rep.Total)
 	}
 }

internal/phraser/failure_test.go

@@ -0,0 +1,72 @@
+package phraser
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// A dead server must be distinguishable from bad phrasing. Both PhraseChat and
+// PhraseQuery keep the turn alive with canned text — "поговорили.", "не знаю.",
+// "вот что я нашла: …" — and every one of those is also a legitimate reply, so
+// the text alone cannot say which happened. The error is the only signal, and
+// before Vikunja #397 it was dropped: the talk scorer reported a full run with
+// zero errors off a server that answered nothing.
+func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "model not loaded", http.StatusServiceUnavailable)
+	}))
+	t.Cleanup(srv.Close)
+	p := NewLLMPhraserAt(srv.URL, Config{})
+
+	cases := []struct {
+		name string
+		call func() (string, error)
+		want string
+	}{
+		{"chat", func() (string, error) {
+			return p.PhraseChat(context.Background(), "как дела", nil)
+		}, "поговорили."},
+		{"knowledge", func() (string, error) {
+			return p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
+		}, "не знаю."},
+		{"evidence", func() (string, error) {
+			return p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"два литра"})
+		}, "вот что я нашла: два литра"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got, err := c.call()
+			if err == nil {
+				t.Fatalf("no error from a dead server; the scorer would count this as bad phrasing")
+			}
+			if got != c.want {
+				t.Errorf("fallback text = %q, want %q — the daemon still has to say something", got, c.want)
+			}
+		})
+	}
+}
+
+// An empty answer is a failure too: the server is up and produced no tokens,
+// which is not an answer and must not score as one.
+func TestEmptyKnowledgeAnswerIsAnError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"choices":[{"message":{"content":""}}]}`))
+	}))
+	t.Cleanup(srv.Close)
+	p := NewLLMPhraserAt(srv.URL, Config{})
+
+	got, err := p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
+	if err == nil {
+		t.Fatal("an empty response scored as an answer")
+	}
+	if got != "не знаю." {
+		t.Errorf("fallback text = %q, want \"не знаю.\"", got)
+	}
+	if !strings.Contains(err.Error(), "empty") {
+		t.Errorf("error = %v; want it to name the empty response", err)
+	}
+}

internal/phraser/llmphraser.go

@@ -428,8 +428,11 @@ func (p *LLMPhraser) PhraseNudge(ctx context.Context, c loop.Candidate) (deliver
 }
 
 // PhraseQuery prompts the LLM with the user's utterance and matching notes to
-// compose a natural answer. Falls back to "вот что я нашла: <notes>" on any
-// LLM error — better to give the raw data than silence.
+// compose a natural answer. On any LLM error it returns BOTH the fallback text
+// ("вот что я нашла: <notes>", or "не знаю." with no notes) AND the error, so a
+// caller that wants to keep the turn alive uses the text and a caller that is
+// measuring counts the failure. Until Vikunja #397 the error was swallowed and a
+// dead server scored as bad phrasing.
 func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) {
 	// Blank sources are no sources. A caller that hands over one empty string —
 	// a page that fetched to nothing, a snippet trimmed away — used to take the
@@ -439,13 +442,15 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
 	if len(notes) == 0 {
 		sys, prompt := p.knowledgePrompt(utterance)
 		resp, err := p.chatWithSystem(ctx, sys, prompt, 768)
-		if err != nil || resp == "" {
-			return "не знаю.", nil
+		if err != nil {
+			return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", err)
+		}
+		if resp == "" {
+			return "не знаю.", errEmptyResponse
 		}
 		text, _, perr := parseResponseMood(resp)
 		if perr != nil {
-			log.Printf("phraser: PhraseQuery: %v", perr)
-			return "не знаю.", nil
+			return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", perr)
 		}
 		if text != "" {
 			return text, nil
@@ -457,13 +462,12 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
 	text, _, perr := parseResponseMood(resp)
 	if err != nil || perr != nil {
 		// Read the notes out rather than ship a broken fragment.
-		if perr != nil {
-			log.Printf("phraser: PhraseQuery: %v", perr)
+		cause := err
+		if cause == nil {
+			cause = perr
 		}
-		if len(notes) == 1 {
-			return "вот что я нашла: " + notes[0], nil
-		}
-		return "вот что я нашла: " + strings.Join(notes, "; "), nil
+		return "вот что я нашла: " + strings.Join(notes, "; "),
+			fmt.Errorf("phrase query (evidence): %w", cause)
 	}
 	if text != "" {
 		return text, nil
@@ -472,8 +476,9 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
 }
 
 // PhraseChat uses the LLM to respond conversationally, building a multi-turn
-// message array from dialogue history + the current user utterance. Falls back
-// to a simple greeting on any LLM error — better to say something than nothing.
+// message array from dialogue history + the current user utterance. On any LLM
+// error it returns both "поговорили." and the error, on the same rule as
+// PhraseQuery: the fallback keeps the turn alive, the error stays visible.
 func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error) {
 	sys := chatSystemPrompt(p.cfg.ContextBlock)
 	msgs := []chatMsg{
@@ -490,13 +495,11 @@ func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history [
 
 	resp, err := p.chatWithMessages(ctx, msgs, 768)
 	if err != nil {
-		log.Printf("phraser: PhraseChat: %v", err)
-		return "поговорили.", nil
+		return "поговорили.", fmt.Errorf("phrase chat: %w", err)
 	}
 	text, _, perr := parseResponseMood(resp)
 	if perr != nil {
-		log.Printf("phraser: PhraseChat: %v", perr)
-		return "поговорили.", nil
+		return "поговорили.", fmt.Errorf("phrase chat: %w", perr)
 	}
 	if text != "" {
 		return text, nil

internal/phraser/swap_test.go

@@ -215,10 +215,12 @@ func TestSwap_RollbackFailureLeavesNoBackendAndDegrades(t *testing.T) {
 	if _, _, aerr := p.acquire(); !errors.Is(aerr, ErrNoBackend) {
 		t.Errorf("acquire error = %v; want ErrNoBackend", aerr)
 	}
-	// Phrasing degrades to its fallback instead of failing the turn.
+	// Phrasing degrades to its fallback instead of failing the turn, and since
+	// Vikunja #397 it reports the error next to that fallback so a measuring
+	// caller can tell "no model" from "bad phrasing".
 	got, err := p.PhraseChat(context.Background(), "привет", nil)
-	if err != nil {
-		t.Fatalf("PhraseChat after a total failure returned an error: %v", err)
+	if !errors.Is(err, ErrNoBackend) {
+		t.Errorf("PhraseChat error = %v; want ErrNoBackend alongside the fallback", err)
 	}
 	if got == "" {
 		t.Error("PhraseChat returned empty; the fallback must still say something")

internal/phraser/world.go

@@ -37,6 +37,11 @@ type Remote interface {
 // (docs/evals/2026-08-02-workstation-gemma4-12b.md).
 var ErrNoWorldModel = errors.New("phraser: no world model available")
 
+// errEmptyResponse — the server answered and said nothing. A separate error from
+// a transport failure because it means the model is up and produced no tokens,
+// which is still not an answer and must not score as one.
+var errEmptyResponse = errors.New("phraser: empty response from the model")
+
 // chatTemperature — what the phraser's own transport has always sampled at.
 // Named so the remote path cannot drift from it silently. Whether 0.7 is right
 // at all is Vikunja #402, and answering that here would hide a phrasing change