From 865623ef3e93d449697a68962abea62c169835ac Mon Sep 17 00:00:00 2001 From: claude Date: Tue, 4 Aug 2026 01:19:41 +0400 Subject: [PATCH] phraser, mavend: read the fallbacks from the file (V-501) The accessors are functions now, so the call sites that compared against one literal compare against the entry instead: IsUnknownFallback and IsSourcesFallback in the daemon tests, the entry key in the phraser tests. A reworded variant no longer breaks a Go test. The eval scores every variant on the persona checks the nudges already pass. --- cmd/mavend/actions.go | 2 +- cmd/mavend/actions_query.go | 2 +- cmd/mavend/query_recall_test.go | 6 +-- cmd/mavend/worldmodel.go | 6 ++- cmd/mavend/worldmodel_test.go | 12 +++--- internal/phraser/eval/fallbacks_test.go | 40 +++++++++++++++++ internal/phraser/failure_test.go | 47 +++++++++++++------- internal/phraser/fallbacks_test.go | 57 +++++++++++++++++++++++++ internal/phraser/llmphraser.go | 12 +++--- internal/phraser/phraser.go | 15 ++----- 10 files changed, 154 insertions(+), 45 deletions(-) create mode 100644 internal/phraser/eval/fallbacks_test.go create mode 100644 internal/phraser/fallbacks_test.go diff --git a/cmd/mavend/actions.go b/cmd/mavend/actions.go index e576e20..4145676 100644 --- a/cmd/mavend/actions.go +++ b/cmd/mavend/actions.go @@ -66,7 +66,7 @@ func (h *reactiveHandler) actionChat(ctx context.Context, dec router.Decision) s log.Printf("voice: chat: %v", err) } if reply == "" { - return phraser.ChatFallback + return phraser.ChatFallback() } return reply } diff --git a/cmd/mavend/actions_query.go b/cmd/mavend/actions_query.go index f2b65d0..42cb70f 100644 --- a/cmd/mavend/actions_query.go +++ b/cmd/mavend/actions_query.go @@ -798,7 +798,7 @@ func (h *reactiveHandler) queryGeneral(ctx context.Context, t *queryTurn) (strin reply, err := h.phraseWorld(ctx, t.dec.Utterance, nil) if errors.Is(err, phraser.ErrNoWorldModel) { log.Printf("voice: %q needs the world model and it is not available", t.dec.Utterance) - return worldGap, true + return worldGap(), true } if err != nil || reply == "" { return "не знаю.", true diff --git a/cmd/mavend/query_recall_test.go b/cmd/mavend/query_recall_test.go index b97c436..431f333 100644 --- a/cmd/mavend/query_recall_test.go +++ b/cmd/mavend/query_recall_test.go @@ -121,8 +121,8 @@ func TestQueryRecallNoteCanWin(t *testing.T) { {text: "выучил пару аккордов", score: 0.50, kind: "note"}, }) reply := askQuery(t, h, q) - if want := "вот что я нашла: молоко стоит в холодильнике"; reply != want { - t.Errorf("reply %q, want %q", reply, want) + if !phraser.IsSourcesFallback(reply, "молоко стоит в холодильнике") { + t.Errorf("reply %q, want the note read back", reply) } // One text, the winning memory's — the answer came from the memory // pass, not from handing the phraser every note in the table. @@ -151,7 +151,7 @@ func TestQueryRecallNoteCanWin(t *testing.T) { {text: "молоко стоит в холодильнике", score: 0.860, kind: "note"}, {text: "молоко закончилось", score: 0.858, kind: "note"}, }) - if reply := askQuery(t, h, q); reply != "не знаю." { + if reply := askQuery(t, h, q); !phraser.IsUnknownFallback(reply) { t.Errorf("reply %q, want silence", reply) } }) diff --git a/cmd/mavend/worldmodel.go b/cmd/mavend/worldmodel.go index b76c63d..bf7072c 100644 --- a/cmd/mavend/worldmodel.go +++ b/cmd/mavend/worldmodel.go @@ -23,7 +23,11 @@ type worldPhraser interface { // question about his meeting came back as a swimming competition in Nottingham. // Naming the gap is the rule CLAUDE.md already applies to a sibling service // being down. -const worldGap = "сейчас не могу ответить — большая модель недоступна, а придумывать не хочу." +// +// The wording lives in fallbacks_ru_v1.json and is fixed there, not picked from +// variants: this sentence names one specific gap and must not drift into a +// general "I don't know". +func worldGap() string { return phraser.WorldGap() } // phraseWorld asks the world model, or reports the gap. // diff --git a/cmd/mavend/worldmodel_test.go b/cmd/mavend/worldmodel_test.go index f8d0ad1..2d7b9d7 100644 --- a/cmd/mavend/worldmodel_test.go +++ b/cmd/mavend/worldmodel_test.go @@ -35,7 +35,7 @@ func TestQueryGeneralNamesTheGap(t *testing.T) { if !ok { t.Fatal("queryGeneral passed on the last source in the chain") } - if reply != worldGap { + if reply != worldGap() { t.Fatalf("reply = %q, want the named gap", reply) } if g.worldCalls != 1 { @@ -51,7 +51,7 @@ func TestQueryGeneralWithoutAWorldModelIsUnchanged(t *testing.T) { if !ok { t.Fatal("queryGeneral passed on the last source in the chain") } - if reply != "не знаю." { + if !phraser.IsUnknownFallback(reply) { t.Fatalf("reply = %q, want the Stub's answer", reply) } } @@ -61,12 +61,12 @@ func TestQueryGeneralWithoutAWorldModelIsUnchanged(t *testing.T) { // English in it. func TestWorldGapIsInPersona(t *testing.T) { for _, bad := range []string{"вы", "ваш", "рад ", "дорогой", "милый"} { - if strings.Contains(worldGap, bad) { - t.Errorf("the gap phrase contains %q: %s", bad, worldGap) + if strings.Contains(worldGap(), bad) { + t.Errorf("the gap phrase contains %q: %s", bad, worldGap()) } } - if strings.ContainsAny(worldGap, "abcdefghijklmnopqrstuvwxyz") { - t.Errorf("the gap phrase has Latin letters in it: %s", worldGap) + if strings.ContainsAny(worldGap(), "abcdefghijklmnopqrstuvwxyz") { + t.Errorf("the gap phrase has Latin letters in it: %s", worldGap()) } } diff --git a/internal/phraser/eval/fallbacks_test.go b/internal/phraser/eval/fallbacks_test.go new file mode 100644 index 0000000..3794a54 --- /dev/null +++ b/internal/phraser/eval/fallbacks_test.go @@ -0,0 +1,40 @@ +package eval + +import ( + "math/rand" + "strings" + "testing" + + "github.com/kami/maven/internal/phraser" +) + +// TestFallbackPersona scores every line in fallbacks_ru_v1.json on the persona +// checks the nudges are already held to. These lines are heard out loud, and +// they live in a JSON file now, so a reworded variant that says "рад" or "вы" +// would otherwise reach him with nothing between it and the speaker. +// +// Only the persona checks run. Mood and topic belong to a nudge, and these are +// not nudges: they are what she says when there is no answer. +func TestFallbackPersona(t *testing.T) { + fb, err := phraser.LoadFallbacks(rand.NewSource(20260804)) + if err != nil { + t.Fatalf("LoadFallbacks: %v", err) + } + persona := map[string]bool{ + CheckLang: true, CheckFeminine: true, CheckHisGender: true, + CheckAddress: true, CheckCringe: true, CheckLength: true, + } + variants := fb.Variants() + if len(variants) == 0 { + t.Fatal("no variants — the file loaded empty") + } + for _, v := range variants { + // {sources} stands for his own notes and never carries persona of its own. + body := strings.ReplaceAll(v, "{sources}", "два литра") + for _, r := range RunChecks(Case{}, body, "neutral") { + if persona[r.Name] && !r.Pass { + t.Errorf("%q fails %s: %s", v, r.Name, r.Detail) + } + } + } +} diff --git a/internal/phraser/failure_test.go b/internal/phraser/failure_test.go index 687bab8..5afbd5c 100644 --- a/internal/phraser/failure_test.go +++ b/internal/phraser/failure_test.go @@ -8,12 +8,28 @@ import ( "testing" ) +// isFallback — the text she says is picked from that entry's variants, so a test +// pins the entry rather than the wording. Pinning one line would make editing +// fallbacks_ru_v1.json break Go tests, which is the coupling this file removed. +func isFallback(t *testing.T, key, sources, got string) bool { + t.Helper() + e, ok := DefaultFallbacks().file.Entries[key] + if !ok { + t.Fatalf("no fallback entry %q", key) + } + for _, v := range e.Variants { + if strings.ReplaceAll(v, "{sources}", sources) == got { + return true + } + } + return false +} + // A dead server must be distinguishable from bad phrasing. Both PhraseChat and -// PhraseQuery keep the turn alive with canned text — ChatFallback, "не знаю.", -// "вот что я нашла: …" — and every one of those is also a legitimate reply, so -// the text alone cannot say which happened. The error is the only signal, and -// before Vikunja #397 it was dropped: the talk scorer reported a full run with -// zero errors off a server that answered nothing. +// PhraseQuery keep the turn alive with canned text — and every one of those +// lines is also a legitimate reply, so the text alone cannot say which happened. +// The error is the only signal, and before Vikunja #397 it was dropped: the talk +// scorer reported a full run with zero errors off a server that answered nothing. func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "model not loaded", http.StatusServiceUnavailable) @@ -22,19 +38,20 @@ func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) { p := NewLLMPhraserAt(srv.URL, Config{}) cases := []struct { - name string - call func() (string, error) - want string + name string + call func() (string, error) + key string + sources string }{ {"chat", func() (string, error) { return p.PhraseChat(context.Background(), "как дела", nil) - }, ChatFallback}, + }, fbChat, ""}, {"knowledge", func() (string, error) { return p.PhraseQuery(context.Background(), "кто написал войну и мир", nil) - }, "не знаю."}, + }, fbQueryUnknown, ""}, {"evidence", func() (string, error) { return p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"два литра"}) - }, "вот что я нашла: два литра"}, + }, fbQuerySources, "два литра"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -42,8 +59,8 @@ func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) { if err == nil { t.Fatalf("no error from a dead server; the scorer would count this as bad phrasing") } - if got != c.want { - t.Errorf("fallback text = %q, want %q — the daemon still has to say something", got, c.want) + if !isFallback(t, c.key, c.sources, got) { + t.Errorf("fallback text = %q, want a %q variant — the daemon still has to say something", got, c.key) } }) } @@ -63,8 +80,8 @@ func TestEmptyKnowledgeAnswerIsAnError(t *testing.T) { if err == nil { t.Fatal("an empty response scored as an answer") } - if got != "не знаю." { - t.Errorf("fallback text = %q, want \"не знаю.\"", got) + if !isFallback(t, fbQueryUnknown, "", got) { + t.Errorf("fallback text = %q, want a %q variant", got, fbQueryUnknown) } if !strings.Contains(err.Error(), "empty") { t.Errorf("error = %v; want it to name the empty response", err) diff --git a/internal/phraser/fallbacks_test.go b/internal/phraser/fallbacks_test.go new file mode 100644 index 0000000..ecec0ec --- /dev/null +++ b/internal/phraser/fallbacks_test.go @@ -0,0 +1,57 @@ +package phraser + +import ( + "math/rand" + "strings" + "testing" +) + +// The embedded file must load, or the daemon speaks from hardFloor and nobody +// finds out until he hears the wrong words. +func TestFallbacksLoad(t *testing.T) { + fb, err := LoadFallbacks(rand.NewSource(1)) + if err != nil { + t.Fatalf("LoadFallbacks: %v", err) + } + if got := fb.FromSources("два литра"); !strings.Contains(got, "два литра") { + t.Errorf("FromSources = %q, want the sources in it", got) + } + if fb.WorldGap() != hardFloor[fbWorldGap] { + t.Errorf("WorldGap = %q, want the fixed wording %q", fb.WorldGap(), hardFloor[fbWorldGap]) + } +} + +// A broken or missing file must not take her last words away: every accessor +// answers from the literal it replaced. +func TestNilFallbacksAnswerFromTheHardFloor(t *testing.T) { + var fb *Fallbacks + if got := fb.Chat(); got != hardFloor[fbChat] { + t.Errorf("Chat = %q, want %q", got, hardFloor[fbChat]) + } + if got := fb.Unknown(); got != hardFloor[fbQueryUnknown] { + t.Errorf("Unknown = %q, want %q", got, hardFloor[fbQueryUnknown]) + } + if got := fb.FromSources("два литра"); got != "вот что я нашла: два литра" { + t.Errorf("FromSources = %q", got) + } + if got := fb.WorldGap(); got != hardFloor[fbWorldGap] { + t.Errorf("WorldGap = %q", got) + } +} + +// Hearing the identical words every time a request fails is how a failure stops +// registering as one. +func TestFallbacksDoNotRepeat(t *testing.T) { + fb, err := LoadFallbacks(rand.NewSource(7)) + if err != nil { + t.Fatalf("LoadFallbacks: %v", err) + } + prev := fb.Chat() + for i := 0; i < 20; i++ { + got := fb.Chat() + if got == prev { + t.Fatalf("chat repeated %q on turn %d", got, i) + } + prev = got + } +} diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index b019eac..b06ecac 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -449,14 +449,14 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes [] sys, prompt := p.knowledgePrompt(utterance) resp, err := p.chatWithSystem(ctx, sys, prompt, 768) if err != nil { - return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", err) + return UnknownFallback(), fmt.Errorf("phrase query (knowledge): %w", err) } if resp == "" { - return "не знаю.", errEmptyResponse + return UnknownFallback(), errEmptyResponse } text, _, perr := parseResponseMood(resp) if perr != nil { - return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", perr) + return UnknownFallback(), fmt.Errorf("phrase query (knowledge): %w", perr) } if text != "" { return text, nil @@ -472,7 +472,7 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes [] if cause == nil { cause = perr } - return "вот что я нашла: " + strings.Join(notes, "; "), + return SourcesFallback(strings.Join(notes, "; ")), fmt.Errorf("phrase query (evidence): %w", cause) } if text != "" { @@ -501,11 +501,11 @@ func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history [ resp, err := p.chatWithMessages(ctx, msgs, 768) if err != nil { - return ChatFallback, fmt.Errorf("phrase chat: %w", err) + return ChatFallback(), fmt.Errorf("phrase chat: %w", err) } text, _, perr := parseResponseMood(resp) if perr != nil { - return ChatFallback, fmt.Errorf("phrase chat: %w", perr) + return ChatFallback(), fmt.Errorf("phrase chat: %w", perr) } if text != "" { return text, nil diff --git a/internal/phraser/phraser.go b/internal/phraser/phraser.go index 5918ac9..8fb9130 100644 --- a/internal/phraser/phraser.go +++ b/internal/phraser/phraser.go @@ -66,28 +66,19 @@ type Stub struct{} // NewStub builds the floor phraser. no config — the Stub is stateless. func NewStub() *Stub { return &Stub{} } -// ChatFallback — what she says on the chat path when the model gave her -// nothing to say. It replaced "поговорили.", which reads as a summary of a -// conversation that did not happen. Said out loud this one is an admission, -// which is what it is. -const ChatFallback = "даже не знаю, что сказать." - // PhraseChat returns a stub reply — the LLMPhraser replaces this with a // prompted response from the model. The history parameter is accepted but // ignored at the stub level (the production impl uses it for multi-turn). func (s *Stub) PhraseChat(_ context.Context, _ string, _ []dialogue.Turn) (string, error) { - return ChatFallback, nil + return ChatFallback(), nil } // PhraseQuery returns a deterministic summary of the best matching notes. func (s *Stub) PhraseQuery(_ context.Context, _ string, notes []string) (string, error) { if len(notes) == 0 { - return "не знаю.", nil + return UnknownFallback(), nil } - if len(notes) == 1 { - return "вот что я нашла: " + notes[0], nil - } - return "вот что я нашла: " + strings.Join(notes, "; "), nil + return SourcesFallback(strings.Join(notes, "; ")), nil } // Close implements Phraser.Close (no-op for the stub).