phraser, mavend: read the fallbacks from the file (V-501)

The accessors are functions now, so the call sites that compared against one
literal compare against the entry instead: IsUnknownFallback and
IsSourcesFallback in the daemon tests, the entry key in the phraser tests. A
reworded variant no longer breaks a Go test.

The eval scores every variant on the persona checks the nudges already pass.
This commit is contained in:
2026-08-04 01:19:41 +04:00
parent 4fdce3ca2c
commit 865623ef3e
10 changed files with 154 additions and 45 deletions
+1 -1
View File
@@ -66,7 +66,7 @@ func (h *reactiveHandler) actionChat(ctx context.Context, dec router.Decision) s
log.Printf("voice: chat: %v", err)
}
if reply == "" {
return phraser.ChatFallback
return phraser.ChatFallback()
}
return reply
}
+1 -1
View File
@@ -798,7 +798,7 @@ func (h *reactiveHandler) queryGeneral(ctx context.Context, t *queryTurn) (strin
reply, err := h.phraseWorld(ctx, t.dec.Utterance, nil)
if errors.Is(err, phraser.ErrNoWorldModel) {
log.Printf("voice: %q needs the world model and it is not available", t.dec.Utterance)
return worldGap, true
return worldGap(), true
}
if err != nil || reply == "" {
return "не знаю.", true
+3 -3
View File
@@ -121,8 +121,8 @@ func TestQueryRecallNoteCanWin(t *testing.T) {
{text: "выучил пару аккордов", score: 0.50, kind: "note"},
})
reply := askQuery(t, h, q)
if want := "вот что я нашла: молоко стоит в холодильнике"; reply != want {
t.Errorf("reply %q, want %q", reply, want)
if !phraser.IsSourcesFallback(reply, "молоко стоит в холодильнике") {
t.Errorf("reply %q, want the note read back", reply)
}
// One text, the winning memory's — the answer came from the memory
// pass, not from handing the phraser every note in the table.
@@ -151,7 +151,7 @@ func TestQueryRecallNoteCanWin(t *testing.T) {
{text: "молоко стоит в холодильнике", score: 0.860, kind: "note"},
{text: "молоко закончилось", score: 0.858, kind: "note"},
})
if reply := askQuery(t, h, q); reply != "не знаю." {
if reply := askQuery(t, h, q); !phraser.IsUnknownFallback(reply) {
t.Errorf("reply %q, want silence", reply)
}
})
+5 -1
View File
@@ -23,7 +23,11 @@ type worldPhraser interface {
// question about his meeting came back as a swimming competition in Nottingham.
// Naming the gap is the rule CLAUDE.md already applies to a sibling service
// being down.
const worldGap = "сейчас не могу ответить — большая модель недоступна, а придумывать не хочу."
//
// The wording lives in fallbacks_ru_v1.json and is fixed there, not picked from
// variants: this sentence names one specific gap and must not drift into a
// general "I don't know".
func worldGap() string { return phraser.WorldGap() }
// phraseWorld asks the world model, or reports the gap.
//
+6 -6
View File
@@ -35,7 +35,7 @@ func TestQueryGeneralNamesTheGap(t *testing.T) {
if !ok {
t.Fatal("queryGeneral passed on the last source in the chain")
}
if reply != worldGap {
if reply != worldGap() {
t.Fatalf("reply = %q, want the named gap", reply)
}
if g.worldCalls != 1 {
@@ -51,7 +51,7 @@ func TestQueryGeneralWithoutAWorldModelIsUnchanged(t *testing.T) {
if !ok {
t.Fatal("queryGeneral passed on the last source in the chain")
}
if reply != "не знаю." {
if !phraser.IsUnknownFallback(reply) {
t.Fatalf("reply = %q, want the Stub's answer", reply)
}
}
@@ -61,12 +61,12 @@ func TestQueryGeneralWithoutAWorldModelIsUnchanged(t *testing.T) {
// English in it.
func TestWorldGapIsInPersona(t *testing.T) {
for _, bad := range []string{"вы", "ваш", "рад ", "дорогой", "милый"} {
if strings.Contains(worldGap, bad) {
t.Errorf("the gap phrase contains %q: %s", bad, worldGap)
if strings.Contains(worldGap(), bad) {
t.Errorf("the gap phrase contains %q: %s", bad, worldGap())
}
}
if strings.ContainsAny(worldGap, "abcdefghijklmnopqrstuvwxyz") {
t.Errorf("the gap phrase has Latin letters in it: %s", worldGap)
if strings.ContainsAny(worldGap(), "abcdefghijklmnopqrstuvwxyz") {
t.Errorf("the gap phrase has Latin letters in it: %s", worldGap())
}
}
+40
View File
@@ -0,0 +1,40 @@
package eval
import (
"math/rand"
"strings"
"testing"
"github.com/kami/maven/internal/phraser"
)
// TestFallbackPersona scores every line in fallbacks_ru_v1.json on the persona
// checks the nudges are already held to. These lines are heard out loud, and
// they live in a JSON file now, so a reworded variant that says "рад" or "вы"
// would otherwise reach him with nothing between it and the speaker.
//
// Only the persona checks run. Mood and topic belong to a nudge, and these are
// not nudges: they are what she says when there is no answer.
func TestFallbackPersona(t *testing.T) {
fb, err := phraser.LoadFallbacks(rand.NewSource(20260804))
if err != nil {
t.Fatalf("LoadFallbacks: %v", err)
}
persona := map[string]bool{
CheckLang: true, CheckFeminine: true, CheckHisGender: true,
CheckAddress: true, CheckCringe: true, CheckLength: true,
}
variants := fb.Variants()
if len(variants) == 0 {
t.Fatal("no variants — the file loaded empty")
}
for _, v := range variants {
// {sources} stands for his own notes and never carries persona of its own.
body := strings.ReplaceAll(v, "{sources}", "два литра")
for _, r := range RunChecks(Case{}, body, "neutral") {
if persona[r.Name] && !r.Pass {
t.Errorf("%q fails %s: %s", v, r.Name, r.Detail)
}
}
}
}
+32 -15
View File
@@ -8,12 +8,28 @@ import (
"testing"
)
// isFallback — the text she says is picked from that entry's variants, so a test
// pins the entry rather than the wording. Pinning one line would make editing
// fallbacks_ru_v1.json break Go tests, which is the coupling this file removed.
func isFallback(t *testing.T, key, sources, got string) bool {
t.Helper()
e, ok := DefaultFallbacks().file.Entries[key]
if !ok {
t.Fatalf("no fallback entry %q", key)
}
for _, v := range e.Variants {
if strings.ReplaceAll(v, "{sources}", sources) == got {
return true
}
}
return false
}
// A dead server must be distinguishable from bad phrasing. Both PhraseChat and
// PhraseQuery keep the turn alive with canned text — ChatFallback, "не знаю.",
// "вот что я нашла: …" — and every one of those is also a legitimate reply, so
// the text alone cannot say which happened. The error is the only signal, and
// before Vikunja #397 it was dropped: the talk scorer reported a full run with
// zero errors off a server that answered nothing.
// PhraseQuery keep the turn alive with canned text — and every one of those
// lines is also a legitimate reply, so the text alone cannot say which happened.
// The error is the only signal, and before Vikunja #397 it was dropped: the talk
// scorer reported a full run with zero errors off a server that answered nothing.
func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "model not loaded", http.StatusServiceUnavailable)
@@ -22,19 +38,20 @@ func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
p := NewLLMPhraserAt(srv.URL, Config{})
cases := []struct {
name string
call func() (string, error)
want string
name string
call func() (string, error)
key string
sources string
}{
{"chat", func() (string, error) {
return p.PhraseChat(context.Background(), "как дела", nil)
}, ChatFallback},
}, fbChat, ""},
{"knowledge", func() (string, error) {
return p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
}, "не знаю."},
}, fbQueryUnknown, ""},
{"evidence", func() (string, error) {
return p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"два литра"})
}, "вот что я нашла: два литра"},
}, fbQuerySources, "два литра"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -42,8 +59,8 @@ func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
if err == nil {
t.Fatalf("no error from a dead server; the scorer would count this as bad phrasing")
}
if got != c.want {
t.Errorf("fallback text = %q, want %q — the daemon still has to say something", got, c.want)
if !isFallback(t, c.key, c.sources, got) {
t.Errorf("fallback text = %q, want a %q variant — the daemon still has to say something", got, c.key)
}
})
}
@@ -63,8 +80,8 @@ func TestEmptyKnowledgeAnswerIsAnError(t *testing.T) {
if err == nil {
t.Fatal("an empty response scored as an answer")
}
if got != "не знаю." {
t.Errorf("fallback text = %q, want \"не знаю.\"", got)
if !isFallback(t, fbQueryUnknown, "", got) {
t.Errorf("fallback text = %q, want a %q variant", got, fbQueryUnknown)
}
if !strings.Contains(err.Error(), "empty") {
t.Errorf("error = %v; want it to name the empty response", err)
+57
View File
@@ -0,0 +1,57 @@
package phraser
import (
"math/rand"
"strings"
"testing"
)
// The embedded file must load, or the daemon speaks from hardFloor and nobody
// finds out until he hears the wrong words.
func TestFallbacksLoad(t *testing.T) {
fb, err := LoadFallbacks(rand.NewSource(1))
if err != nil {
t.Fatalf("LoadFallbacks: %v", err)
}
if got := fb.FromSources("два литра"); !strings.Contains(got, "два литра") {
t.Errorf("FromSources = %q, want the sources in it", got)
}
if fb.WorldGap() != hardFloor[fbWorldGap] {
t.Errorf("WorldGap = %q, want the fixed wording %q", fb.WorldGap(), hardFloor[fbWorldGap])
}
}
// A broken or missing file must not take her last words away: every accessor
// answers from the literal it replaced.
func TestNilFallbacksAnswerFromTheHardFloor(t *testing.T) {
var fb *Fallbacks
if got := fb.Chat(); got != hardFloor[fbChat] {
t.Errorf("Chat = %q, want %q", got, hardFloor[fbChat])
}
if got := fb.Unknown(); got != hardFloor[fbQueryUnknown] {
t.Errorf("Unknown = %q, want %q", got, hardFloor[fbQueryUnknown])
}
if got := fb.FromSources("два литра"); got != "вот что я нашла: два литра" {
t.Errorf("FromSources = %q", got)
}
if got := fb.WorldGap(); got != hardFloor[fbWorldGap] {
t.Errorf("WorldGap = %q", got)
}
}
// Hearing the identical words every time a request fails is how a failure stops
// registering as one.
func TestFallbacksDoNotRepeat(t *testing.T) {
fb, err := LoadFallbacks(rand.NewSource(7))
if err != nil {
t.Fatalf("LoadFallbacks: %v", err)
}
prev := fb.Chat()
for i := 0; i < 20; i++ {
got := fb.Chat()
if got == prev {
t.Fatalf("chat repeated %q on turn %d", got, i)
}
prev = got
}
}
+6 -6
View File
@@ -449,14 +449,14 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
sys, prompt := p.knowledgePrompt(utterance)
resp, err := p.chatWithSystem(ctx, sys, prompt, 768)
if err != nil {
return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", err)
return UnknownFallback(), fmt.Errorf("phrase query (knowledge): %w", err)
}
if resp == "" {
return "не знаю.", errEmptyResponse
return UnknownFallback(), errEmptyResponse
}
text, _, perr := parseResponseMood(resp)
if perr != nil {
return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", perr)
return UnknownFallback(), fmt.Errorf("phrase query (knowledge): %w", perr)
}
if text != "" {
return text, nil
@@ -472,7 +472,7 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
if cause == nil {
cause = perr
}
return "вот что я нашла: " + strings.Join(notes, "; "),
return SourcesFallback(strings.Join(notes, "; ")),
fmt.Errorf("phrase query (evidence): %w", cause)
}
if text != "" {
@@ -501,11 +501,11 @@ func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history [
resp, err := p.chatWithMessages(ctx, msgs, 768)
if err != nil {
return ChatFallback, fmt.Errorf("phrase chat: %w", err)
return ChatFallback(), fmt.Errorf("phrase chat: %w", err)
}
text, _, perr := parseResponseMood(resp)
if perr != nil {
return ChatFallback, fmt.Errorf("phrase chat: %w", perr)
return ChatFallback(), fmt.Errorf("phrase chat: %w", perr)
}
if text != "" {
return text, nil
+3 -12
View File
@@ -66,28 +66,19 @@ type Stub struct{}
// NewStub builds the floor phraser. no config — the Stub is stateless.
func NewStub() *Stub { return &Stub{} }
// ChatFallback — what she says on the chat path when the model gave her
// nothing to say. It replaced "поговорили.", which reads as a summary of a
// conversation that did not happen. Said out loud this one is an admission,
// which is what it is.
const ChatFallback = "даже не знаю, что сказать."
// PhraseChat returns a stub reply — the LLMPhraser replaces this with a
// prompted response from the model. The history parameter is accepted but
// ignored at the stub level (the production impl uses it for multi-turn).
func (s *Stub) PhraseChat(_ context.Context, _ string, _ []dialogue.Turn) (string, error) {
return ChatFallback, nil
return ChatFallback(), nil
}
// PhraseQuery returns a deterministic summary of the best matching notes.
func (s *Stub) PhraseQuery(_ context.Context, _ string, notes []string) (string, error) {
if len(notes) == 0 {
return "не знаю.", nil
return UnknownFallback(), nil
}
if len(notes) == 1 {
return "вот что я нашла: " + notes[0], nil
}
return "вот что я нашла: " + strings.Join(notes, "; "), nil
return SourcesFallback(strings.Join(notes, "; ")), nil
}
// Close implements Phraser.Close (no-op for the stub).