Compare commits

...

2 Commits

Author SHA1 Message Date
claude 865623ef3e phraser, mavend: read the fallbacks from the file (V-501)
The accessors are functions now, so the call sites that compared against one
literal compare against the entry instead: IsUnknownFallback and
IsSourcesFallback in the daemon tests, the entry key in the phraser tests. A
reworded variant no longer breaks a Go test.

The eval scores every variant on the persona checks the nudges already pass.
2026-08-04 01:19:41 +04:00
claude 4fdce3ca2c phraser: put the phrasing fallbacks in a versioned json (V-501)
Four lines he hears out loud lived as string literals in three Go files, so
rewording one meant a rebuild. They move to fallbacks_ru_v1.json on the shape
nudges_ru_v1.json already uses: embedded, schema-versioned, several variants,
never the same one twice running.

The gap phrase is marked fixed, because it names one specific missing model and
must not drift into a general "I do not know". Every accessor falls back to the
literal it replaced, including on a nil receiver: these strings exist because
something already failed, so a broken template file must not take her last
words away.
2026-08-04 01:19:41 +04:00
12 changed files with 434 additions and 45 deletions
+1 -1
View File
@@ -66,7 +66,7 @@ func (h *reactiveHandler) actionChat(ctx context.Context, dec router.Decision) s
log.Printf("voice: chat: %v", err)
}
if reply == "" {
return phraser.ChatFallback
return phraser.ChatFallback()
}
return reply
}
+1 -1
View File
@@ -798,7 +798,7 @@ func (h *reactiveHandler) queryGeneral(ctx context.Context, t *queryTurn) (strin
reply, err := h.phraseWorld(ctx, t.dec.Utterance, nil)
if errors.Is(err, phraser.ErrNoWorldModel) {
log.Printf("voice: %q needs the world model and it is not available", t.dec.Utterance)
return worldGap, true
return worldGap(), true
}
if err != nil || reply == "" {
return "не знаю.", true
+3 -3
View File
@@ -121,8 +121,8 @@ func TestQueryRecallNoteCanWin(t *testing.T) {
{text: "выучил пару аккордов", score: 0.50, kind: "note"},
})
reply := askQuery(t, h, q)
if want := "вот что я нашла: молоко стоит в холодильнике"; reply != want {
t.Errorf("reply %q, want %q", reply, want)
if !phraser.IsSourcesFallback(reply, "молоко стоит в холодильнике") {
t.Errorf("reply %q, want the note read back", reply)
}
// One text, the winning memory's — the answer came from the memory
// pass, not from handing the phraser every note in the table.
@@ -151,7 +151,7 @@ func TestQueryRecallNoteCanWin(t *testing.T) {
{text: "молоко стоит в холодильнике", score: 0.860, kind: "note"},
{text: "молоко закончилось", score: 0.858, kind: "note"},
})
if reply := askQuery(t, h, q); reply != "не знаю." {
if reply := askQuery(t, h, q); !phraser.IsUnknownFallback(reply) {
t.Errorf("reply %q, want silence", reply)
}
})
+5 -1
View File
@@ -23,7 +23,11 @@ type worldPhraser interface {
// question about his meeting came back as a swimming competition in Nottingham.
// Naming the gap is the rule CLAUDE.md already applies to a sibling service
// being down.
const worldGap = "сейчас не могу ответить — большая модель недоступна, а придумывать не хочу."
//
// The wording lives in fallbacks_ru_v1.json and is fixed there, not picked from
// variants: this sentence names one specific gap and must not drift into a
// general "I don't know".
func worldGap() string { return phraser.WorldGap() }
// phraseWorld asks the world model, or reports the gap.
//
+6 -6
View File
@@ -35,7 +35,7 @@ func TestQueryGeneralNamesTheGap(t *testing.T) {
if !ok {
t.Fatal("queryGeneral passed on the last source in the chain")
}
if reply != worldGap {
if reply != worldGap() {
t.Fatalf("reply = %q, want the named gap", reply)
}
if g.worldCalls != 1 {
@@ -51,7 +51,7 @@ func TestQueryGeneralWithoutAWorldModelIsUnchanged(t *testing.T) {
if !ok {
t.Fatal("queryGeneral passed on the last source in the chain")
}
if reply != "не знаю." {
if !phraser.IsUnknownFallback(reply) {
t.Fatalf("reply = %q, want the Stub's answer", reply)
}
}
@@ -61,12 +61,12 @@ func TestQueryGeneralWithoutAWorldModelIsUnchanged(t *testing.T) {
// English in it.
func TestWorldGapIsInPersona(t *testing.T) {
for _, bad := range []string{"вы", "ваш", "рад ", "дорогой", "милый"} {
if strings.Contains(worldGap, bad) {
t.Errorf("the gap phrase contains %q: %s", bad, worldGap)
if strings.Contains(worldGap(), bad) {
t.Errorf("the gap phrase contains %q: %s", bad, worldGap())
}
}
if strings.ContainsAny(worldGap, "abcdefghijklmnopqrstuvwxyz") {
t.Errorf("the gap phrase has Latin letters in it: %s", worldGap)
if strings.ContainsAny(worldGap(), "abcdefghijklmnopqrstuvwxyz") {
t.Errorf("the gap phrase has Latin letters in it: %s", worldGap())
}
}
+40
View File
@@ -0,0 +1,40 @@
package eval
import (
"math/rand"
"strings"
"testing"
"github.com/kami/maven/internal/phraser"
)
// TestFallbackPersona scores every line in fallbacks_ru_v1.json on the persona
// checks the nudges are already held to. These lines are heard out loud, and
// they live in a JSON file now, so a reworded variant that says "рад" or "вы"
// would otherwise reach him with nothing between it and the speaker.
//
// Only the persona checks run. Mood and topic belong to a nudge, and these are
// not nudges: they are what she says when there is no answer.
func TestFallbackPersona(t *testing.T) {
fb, err := phraser.LoadFallbacks(rand.NewSource(20260804))
if err != nil {
t.Fatalf("LoadFallbacks: %v", err)
}
persona := map[string]bool{
CheckLang: true, CheckFeminine: true, CheckHisGender: true,
CheckAddress: true, CheckCringe: true, CheckLength: true,
}
variants := fb.Variants()
if len(variants) == 0 {
t.Fatal("no variants — the file loaded empty")
}
for _, v := range variants {
// {sources} stands for his own notes and never carries persona of its own.
body := strings.ReplaceAll(v, "{sources}", "два литра")
for _, r := range RunChecks(Case{}, body, "neutral") {
if persona[r.Name] && !r.Pass {
t.Errorf("%q fails %s: %s", v, r.Name, r.Detail)
}
}
}
}
+32 -15
View File
@@ -8,12 +8,28 @@ import (
"testing"
)
// isFallback — the text she says is picked from that entry's variants, so a test
// pins the entry rather than the wording. Pinning one line would make editing
// fallbacks_ru_v1.json break Go tests, which is the coupling this file removed.
func isFallback(t *testing.T, key, sources, got string) bool {
t.Helper()
e, ok := DefaultFallbacks().file.Entries[key]
if !ok {
t.Fatalf("no fallback entry %q", key)
}
for _, v := range e.Variants {
if strings.ReplaceAll(v, "{sources}", sources) == got {
return true
}
}
return false
}
// A dead server must be distinguishable from bad phrasing. Both PhraseChat and
// PhraseQuery keep the turn alive with canned text — ChatFallback, "не знаю.",
// "вот что я нашла: …" — and every one of those is also a legitimate reply, so
// the text alone cannot say which happened. The error is the only signal, and
// before Vikunja #397 it was dropped: the talk scorer reported a full run with
// zero errors off a server that answered nothing.
// PhraseQuery keep the turn alive with canned text — and every one of those
// lines is also a legitimate reply, so the text alone cannot say which happened.
// The error is the only signal, and before Vikunja #397 it was dropped: the talk
// scorer reported a full run with zero errors off a server that answered nothing.
func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "model not loaded", http.StatusServiceUnavailable)
@@ -22,19 +38,20 @@ func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
p := NewLLMPhraserAt(srv.URL, Config{})
cases := []struct {
name string
call func() (string, error)
want string
name string
call func() (string, error)
key string
sources string
}{
{"chat", func() (string, error) {
return p.PhraseChat(context.Background(), "как дела", nil)
}, ChatFallback},
}, fbChat, ""},
{"knowledge", func() (string, error) {
return p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
}, "не знаю."},
}, fbQueryUnknown, ""},
{"evidence", func() (string, error) {
return p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"два литра"})
}, "вот что я нашла: два литра"},
}, fbQuerySources, "два литра"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -42,8 +59,8 @@ func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
if err == nil {
t.Fatalf("no error from a dead server; the scorer would count this as bad phrasing")
}
if got != c.want {
t.Errorf("fallback text = %q, want %q — the daemon still has to say something", got, c.want)
if !isFallback(t, c.key, c.sources, got) {
t.Errorf("fallback text = %q, want a %q variant — the daemon still has to say something", got, c.key)
}
})
}
@@ -63,8 +80,8 @@ func TestEmptyKnowledgeAnswerIsAnError(t *testing.T) {
if err == nil {
t.Fatal("an empty response scored as an answer")
}
if got != "не знаю." {
t.Errorf("fallback text = %q, want \"не знаю.\"", got)
if !isFallback(t, fbQueryUnknown, "", got) {
t.Errorf("fallback text = %q, want a %q variant", got, fbQueryUnknown)
}
if !strings.Contains(err.Error(), "empty") {
t.Errorf("error = %v; want it to name the empty response", err)
+238
View File
@@ -0,0 +1,238 @@
package phraser
// The phrasing fallbacks — what she says when the model gave her nothing usable.
//
// They were four string literals spread across phraser.go, llmphraser.go and
// cmd/mavend/worldmodel.go. Every one of them is a line he hears out loud, so
// rewording one was a Go edit, a rebuild and a redeploy for what is product copy.
// This is the same shape nudges_ru_v1.json already uses for nudges: embedded,
// schema-versioned, several variants, and never the same variant twice running.
//
// The floor under the floor is deliberate. These strings exist because something
// already failed, so a broken template file must not be able to take the last
// words she has: every accessor falls back to the literal it replaced.
import (
_ "embed"
"encoding/json"
"fmt"
"log"
"math/rand"
"strings"
"sync"
"time"
)
//go:embed fallbacks_ru_v1.json
var fallbackJSON []byte
// FallbackSchemaVersion — the version this code understands. Its own constant,
// not shared with the nudge templates or the eval fixtures: two files that change
// on different days cannot be versioned by one number (Vikunja #397).
const FallbackSchemaVersion = 1
// The entry keys. Every one of them is read by a method below, so a typo in the
// file is caught at load rather than at the moment she needs the words.
const (
fbChat = "chat"
fbQueryUnknown = "query_unknown"
fbQuerySources = "query_sources"
fbWorldGap = "world_gap"
)
// fbKeys — every key the code requires the file to define.
var fbKeys = []string{fbChat, fbQueryUnknown, fbQuerySources, fbWorldGap}
// hardFloor — the literal each key falls back to when the file is unusable.
// These are the exact strings that lived in Go before this file existed.
var hardFloor = map[string]string{
fbChat: "даже не знаю, что сказать.",
fbQueryUnknown: "не знаю.",
fbQuerySources: "вот что я нашла: {sources}",
fbWorldGap: "сейчас не могу ответить — большая модель недоступна, а придумывать не хочу.",
}
type fallbackEntry struct {
// Fixed — one variant, never picked between. For wording that must not drift
// from turn to turn, like the gap phrase that names an unavailable model.
Fixed bool `json:"fixed"`
Variants []string `json:"variants"`
}
type fallbackFile struct {
SchemaVersion int `json:"schema_version"`
Name string `json:"name"`
Notes []string `json:"notes"`
Entries map[string]fallbackEntry `json:"entries"`
}
// Fallbacks picks a hand-written Russian fallback line.
//
// Safe for concurrent use. Never the same variant twice in a row for the same
// entry: hearing the identical words every time a request fails is how a failure
// stops registering as one.
type Fallbacks struct {
mu sync.Mutex
rnd *rand.Rand
last map[string]string
file fallbackFile
}
// LoadFallbacks reads the embedded file. Pass a source to make the picking
// reproducible in tests; nil seeds from the clock.
func LoadFallbacks(src rand.Source) (*Fallbacks, error) {
var f fallbackFile
if err := json.Unmarshal(fallbackJSON, &f); err != nil {
return nil, fmt.Errorf("fallbacks: parse: %w", err)
}
if f.SchemaVersion != FallbackSchemaVersion {
return nil, fmt.Errorf("fallbacks: schema_version %d, want %d",
f.SchemaVersion, FallbackSchemaVersion)
}
for _, k := range fbKeys {
e, ok := f.Entries[k]
if !ok || len(e.Variants) == 0 {
return nil, fmt.Errorf("fallbacks: entry %q is missing or empty", k)
}
if e.Fixed && len(e.Variants) != 1 {
return nil, fmt.Errorf("fallbacks: entry %q is fixed but has %d variants", k, len(e.Variants))
}
}
// query_sources is the one entry whose whole job is to read something back,
// so a variant without the placeholder would silently drop the sources.
for _, v := range f.Entries[fbQuerySources].Variants {
if !strings.Contains(v, "{sources}") {
return nil, fmt.Errorf("fallbacks: %q variant %q does not use {sources}", fbQuerySources, v)
}
}
if src == nil {
src = rand.NewSource(time.Now().UnixNano())
}
return &Fallbacks{rnd: rand.New(src), last: map[string]string{}, file: f}, nil
}
// text returns one variant for key, with {sources} filled in. A nil receiver is
// the unloadable-file case and answers from hardFloor, so the caller never has
// to check whether the templates loaded.
func (f *Fallbacks) text(key, sources string) string {
tmpl := hardFloor[key]
if f != nil {
if e, ok := f.file.Entries[key]; ok && len(e.Variants) > 0 {
tmpl = f.pick(key, e)
}
}
return strings.ReplaceAll(tmpl, "{sources}", sources)
}
// pick chooses at random, skipping whatever this entry said last time.
func (f *Fallbacks) pick(key string, e fallbackEntry) string {
f.mu.Lock()
defer f.mu.Unlock()
choices := e.Variants
if len(choices) > 1 {
fresh := make([]string, 0, len(choices))
for _, v := range choices {
if v != f.last[key] {
fresh = append(fresh, v)
}
}
if len(fresh) > 0 {
choices = fresh
}
}
got := choices[f.rnd.Intn(len(choices))]
f.last[key] = got
return got
}
// Chat — nothing usable came back on the chat path.
func (f *Fallbacks) Chat() string { return f.text(fbChat, "") }
// Unknown — a question she cannot answer and will not guess at.
func (f *Fallbacks) Unknown() string { return f.text(fbQueryUnknown, "") }
// FromSources — read back what she was handed, because phrasing it failed.
func (f *Fallbacks) FromSources(sources string) string {
return f.text(fbQuerySources, sources)
}
// WorldGap — the world model is the one configured to answer and it is not
// answering. Fixed wording: it names a specific gap, and a variant set here
// would let "the big model is asleep" drift into "I don't know".
func (f *Fallbacks) WorldGap() string { return f.text(fbWorldGap, "") }
// Variants returns every line the file can produce, for the persona scorer.
// Order is stable so a failure names the same variant twice running.
func (f *Fallbacks) Variants() []string {
var out []string
for _, k := range fbKeys {
out = append(out, f.file.Entries[k].Variants...)
}
return out
}
// The process-wide instance. Package-level because these lines are needed on
// paths that have no phraser to hand — cmd/mavend names the world gap without
// one — and because a template file that is embedded and validated at load has
// nothing per-instance to configure.
var (
fallbackOnce sync.Once
fallbacks *Fallbacks
)
// DefaultFallbacks returns the shared instance, loading it on first use. A
// broken file logs once and leaves a nil *Fallbacks, which still answers from
// hardFloor — a daemon must not fail to boot over its own copy deck.
func DefaultFallbacks() *Fallbacks {
fallbackOnce.Do(func() {
fb, err := LoadFallbacks(nil)
if err != nil {
log.Printf("phraser: fallbacks unavailable, using the built-in lines: %v", err)
return
}
fallbacks = fb
})
return fallbacks
}
// ChatFallback — what she says when the chat path produced nothing.
func ChatFallback() string { return DefaultFallbacks().Chat() }
// UnknownFallback — what she says when she has no answer and will not invent one.
func UnknownFallback() string { return DefaultFallbacks().Unknown() }
// SourcesFallback — read the sources back rather than ship a broken fragment.
func SourcesFallback(sources string) string { return DefaultFallbacks().FromSources(sources) }
// WorldGap — what he hears when the world model is configured and unreachable.
func WorldGap() string { return DefaultFallbacks().WorldGap() }
// matches reports whether text is a line the given entry could have produced.
// A caller that has to recognise a fallback cannot compare against one literal
// any more, because the entry picks between variants.
func (f *Fallbacks) matches(key, sources, text string) bool {
if strings.ReplaceAll(hardFloor[key], "{sources}", sources) == text {
return true
}
if f == nil {
return false
}
for _, v := range f.file.Entries[key].Variants {
if strings.ReplaceAll(v, "{sources}", sources) == text {
return true
}
}
return false
}
// IsUnknownFallback reports whether text is one of her "I do not know" lines.
// The daemon tests read it to tell an answer from a shrug.
func IsUnknownFallback(text string) bool {
return DefaultFallbacks().matches(fbQueryUnknown, "", text)
}
// IsSourcesFallback reports whether text is sources read back verbatim.
func IsSourcesFallback(text, sources string) bool {
return DefaultFallbacks().matches(fbQuerySources, sources, text)
}
+42
View File
@@ -0,0 +1,42 @@
{
"schema_version": 1,
"name": "russian phrasing fallbacks v1",
"notes": [
"What she says when the model gave her nothing usable. Edit the wording here, no Go changes needed.",
"Rules: she is feminine about herself, he is a man addressed as ты. Never вы/вас/ваш, never plural imperatives, never он/его about him. No pet names.",
"These are heard after a failure, so they stay short and admit the gap. None of them may claim knowledge she does not have.",
"Placeholders: {sources} the notes or passages she was handed. A variant whose placeholder has no value is skipped, so every entry needs at least one variant with no placeholder — except query_sources, which exists only to read sources back.",
"fixed: true means exactly one variant and no picking. Used where the wording is load-bearing and must not drift between turns."
],
"entries": {
"chat": {
"variants": [
"даже не знаю, что сказать.",
"не могу найти слов.",
"мысль ускользнула, повтори?",
"у меня сейчас пусто в голове."
]
},
"query_unknown": {
"variants": [
"не знаю.",
"не знаю, честно.",
"тут я пас.",
"не скажу, не знаю."
]
},
"query_sources": {
"variants": [
"вот что я нашла: {sources}",
"нашла вот это: {sources}",
"есть только это: {sources}"
]
},
"world_gap": {
"fixed": true,
"variants": [
"сейчас не могу ответить — большая модель недоступна, а придумывать не хочу."
]
}
}
}
+57
View File
@@ -0,0 +1,57 @@
package phraser
import (
"math/rand"
"strings"
"testing"
)
// The embedded file must load, or the daemon speaks from hardFloor and nobody
// finds out until he hears the wrong words.
func TestFallbacksLoad(t *testing.T) {
fb, err := LoadFallbacks(rand.NewSource(1))
if err != nil {
t.Fatalf("LoadFallbacks: %v", err)
}
if got := fb.FromSources("два литра"); !strings.Contains(got, "два литра") {
t.Errorf("FromSources = %q, want the sources in it", got)
}
if fb.WorldGap() != hardFloor[fbWorldGap] {
t.Errorf("WorldGap = %q, want the fixed wording %q", fb.WorldGap(), hardFloor[fbWorldGap])
}
}
// A broken or missing file must not take her last words away: every accessor
// answers from the literal it replaced.
func TestNilFallbacksAnswerFromTheHardFloor(t *testing.T) {
var fb *Fallbacks
if got := fb.Chat(); got != hardFloor[fbChat] {
t.Errorf("Chat = %q, want %q", got, hardFloor[fbChat])
}
if got := fb.Unknown(); got != hardFloor[fbQueryUnknown] {
t.Errorf("Unknown = %q, want %q", got, hardFloor[fbQueryUnknown])
}
if got := fb.FromSources("два литра"); got != "вот что я нашла: два литра" {
t.Errorf("FromSources = %q", got)
}
if got := fb.WorldGap(); got != hardFloor[fbWorldGap] {
t.Errorf("WorldGap = %q", got)
}
}
// Hearing the identical words every time a request fails is how a failure stops
// registering as one.
func TestFallbacksDoNotRepeat(t *testing.T) {
fb, err := LoadFallbacks(rand.NewSource(7))
if err != nil {
t.Fatalf("LoadFallbacks: %v", err)
}
prev := fb.Chat()
for i := 0; i < 20; i++ {
got := fb.Chat()
if got == prev {
t.Fatalf("chat repeated %q on turn %d", got, i)
}
prev = got
}
}
+6 -6
View File
@@ -449,14 +449,14 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
sys, prompt := p.knowledgePrompt(utterance)
resp, err := p.chatWithSystem(ctx, sys, prompt, 768)
if err != nil {
return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", err)
return UnknownFallback(), fmt.Errorf("phrase query (knowledge): %w", err)
}
if resp == "" {
return "не знаю.", errEmptyResponse
return UnknownFallback(), errEmptyResponse
}
text, _, perr := parseResponseMood(resp)
if perr != nil {
return "не знаю.", fmt.Errorf("phrase query (knowledge): %w", perr)
return UnknownFallback(), fmt.Errorf("phrase query (knowledge): %w", perr)
}
if text != "" {
return text, nil
@@ -472,7 +472,7 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
if cause == nil {
cause = perr
}
return "вот что я нашла: " + strings.Join(notes, "; "),
return SourcesFallback(strings.Join(notes, "; ")),
fmt.Errorf("phrase query (evidence): %w", cause)
}
if text != "" {
@@ -501,11 +501,11 @@ func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history [
resp, err := p.chatWithMessages(ctx, msgs, 768)
if err != nil {
return ChatFallback, fmt.Errorf("phrase chat: %w", err)
return ChatFallback(), fmt.Errorf("phrase chat: %w", err)
}
text, _, perr := parseResponseMood(resp)
if perr != nil {
return ChatFallback, fmt.Errorf("phrase chat: %w", perr)
return ChatFallback(), fmt.Errorf("phrase chat: %w", perr)
}
if text != "" {
return text, nil
+3 -12
View File
@@ -66,28 +66,19 @@ type Stub struct{}
// NewStub builds the floor phraser. no config — the Stub is stateless.
func NewStub() *Stub { return &Stub{} }
// ChatFallback — what she says on the chat path when the model gave her
// nothing to say. It replaced "поговорили.", which reads as a summary of a
// conversation that did not happen. Said out loud this one is an admission,
// which is what it is.
const ChatFallback = "даже не знаю, что сказать."
// PhraseChat returns a stub reply — the LLMPhraser replaces this with a
// prompted response from the model. The history parameter is accepted but
// ignored at the stub level (the production impl uses it for multi-turn).
func (s *Stub) PhraseChat(_ context.Context, _ string, _ []dialogue.Turn) (string, error) {
return ChatFallback, nil
return ChatFallback(), nil
}
// PhraseQuery returns a deterministic summary of the best matching notes.
func (s *Stub) PhraseQuery(_ context.Context, _ string, notes []string) (string, error) {
if len(notes) == 0 {
return "не знаю.", nil
return UnknownFallback(), nil
}
if len(notes) == 1 {
return "вот что я нашла: " + notes[0], nil
}
return "вот что я нашла: " + strings.Join(notes, "; "), nil
return SourcesFallback(strings.Join(notes, "; ")), nil
}
// Close implements Phraser.Close (no-op for the stub).