phraser, mavend: read the fallbacks from the file (V-501)
The accessors are functions now, so the call sites that compared against one literal compare against the entry instead: IsUnknownFallback and IsSourcesFallback in the daemon tests, the entry key in the phraser tests. A reworded variant no longer breaks a Go test. The eval scores every variant on the persona checks the nudges already pass.
This commit is contained in:
@@ -8,12 +8,28 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// isFallback — the text she says is picked from that entry's variants, so a test
|
||||
// pins the entry rather than the wording. Pinning one line would make editing
|
||||
// fallbacks_ru_v1.json break Go tests, which is the coupling this file removed.
|
||||
func isFallback(t *testing.T, key, sources, got string) bool {
|
||||
t.Helper()
|
||||
e, ok := DefaultFallbacks().file.Entries[key]
|
||||
if !ok {
|
||||
t.Fatalf("no fallback entry %q", key)
|
||||
}
|
||||
for _, v := range e.Variants {
|
||||
if strings.ReplaceAll(v, "{sources}", sources) == got {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A dead server must be distinguishable from bad phrasing. Both PhraseChat and
|
||||
// PhraseQuery keep the turn alive with canned text — ChatFallback, "не знаю.",
|
||||
// "вот что я нашла: …" — and every one of those is also a legitimate reply, so
|
||||
// the text alone cannot say which happened. The error is the only signal, and
|
||||
// before Vikunja #397 it was dropped: the talk scorer reported a full run with
|
||||
// zero errors off a server that answered nothing.
|
||||
// PhraseQuery keep the turn alive with canned text — and every one of those
|
||||
// lines is also a legitimate reply, so the text alone cannot say which happened.
|
||||
// The error is the only signal, and before Vikunja #397 it was dropped: the talk
|
||||
// scorer reported a full run with zero errors off a server that answered nothing.
|
||||
func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "model not loaded", http.StatusServiceUnavailable)
|
||||
@@ -22,19 +38,20 @@ func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
|
||||
p := NewLLMPhraserAt(srv.URL, Config{})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
call func() (string, error)
|
||||
want string
|
||||
name string
|
||||
call func() (string, error)
|
||||
key string
|
||||
sources string
|
||||
}{
|
||||
{"chat", func() (string, error) {
|
||||
return p.PhraseChat(context.Background(), "как дела", nil)
|
||||
}, ChatFallback},
|
||||
}, fbChat, ""},
|
||||
{"knowledge", func() (string, error) {
|
||||
return p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
|
||||
}, "не знаю."},
|
||||
}, fbQueryUnknown, ""},
|
||||
{"evidence", func() (string, error) {
|
||||
return p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"два литра"})
|
||||
}, "вот что я нашла: два литра"},
|
||||
}, fbQuerySources, "два литра"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
@@ -42,8 +59,8 @@ func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatalf("no error from a dead server; the scorer would count this as bad phrasing")
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("fallback text = %q, want %q — the daemon still has to say something", got, c.want)
|
||||
if !isFallback(t, c.key, c.sources, got) {
|
||||
t.Errorf("fallback text = %q, want a %q variant — the daemon still has to say something", got, c.key)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -63,8 +80,8 @@ func TestEmptyKnowledgeAnswerIsAnError(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("an empty response scored as an answer")
|
||||
}
|
||||
if got != "не знаю." {
|
||||
t.Errorf("fallback text = %q, want \"не знаю.\"", got)
|
||||
if !isFallback(t, fbQueryUnknown, "", got) {
|
||||
t.Errorf("fallback text = %q, want a %q variant", got, fbQueryUnknown)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "empty") {
|
||||
t.Errorf("error = %v; want it to name the empty response", err)
|
||||
|
||||
Reference in New Issue
Block a user