claude
d1f8a734c5
PhraseChat and PhraseQuery returned canned text with a nil error, so a dead or OOM-killed server was indistinguishable from bad phrasing — "не знаю." is also a legitimate answer. Both now return the fallback text AND the error. The daemon keeps using the text, so the turn still survives; a measuring caller counts a real failure. An empty response is its own error: the model is up and said nothing. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
73 lines
2.6 KiB
Go
73 lines
2.6 KiB
Go
package phraser
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// A dead server must be distinguishable from bad phrasing. Both PhraseChat and
|
|
// PhraseQuery keep the turn alive with canned text — "поговорили.", "не знаю.",
|
|
// "вот что я нашла: …" — and every one of those is also a legitimate reply, so
|
|
// the text alone cannot say which happened. The error is the only signal, and
|
|
// before Vikunja #397 it was dropped: the talk scorer reported a full run with
|
|
// zero errors off a server that answered nothing.
|
|
func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "model not loaded", http.StatusServiceUnavailable)
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
p := NewLLMPhraserAt(srv.URL, Config{})
|
|
|
|
cases := []struct {
|
|
name string
|
|
call func() (string, error)
|
|
want string
|
|
}{
|
|
{"chat", func() (string, error) {
|
|
return p.PhraseChat(context.Background(), "как дела", nil)
|
|
}, "поговорили."},
|
|
{"knowledge", func() (string, error) {
|
|
return p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
|
|
}, "не знаю."},
|
|
{"evidence", func() (string, error) {
|
|
return p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"два литра"})
|
|
}, "вот что я нашла: два литра"},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
got, err := c.call()
|
|
if err == nil {
|
|
t.Fatalf("no error from a dead server; the scorer would count this as bad phrasing")
|
|
}
|
|
if got != c.want {
|
|
t.Errorf("fallback text = %q, want %q — the daemon still has to say something", got, c.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// An empty answer is a failure too: the server is up and produced no tokens,
|
|
// which is not an answer and must not score as one.
|
|
func TestEmptyKnowledgeAnswerIsAnError(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(`{"choices":[{"message":{"content":""}}]}`))
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
p := NewLLMPhraserAt(srv.URL, Config{})
|
|
|
|
got, err := p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
|
|
if err == nil {
|
|
t.Fatal("an empty response scored as an answer")
|
|
}
|
|
if got != "не знаю." {
|
|
t.Errorf("fallback text = %q, want \"не знаю.\"", got)
|
|
}
|
|
if !strings.Contains(err.Error(), "empty") {
|
|
t.Errorf("error = %v; want it to name the empty response", err)
|
|
}
|
|
}
|