81 lines
3.0 KiB
Go
81 lines
3.0 KiB
Go
package phraser
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// isFallback — the text she says is picked from that entry's variants, so a test
|
|
// pins the entry rather than the wording. Pinning one line would make editing
|
|
// fallbacks_ru_v1.json break Go tests, which is the coupling this file removed.
|
|
func isFallback(t *testing.T, key, sources, got string) bool {
|
|
t.Helper()
|
|
return DefaultFallbacks().deck().matches(key, map[string]string{"sources": sources}, got)
|
|
}
|
|
|
|
// A dead server must be distinguishable from bad phrasing. Both PhraseChat and
|
|
// PhraseQuery keep the turn alive with canned text — and every one of those
|
|
// lines is also a legitimate reply, so the text alone cannot say which happened.
|
|
// The error is the only signal, and before Vikunja #397 it was dropped: the talk
|
|
// scorer reported a full run with zero errors off a server that answered nothing.
|
|
func TestPhrasingReportsTheFailureWithTheFallback(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "model not loaded", http.StatusServiceUnavailable)
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
p := NewLLMPhraserAt(srv.URL, Config{})
|
|
|
|
cases := []struct {
|
|
name string
|
|
call func() (string, error)
|
|
key string
|
|
sources string
|
|
}{
|
|
{"chat", func() (string, error) {
|
|
return p.PhraseChat(context.Background(), "как дела", nil)
|
|
}, fbChat, ""},
|
|
{"knowledge", func() (string, error) {
|
|
return p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
|
|
}, fbQueryUnknown, ""},
|
|
{"evidence", func() (string, error) {
|
|
return p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"два литра"})
|
|
}, fbQuerySources, "два литра"},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
got, err := c.call()
|
|
if err == nil {
|
|
t.Fatalf("no error from a dead server; the scorer would count this as bad phrasing")
|
|
}
|
|
if !isFallback(t, c.key, c.sources, got) {
|
|
t.Errorf("fallback text = %q, want a %q variant — the daemon still has to say something", got, c.key)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// An empty answer is a failure too: the server is up and produced no tokens,
|
|
// which is not an answer and must not score as one.
|
|
func TestEmptyKnowledgeAnswerIsAnError(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(`{"choices":[{"message":{"content":""}}]}`))
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
p := NewLLMPhraserAt(srv.URL, Config{})
|
|
|
|
got, err := p.PhraseQuery(context.Background(), "кто написал войну и мир", nil)
|
|
if err == nil {
|
|
t.Fatal("an empty response scored as an answer")
|
|
}
|
|
if !isFallback(t, fbQueryUnknown, "", got) {
|
|
t.Errorf("fallback text = %q, want a %q variant", got, fbQueryUnknown)
|
|
}
|
|
if !strings.Contains(err.Error(), "empty") {
|
|
t.Errorf("error = %v; want it to name the empty response", err)
|
|
}
|
|
}
|