package phraser import ( "context" "encoding/json" "net/http" "net/http/httptest" "strings" "testing" ) // promptSpy records the system and user strings of every request, which is // where the evidence-first discipline either exists or does not. type promptSpy struct { srv *httptest.Server system []string user []string } func newPromptSpy(t *testing.T) *promptSpy { t.Helper() s := &promptSpy{} s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req chatReq if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Errorf("spy: decode request: %v", err) } for _, m := range req.Messages { switch m.Role { case "system": s.system = append(s.system, m.Content) case "user": s.user = append(s.user, m.Content) } } w.Header().Set("Content-Type", "application/json") w.Write([]byte(`{"choices":[{"message":{"content":"{\"response\": \"вот что я нашла: два литра\", \"mood\": \"neutral\"}"}}]}`)) })) t.Cleanup(s.srv.Close) return s } func TestEvidenceReachesTheModelAsNumberedSources(t *testing.T) { spy := newPromptSpy(t) p := NewLLMPhraserAt(spy.srv.URL, Config{}) if _, err := p.PhraseQuery(context.Background(), "сколько воды я выпил", []string{"выпил два литра", "бутылка на 0.7"}); err != nil { t.Fatal(err) } if len(spy.user) != 1 { t.Fatalf("got %d user messages, want 1", len(spy.user)) } for _, want := range []string{"[1] выпил два литра", "[2] бутылка на 0.7", "Источники:"} { if !strings.Contains(spy.user[0], want) { t.Errorf("user prompt is missing %q:\n%s", want, spy.user[0]) } } } // The system prompt is the whole fix for the Левитан fabrication: answer from // the sources, say so when they do not answer, add nothing from memory. func TestEvidencePromptForbidsAnsweringFromMemory(t *testing.T) { spy := newPromptSpy(t) p := NewLLMPhraserAt(spy.srv.URL, Config{}) if _, err := p.PhraseQuery(context.Background(), "кто написал войну и мир", []string{"Лев Толстой, роман 1869 года"}); err != nil { t.Fatal(err) } sys := spy.system[0] for _, want := range []string{"ТОЛЬКО по ним", "Если ответа в них нет"} { if !strings.Contains(sys, want) { t.Errorf("system prompt is missing %q:\n%s", want, sys) } } if strings.Contains(sys, "заметк") { t.Errorf("system prompt still calls every source a note:\n%s", sys) } } // A source that trimmed away is not a source. Handing the evidence branch an // empty list is the one prompt that reliably makes a small model invent. func TestBlankSourcesTakeTheKnowledgeBranch(t *testing.T) { spy := newPromptSpy(t) p := NewLLMPhraserAt(spy.srv.URL, Config{}) if _, err := p.PhraseQuery(context.Background(), "что я записывал", []string{"", " "}); err != nil { t.Fatal(err) } if strings.Contains(spy.user[0], "Источники:") { t.Errorf("blank sources still took the evidence branch:\n%s", spy.user[0]) } } func TestNonEmptyDoesNotMutateTheCallersSlice(t *testing.T) { in := []string{" один ", "", "два"} got := nonEmpty(in) if want := []string{"один", "два"}; len(got) != 2 || got[0] != want[0] || got[1] != want[1] { t.Errorf("nonEmpty = %q, want %q", got, want) } if in[0] != " один " { t.Errorf("caller's slice was mutated: %q", in) } }