kami
63a389a1f8
The evidence branch of PhraseQuery framed every source as "твои заметки" and joined them into one quoted run-on. A live search snippet is not his note, and a run-on gives a 1.7B one blurred claim to merge rather than sources to answer from. That is the shape that named Левитан as the author of Война и мир. Sources now arrive numbered, one per line, and the system prompt says three ways that the answer comes out of them: only from the sources, say plainly when they do not answer, add nothing of your own. Blank sources take the knowledge branch. One empty string used to reach the evidence branch and ask the model to answer from an empty list, which is the one prompt guaranteed to make it fill the gap from memory. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
103 lines
3.4 KiB
Go
103 lines
3.4 KiB
Go
package phraser
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// promptSpy records the system and user strings of every request, which is
|
|
// where the evidence-first discipline either exists or does not.
|
|
type promptSpy struct {
|
|
srv *httptest.Server
|
|
system []string
|
|
user []string
|
|
}
|
|
|
|
func newPromptSpy(t *testing.T) *promptSpy {
|
|
t.Helper()
|
|
s := &promptSpy{}
|
|
s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var req chatReq
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
t.Errorf("spy: decode request: %v", err)
|
|
}
|
|
for _, m := range req.Messages {
|
|
switch m.Role {
|
|
case "system":
|
|
s.system = append(s.system, m.Content)
|
|
case "user":
|
|
s.user = append(s.user, m.Content)
|
|
}
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(`{"choices":[{"message":{"content":"{\"response\": \"вот что я нашла: два литра\", \"mood\": \"neutral\"}"}}]}`))
|
|
}))
|
|
t.Cleanup(s.srv.Close)
|
|
return s
|
|
}
|
|
|
|
func TestEvidenceReachesTheModelAsNumberedSources(t *testing.T) {
|
|
spy := newPromptSpy(t)
|
|
p := NewLLMPhraserAt(spy.srv.URL, Config{})
|
|
if _, err := p.PhraseQuery(context.Background(), "сколько воды я выпил",
|
|
[]string{"выпил два литра", "бутылка на 0.7"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(spy.user) != 1 {
|
|
t.Fatalf("got %d user messages, want 1", len(spy.user))
|
|
}
|
|
for _, want := range []string{"[1] выпил два литра", "[2] бутылка на 0.7", "Источники:"} {
|
|
if !strings.Contains(spy.user[0], want) {
|
|
t.Errorf("user prompt is missing %q:\n%s", want, spy.user[0])
|
|
}
|
|
}
|
|
}
|
|
|
|
// The system prompt is the whole fix for the Левитан fabrication: answer from
|
|
// the sources, say so when they do not answer, add nothing from memory.
|
|
func TestEvidencePromptForbidsAnsweringFromMemory(t *testing.T) {
|
|
spy := newPromptSpy(t)
|
|
p := NewLLMPhraserAt(spy.srv.URL, Config{})
|
|
if _, err := p.PhraseQuery(context.Background(), "кто написал войну и мир",
|
|
[]string{"Лев Толстой, роман 1869 года"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sys := spy.system[0]
|
|
for _, want := range []string{"ТОЛЬКО по ним", "Если ответа в них нет"} {
|
|
if !strings.Contains(sys, want) {
|
|
t.Errorf("system prompt is missing %q:\n%s", want, sys)
|
|
}
|
|
}
|
|
if strings.Contains(sys, "заметк") {
|
|
t.Errorf("system prompt still calls every source a note:\n%s", sys)
|
|
}
|
|
}
|
|
|
|
// A source that trimmed away is not a source. Handing the evidence branch an
|
|
// empty list is the one prompt that reliably makes a small model invent.
|
|
func TestBlankSourcesTakeTheKnowledgeBranch(t *testing.T) {
|
|
spy := newPromptSpy(t)
|
|
p := NewLLMPhraserAt(spy.srv.URL, Config{})
|
|
if _, err := p.PhraseQuery(context.Background(), "что я записывал", []string{"", " "}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(spy.user[0], "Источники:") {
|
|
t.Errorf("blank sources still took the evidence branch:\n%s", spy.user[0])
|
|
}
|
|
}
|
|
|
|
func TestNonEmptyDoesNotMutateTheCallersSlice(t *testing.T) {
|
|
in := []string{" один ", "", "два"}
|
|
got := nonEmpty(in)
|
|
if want := []string{"один", "два"}; len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
|
|
t.Errorf("nonEmpty = %q, want %q", got, want)
|
|
}
|
|
if in[0] != " один " {
|
|
t.Errorf("caller's slice was mutated: %q", in)
|
|
}
|
|
}
|