phraser: make her read the sources instead of recalling them

The evidence branch of PhraseQuery framed every source as "твои заметки" and joined them into one quoted run-on. A live search snippet is not his note, and a run-on gives a 1.7B one blurred claim to merge rather than sources to answer from. That is the shape that named Левитан as the author of Война и мир. Sources now arrive numbered, one per line, and the system prompt says three ways that the answer comes out of them: only from the sources, say plainly when they do not answer, add nothing of your own. Blank sources take the knowledge branch. One empty string used to reach the evidence branch and ask the model to answer from an empty list, which is the one prompt guaranteed to make it fill the gap from memory. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-02 02:33:35 +04:00
parent 2150a18e98
commit 63a389a1f8
2 changed files with 163 additions and 16 deletions
@@ -0,0 +1,102 @@
+package phraser
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// promptSpy records the system and user strings of every request, which is
+// where the evidence-first discipline either exists or does not.
+type promptSpy struct {
+	srv    *httptest.Server
+	system []string
+	user   []string
+}
+
+func newPromptSpy(t *testing.T) *promptSpy {
+	t.Helper()
+	s := &promptSpy{}
+	s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var req chatReq
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			t.Errorf("spy: decode request: %v", err)
+		}
+		for _, m := range req.Messages {
+			switch m.Role {
+			case "system":
+				s.system = append(s.system, m.Content)
+			case "user":
+				s.user = append(s.user, m.Content)
+			}
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"choices":[{"message":{"content":"{\"response\": \"вот что я нашла: два литра\", \"mood\": \"neutral\"}"}}]}`))
+	}))
+	t.Cleanup(s.srv.Close)
+	return s
+}
+
+func TestEvidenceReachesTheModelAsNumberedSources(t *testing.T) {
+	spy := newPromptSpy(t)
+	p := NewLLMPhraserAt(spy.srv.URL, Config{})
+	if _, err := p.PhraseQuery(context.Background(), "сколько воды я выпил",
+		[]string{"выпил два литра", "бутылка на 0.7"}); err != nil {
+		t.Fatal(err)
+	}
+	if len(spy.user) != 1 {
+		t.Fatalf("got %d user messages, want 1", len(spy.user))
+	}
+	for _, want := range []string{"[1] выпил два литра", "[2] бутылка на 0.7", "Источники:"} {
+		if !strings.Contains(spy.user[0], want) {
+			t.Errorf("user prompt is missing %q:\n%s", want, spy.user[0])
+		}
+	}
+}
+
+// The system prompt is the whole fix for the Левитан fabrication: answer from
+// the sources, say so when they do not answer, add nothing from memory.
+func TestEvidencePromptForbidsAnsweringFromMemory(t *testing.T) {
+	spy := newPromptSpy(t)
+	p := NewLLMPhraserAt(spy.srv.URL, Config{})
+	if _, err := p.PhraseQuery(context.Background(), "кто написал войну и мир",
+		[]string{"Лев Толстой, роман 1869 года"}); err != nil {
+		t.Fatal(err)
+	}
+	sys := spy.system[0]
+	for _, want := range []string{"ТОЛЬКО по ним", "Если ответа в них нет"} {
+		if !strings.Contains(sys, want) {
+			t.Errorf("system prompt is missing %q:\n%s", want, sys)
+		}
+	}
+	if strings.Contains(sys, "заметк") {
+		t.Errorf("system prompt still calls every source a note:\n%s", sys)
+	}
+}
+
+// A source that trimmed away is not a source. Handing the evidence branch an
+// empty list is the one prompt that reliably makes a small model invent.
+func TestBlankSourcesTakeTheKnowledgeBranch(t *testing.T) {
+	spy := newPromptSpy(t)
+	p := NewLLMPhraserAt(spy.srv.URL, Config{})
+	if _, err := p.PhraseQuery(context.Background(), "что я записывал", []string{"", "   "}); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(spy.user[0], "Источники:") {
+		t.Errorf("blank sources still took the evidence branch:\n%s", spy.user[0])
+	}
+}
+
+func TestNonEmptyDoesNotMutateTheCallersSlice(t *testing.T) {
+	in := []string{"  один  ", "", "два"}
+	got := nonEmpty(in)
+	if want := []string{"один", "два"}; len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
+		t.Errorf("nonEmpty = %q, want %q", got, want)
+	}
+	if in[0] != "  один  " {
+		t.Errorf("caller's slice was mutated: %q", in)
+	}
+}
@@ -357,6 +357,11 @@ func (p *LLMPhraser) PhraseNudge(ctx context.Context, c loop.Candidate) (deliver
 // compose a natural answer. Falls back to "вот что я нашла: <notes>" on any
 // LLM error — better to give the raw data than silence.
 func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) {
+	// Blank sources are no sources. A caller that hands over one empty string —
+	// a page that fetched to nothing, a snippet trimmed away — used to take the
+	// evidence branch and be told to answer from an empty list, which is the one
+	// prompt guaranteed to make a small model fill the gap from memory.
+	notes = nonEmpty(notes)
 	if len(notes) == 0 {
 		// General knowledge — no notes to ground the answer. The system
 		// prompt is the single tested source in router.KnowledgePrompt.
@@ -376,13 +381,10 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []
 		}
 		return resp, nil
 	}
-	if len(notes) == 1 {
-		notes[0] = strings.TrimSpace(notes[0])
-	}
 	sys := p.querySystemPrompt()
 	prompt := fmt.Sprintf(
-		`Он спрашивает: "%s". В твоих заметках по этому вопросу написано: "%s". Ответь ему коротко и своими словами. Если в заметках ответа нет — так и скажи.`,
-		utterance, strings.Join(notes, `"; "`),
+		"Он спрашивает: \"%s\"\n\nИсточники:\n%s\nОтветь ему коротко и своими словами, опираясь только на эти источники. Если ответа в них нет — так и скажи.",
+		utterance, evidenceBlock(notes),
 	)
 	resp, err := p.chatWithSystem(ctx, sys, prompt, 768)
 	text, _, perr := parseResponseMood(resp)
@@ -725,21 +727,64 @@ func (p *LLMPhraser) systemPrompt() string {
 	return persona.Prepend(p.cfg.ContextBlock, nudgeSystem)
 }

-// querySystemPrompt returns the system prompt for PhraseQuery (notes + general
-// knowledge). Prepends the configured persona when set.
+// querySystemPrompt returns the system prompt for the evidence branch of
+// PhraseQuery. Prepends the configured persona when set.
+//
+// Evidence-first, and that is the whole point of this prompt. Every source that
+// reaches PhraseQuery with something in hand — his notes, a stored fact, a page,
+// a live search, a ZIM article — arrives as numbered sources, and the model's
+// job here is to READ them, not to recall. A 1.7B asked a world question
+// answers from its weights with total confidence and no signal that it is
+// guessing; that is how "Война и мир" got Левитан as its author. The rule that
+// prevents it is stated three ways, because one way did not hold: answer from
+// the sources, say plainly when they do not answer, add nothing of your own.
+//
+// It no longer says "заметки". The sources are not always his notes, and
+// calling a Wikipedia paragraph his note both misleads him and licenses the
+// model to blur where an answer came from.
+//
+// No self-introduction here: the persona block prepended one line above already
+// says who she is, same as router.KnowledgePrompt.
+//
+// The opener is deliberate and stays: the fixed prefix is what marks the answer
+// as a lookup rather than as something she knows. The grammar examples are not
+// deliberate — same defect chatSystemPrompt had, where a 1.7B copies a quoted
+// word instead of generalising from it. Stated as morphology instead.
 func (p *LLMPhraser) querySystemPrompt() string {
-	// No self-introduction here: the persona block prepended one line above
-	// already says who she is, same as router.KnowledgePrompt.
-	//
-	// The opener is deliberate and stays: this path answers from his own notes
-	// and the fixed prefix is what marks the answer as a lookup rather than as
-	// something she knows. The grammar examples are not deliberate — same
-	// defect chatSystemPrompt had, where a 1.7B copies a quoted word instead of
-	// generalising from it. Stated as morphology instead.
-	base := "Ты отвечаешь ему по своим заметкам. Отвечай по-русски, коротко и своими словами, начинай с \"вот что я нашла: \". О себе — в женском роде, глаголы в прошедшем времени с окончанием -ла. Он мужчина, обращайся к нему на \"ты\". Respond ONLY with valid JSON: {\"response\": \"...\", \"mood\": \"neutral\"}."
+	base := "Ты отвечаешь ему по источникам, которые тебе дали. Отвечай ТОЛЬКО по ним: всё, что ты говоришь, должно быть написано в источниках. " +
+		"Если ответа в них нет — так и скажи и на этом остановись; не добавляй ничего из своих знаний и не догадывайся. " +
+		"Не приплетай прошлые реплики разговора. " +
+		"Отвечай по-русски, коротко и своими словами, начинай с \"вот что я нашла: \". О себе — в женском роде, глаголы в прошедшем времени с окончанием -ла. Он мужчина, обращайся к нему на \"ты\". Respond ONLY with valid JSON: {\"response\": \"...\", \"mood\": \"neutral\"}."
 	return persona.Prepend(p.cfg.ContextBlock, base)
 }

+// evidenceBlock renders the sources for the evidence branch of PhraseQuery.
+//
+// Numbered lines, one source each, rather than the quoted semicolon-joined
+// string this used to build. Two reasons, both measured on small models: a
+// numbered list survives being long, where a run-on quoted string blurs into
+// one claim the model then merges; and the numbering gives it something to
+// answer FROM, which is what makes "этого в источниках нет" reachable at all.
+func evidenceBlock(sources []string) string {
+	var b strings.Builder
+	for i, s := range sources {
+		fmt.Fprintf(&b, "[%d] %s\n", i+1, s)
+	}
+	return b.String()
+}
+
+// nonEmpty drops blank sources and trims the rest, without touching the
+// caller's slice.
+func nonEmpty(sources []string) []string {
+	out := make([]string, 0, len(sources))
+	for _, s := range sources {
+		if s = strings.TrimSpace(s); s != "" {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
 // ruleTopics — Russian gloss for each built-in rule name. The rule names are
 // English identifiers; a 0.8B asked to nudge about "netdata_critical" writes
 // about nothing. The daemon knows what its own rules mean, so it says so.