diff --git a/internal/phraser/evidence_test.go b/internal/phraser/evidence_test.go new file mode 100644 index 0000000..7ebf0ce --- /dev/null +++ b/internal/phraser/evidence_test.go @@ -0,0 +1,102 @@ +package phraser + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// promptSpy records the system and user strings of every request, which is +// where the evidence-first discipline either exists or does not. +type promptSpy struct { + srv *httptest.Server + system []string + user []string +} + +func newPromptSpy(t *testing.T) *promptSpy { + t.Helper() + s := &promptSpy{} + s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req chatReq + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("spy: decode request: %v", err) + } + for _, m := range req.Messages { + switch m.Role { + case "system": + s.system = append(s.system, m.Content) + case "user": + s.user = append(s.user, m.Content) + } + } + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"choices":[{"message":{"content":"{\"response\": \"вот что я нашла: два литра\", \"mood\": \"neutral\"}"}}]}`)) + })) + t.Cleanup(s.srv.Close) + return s +} + +func TestEvidenceReachesTheModelAsNumberedSources(t *testing.T) { + spy := newPromptSpy(t) + p := NewLLMPhraserAt(spy.srv.URL, Config{}) + if _, err := p.PhraseQuery(context.Background(), "сколько воды я выпил", + []string{"выпил два литра", "бутылка на 0.7"}); err != nil { + t.Fatal(err) + } + if len(spy.user) != 1 { + t.Fatalf("got %d user messages, want 1", len(spy.user)) + } + for _, want := range []string{"[1] выпил два литра", "[2] бутылка на 0.7", "Источники:"} { + if !strings.Contains(spy.user[0], want) { + t.Errorf("user prompt is missing %q:\n%s", want, spy.user[0]) + } + } +} + +// The system prompt is the whole fix for the Левитан fabrication: answer from +// the sources, say so when they do not answer, add nothing from memory. +func TestEvidencePromptForbidsAnsweringFromMemory(t *testing.T) { + spy := newPromptSpy(t) + p := NewLLMPhraserAt(spy.srv.URL, Config{}) + if _, err := p.PhraseQuery(context.Background(), "кто написал войну и мир", + []string{"Лев Толстой, роман 1869 года"}); err != nil { + t.Fatal(err) + } + sys := spy.system[0] + for _, want := range []string{"ТОЛЬКО по ним", "Если ответа в них нет"} { + if !strings.Contains(sys, want) { + t.Errorf("system prompt is missing %q:\n%s", want, sys) + } + } + if strings.Contains(sys, "заметк") { + t.Errorf("system prompt still calls every source a note:\n%s", sys) + } +} + +// A source that trimmed away is not a source. Handing the evidence branch an +// empty list is the one prompt that reliably makes a small model invent. +func TestBlankSourcesTakeTheKnowledgeBranch(t *testing.T) { + spy := newPromptSpy(t) + p := NewLLMPhraserAt(spy.srv.URL, Config{}) + if _, err := p.PhraseQuery(context.Background(), "что я записывал", []string{"", " "}); err != nil { + t.Fatal(err) + } + if strings.Contains(spy.user[0], "Источники:") { + t.Errorf("blank sources still took the evidence branch:\n%s", spy.user[0]) + } +} + +func TestNonEmptyDoesNotMutateTheCallersSlice(t *testing.T) { + in := []string{" один ", "", "два"} + got := nonEmpty(in) + if want := []string{"один", "два"}; len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Errorf("nonEmpty = %q, want %q", got, want) + } + if in[0] != " один " { + t.Errorf("caller's slice was mutated: %q", in) + } +} diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index 51697d3..fd34770 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -357,6 +357,11 @@ func (p *LLMPhraser) PhraseNudge(ctx context.Context, c loop.Candidate) (deliver // compose a natural answer. Falls back to "вот что я нашла: " on any // LLM error — better to give the raw data than silence. func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) { + // Blank sources are no sources. A caller that hands over one empty string — + // a page that fetched to nothing, a snippet trimmed away — used to take the + // evidence branch and be told to answer from an empty list, which is the one + // prompt guaranteed to make a small model fill the gap from memory. + notes = nonEmpty(notes) if len(notes) == 0 { // General knowledge — no notes to ground the answer. The system // prompt is the single tested source in router.KnowledgePrompt. @@ -376,13 +381,10 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes [] } return resp, nil } - if len(notes) == 1 { - notes[0] = strings.TrimSpace(notes[0]) - } sys := p.querySystemPrompt() prompt := fmt.Sprintf( - `Он спрашивает: "%s". В твоих заметках по этому вопросу написано: "%s". Ответь ему коротко и своими словами. Если в заметках ответа нет — так и скажи.`, - utterance, strings.Join(notes, `"; "`), + "Он спрашивает: \"%s\"\n\nИсточники:\n%s\nОтветь ему коротко и своими словами, опираясь только на эти источники. Если ответа в них нет — так и скажи.", + utterance, evidenceBlock(notes), ) resp, err := p.chatWithSystem(ctx, sys, prompt, 768) text, _, perr := parseResponseMood(resp) @@ -725,21 +727,64 @@ func (p *LLMPhraser) systemPrompt() string { return persona.Prepend(p.cfg.ContextBlock, nudgeSystem) } -// querySystemPrompt returns the system prompt for PhraseQuery (notes + general -// knowledge). Prepends the configured persona when set. +// querySystemPrompt returns the system prompt for the evidence branch of +// PhraseQuery. Prepends the configured persona when set. +// +// Evidence-first, and that is the whole point of this prompt. Every source that +// reaches PhraseQuery with something in hand — his notes, a stored fact, a page, +// a live search, a ZIM article — arrives as numbered sources, and the model's +// job here is to READ them, not to recall. A 1.7B asked a world question +// answers from its weights with total confidence and no signal that it is +// guessing; that is how "Война и мир" got Левитан as its author. The rule that +// prevents it is stated three ways, because one way did not hold: answer from +// the sources, say plainly when they do not answer, add nothing of your own. +// +// It no longer says "заметки". The sources are not always his notes, and +// calling a Wikipedia paragraph his note both misleads him and licenses the +// model to blur where an answer came from. +// +// No self-introduction here: the persona block prepended one line above already +// says who she is, same as router.KnowledgePrompt. +// +// The opener is deliberate and stays: the fixed prefix is what marks the answer +// as a lookup rather than as something she knows. The grammar examples are not +// deliberate — same defect chatSystemPrompt had, where a 1.7B copies a quoted +// word instead of generalising from it. Stated as morphology instead. func (p *LLMPhraser) querySystemPrompt() string { - // No self-introduction here: the persona block prepended one line above - // already says who she is, same as router.KnowledgePrompt. - // - // The opener is deliberate and stays: this path answers from his own notes - // and the fixed prefix is what marks the answer as a lookup rather than as - // something she knows. The grammar examples are not deliberate — same - // defect chatSystemPrompt had, where a 1.7B copies a quoted word instead of - // generalising from it. Stated as morphology instead. - base := "Ты отвечаешь ему по своим заметкам. Отвечай по-русски, коротко и своими словами, начинай с \"вот что я нашла: \". О себе — в женском роде, глаголы в прошедшем времени с окончанием -ла. Он мужчина, обращайся к нему на \"ты\". Respond ONLY with valid JSON: {\"response\": \"...\", \"mood\": \"neutral\"}." + base := "Ты отвечаешь ему по источникам, которые тебе дали. Отвечай ТОЛЬКО по ним: всё, что ты говоришь, должно быть написано в источниках. " + + "Если ответа в них нет — так и скажи и на этом остановись; не добавляй ничего из своих знаний и не догадывайся. " + + "Не приплетай прошлые реплики разговора. " + + "Отвечай по-русски, коротко и своими словами, начинай с \"вот что я нашла: \". О себе — в женском роде, глаголы в прошедшем времени с окончанием -ла. Он мужчина, обращайся к нему на \"ты\". Respond ONLY with valid JSON: {\"response\": \"...\", \"mood\": \"neutral\"}." return persona.Prepend(p.cfg.ContextBlock, base) } +// evidenceBlock renders the sources for the evidence branch of PhraseQuery. +// +// Numbered lines, one source each, rather than the quoted semicolon-joined +// string this used to build. Two reasons, both measured on small models: a +// numbered list survives being long, where a run-on quoted string blurs into +// one claim the model then merges; and the numbering gives it something to +// answer FROM, which is what makes "этого в источниках нет" reachable at all. +func evidenceBlock(sources []string) string { + var b strings.Builder + for i, s := range sources { + fmt.Fprintf(&b, "[%d] %s\n", i+1, s) + } + return b.String() +} + +// nonEmpty drops blank sources and trims the rest, without touching the +// caller's slice. +func nonEmpty(sources []string) []string { + out := make([]string, 0, len(sources)) + for _, s := range sources { + if s = strings.TrimSpace(s); s != "" { + out = append(out, s) + } + } + return out +} + // ruleTopics — Russian gloss for each built-in rule name. The rule names are // English identifiers; a 0.8B asked to nudge about "netdata_critical" writes // about nothing. The daemon knows what its own rules mean, so it says so.