memeval: scope the evaluator's note windows by source
Both windows the evaluator keeps over the notes table were row budgets over every writer. The dedupe read 200 recent notes and kept the eval ones, so after 200 ordinary notes an old observation left the window and the next evaluation wrote the same sentence again. The snapshot asked for MaxItems notes and then discarded her own, so once hourly evaluation had run for a few weeks the model saw almost no real notes. Both reads are now filtered in SQL, by RecentNotesBySource and RecentNotesExcludingSource. Two smaller things in the same area. The dedupe key stripped any trailing bracketed clause, so an observation ending in one hashed differently from its stored form; it now strips only the recorded action. The evaluation timeout was five minutes on the one llama-server that also answers voice turns, which made a collision a five-minute mute assistant, and is now sixty seconds. Found in review of #55.
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -269,3 +270,95 @@ func TestParseObservationsTolerantAndBounded(t *testing.T) {
|
||||
t.Fatalf("parsed %d observations, want the %d cap", len(obs), MaxObservations)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDedupeSurvivesOrdinaryNotes — the dedupe window is a count of HER notes,
|
||||
// not of all notes. With a plain recent-notes read, DedupeWindow ordinary notes
|
||||
// written after an observation pushed it out of sight and the same sentence was
|
||||
// written again on the next evaluation.
|
||||
func TestDedupeSurvivesOrdinaryNotes(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := refNow()
|
||||
seedMemory(t, st, ctx, now)
|
||||
|
||||
same := `[{"observation":"ты три дня не записывал еду","confidence":0.9,"suggested_action":"note"}]`
|
||||
f := &fakeLLM{replies: []string{same, same}}
|
||||
ev := NewEvaluator(st, st, f, Config{})
|
||||
|
||||
if _, err := ev.Evaluate(ctx, now); err != nil {
|
||||
t.Fatalf("Evaluate: %v", err)
|
||||
}
|
||||
// A few months of ordinary use between the two evaluations.
|
||||
for i := 0; i < DedupeWindow*2; i++ {
|
||||
if _, err := st.WriteNote(ctx, now.Add(time.Duration(i+1)*time.Minute),
|
||||
fmt.Sprintf("обычная заметка %d", i), nil, "tap:voice"); err != nil {
|
||||
t.Fatalf("write note: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := ev.Evaluate(ctx, now.Add(24*time.Hour)); err != nil {
|
||||
t.Fatalf("Evaluate: %v", err)
|
||||
}
|
||||
|
||||
own, err := st.RecentNotesBySource(ctx, EvalNoteSource, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentNotesBySource: %v", err)
|
||||
}
|
||||
if len(own) != 1 {
|
||||
t.Fatalf("eval notes = %d, want 1 — the repeat was not deduped", len(own))
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotBudgetIsNotEatenByOwnNotes — MaxItems notes must be MaxItems of
|
||||
// HIS notes. Reading MaxItems rows and then discarding hers left the model a
|
||||
// handful of real notes once hourly evaluation had run for a few weeks.
|
||||
func TestSnapshotBudgetIsNotEatenByOwnNotes(t *testing.T) {
|
||||
st := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
now := refNow()
|
||||
|
||||
// His notes first, then a long run of hers on top of them.
|
||||
for i := 0; i < 5; i++ {
|
||||
if _, err := st.WriteNote(ctx, now.Add(-time.Duration(100-i)*time.Hour),
|
||||
fmt.Sprintf("его заметка %d", i), nil, "tap:voice"); err != nil {
|
||||
t.Fatalf("write note: %v", err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 50; i++ {
|
||||
if _, err := st.WriteNote(ctx, now.Add(-time.Duration(50-i)*time.Hour),
|
||||
fmt.Sprintf("я заметила кое-что %d [note]", i), nil, EvalNoteSource); err != nil {
|
||||
t.Fatalf("write note: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
f := &fakeLLM{replies: []string{"[]"}}
|
||||
ev := NewEvaluator(st, st, f, Config{MaxItems: 5})
|
||||
if _, err := ev.Evaluate(ctx, now); err != nil {
|
||||
t.Fatalf("Evaluate: %v", err)
|
||||
}
|
||||
if len(f.calls) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(f.calls))
|
||||
}
|
||||
prompt := f.calls[0].User
|
||||
if strings.Contains(prompt, "я заметила") {
|
||||
t.Errorf("her own observations reached the prompt:\n%s", prompt)
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
if !strings.Contains(prompt, fmt.Sprintf("его заметка %d", i)) {
|
||||
t.Errorf("his note %d missing from the prompt:\n%s", i, prompt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStripActionKeepsBracketedTail — the dedupe key strips the recorded
|
||||
// action and nothing else. Cutting at the last " [" ate the end of an
|
||||
// observation that itself ends on a bracketed clause, so the same sentence
|
||||
// hashed two ways.
|
||||
func TestStripActionKeepsBracketedTail(t *testing.T) {
|
||||
text := "ты не пил воду [со вторника]"
|
||||
if got := stripAction(formatNote(Observation{Text: text, Action: "note"})); got != text {
|
||||
t.Errorf("stripAction = %q, want %q", got, text)
|
||||
}
|
||||
if got := stripAction(text); got != text {
|
||||
t.Errorf("stripAction = %q, want it untouched", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user