Maven/internal/memeval/eval_test.go

package memeval

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/kami/maven/internal/llm"
	"github.com/kami/maven/internal/store"
)

// fakeLLM — canned replies, one per call, and a record of what it was asked.
type fakeLLM struct {
	replies []string
	calls   []llm.Req
	err     error
}

func (f *fakeLLM) Complete(_ context.Context, r llm.Req) (string, error) {
	f.calls = append(f.calls, r)
	if f.err != nil {
		return "", f.err
	}
	if len(f.replies) == 0 {
		return "[]", nil
	}
	out := f.replies[0]
	f.replies = f.replies[1:]
	return out, nil
}

func newTestStore(t *testing.T) *store.Store {
	t.Helper()
	st, err := store.Open(context.Background(), filepath.Join(t.TempDir(), "memeval_test.db"))
	if err != nil {
		t.Fatalf("store.Open: %v", err)
	}
	t.Cleanup(func() { _ = st.Close() })
	return st
}

func refNow() time.Time { return time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) }

// seedMemory writes a little of everything the evaluator reads.
func seedMemory(t *testing.T, st *store.Store, ctx context.Context, now time.Time) {
	t.Helper()
	for i := 0; i < 3; i++ {
		ts := now.Add(-time.Duration(i+1) * 24 * time.Hour)
		if _, err := st.WriteFact(ctx, ts, store.KindSelf, "water_ml", "500", "tap:desk", 1.0, sql.NullInt64{}); err != nil {
			t.Fatalf("write fact: %v", err)
		}
	}
	if _, err := st.WriteNote(ctx, now.Add(-2*time.Hour), "купить корм для кота", nil, "tap:voice"); err != nil {
		t.Fatalf("write note: %v", err)
	}
	if _, err := st.RecordNudge(ctx, "water", "voice", "пора выпить воды", now.Add(-time.Hour)); err != nil {
		t.Fatalf("record nudge: %v", err)
	}
}

// TestEvaluateEmptyStoreAsksNothing — the "shuts up when uncertain" floor. An
// empty store must not even reach the model: a small model asked to find a
// pattern in nothing will invent one.
func TestEvaluateEmptyStoreAsksNothing(t *testing.T) {
	st := newTestStore(t)
	ctx := context.Background()
	f := &fakeLLM{}
	ev := NewEvaluator(st, st, f, Config{})

	obs, err := ev.Evaluate(ctx, refNow())
	if err != nil {
		t.Fatalf("Evaluate: %v", err)
	}
	if len(obs) != 0 {
		t.Fatalf("observations on an empty store = %d, want 0", len(obs))
	}
	if len(f.calls) != 0 {
		t.Fatalf("LLM called %d times on an empty store, want 0", len(f.calls))
	}
}

// TestEvaluateWritesHighConfidenceObservations — the happy path. Confident
// observations are written as notes stamped infer:memory-eval, and the low
// ones are dropped.
func TestEvaluateWritesHighConfidenceObservations(t *testing.T) {
	st := newTestStore(t)
	ctx := context.Background()
	now := refNow()
	seedMemory(t, st, ctx, now)

	f := &fakeLLM{replies: []string{`[
		{"observation":"ты три дня не записывал еду","confidence":0.9,"suggested_action":"notify"},
		{"observation":"может быть, ты стал меньше пить воды","confidence":0.3,"suggested_action":"note"}
	]`}}
	ev := NewEvaluator(st, st, f, Config{})

	obs, err := ev.Evaluate(ctx, now)
	if err != nil {
		t.Fatalf("Evaluate: %v", err)
	}
	if len(obs) != 1 {
		t.Fatalf("kept %d observations, want 1 (the 0.3 one is below the floor): %+v", len(obs), obs)
	}
	if obs[0].Text != "ты три дня не записывал еду" {
		t.Errorf("kept the wrong observation: %q", obs[0].Text)
	}

	notes, err := st.RecentNotes(ctx, 50)
	if err != nil {
		t.Fatalf("RecentNotes: %v", err)
	}
	var written []store.Note
	for _, n := range notes {
		if n.Source == EvalNoteSource {
			written = append(written, n)
		}
	}
	if len(written) != 1 {
		t.Fatalf("notes with source %s = %d, want 1", EvalNoteSource, len(written))
	}
	if !strings.Contains(written[0].Text, "ты три дня не записывал еду") {
		t.Errorf("note text = %q", written[0].Text)
	}
	if !strings.Contains(written[0].Text, "[notify]") {
		t.Errorf("note text = %q, want the suggested action recorded", written[0].Text)
	}

	// The prompt must carry the memory it is evaluating, and must not carry a
	// grammar-free request.
	if len(f.calls) != 1 {
		t.Fatalf("LLM calls = %d, want 1", len(f.calls))
	}
	if !strings.Contains(f.calls[0].User, "water_ml") {
		t.Errorf("prompt does not mention the seeded facts:\n%s", f.calls[0].User)
	}
	if f.calls[0].Grammar == "" {
		t.Error("evaluation ran without a grammar")
	}
}

// TestEvaluateDeduplicatesAcrossRuns — the failure mode that would make this
// feature unusable: an hourly loop over a store that barely changes writing the
// same sentence every hour until /dash is nothing but the evaluator.
func TestEvaluateDeduplicatesAcrossRuns(t *testing.T) {
	st := newTestStore(t)
	ctx := context.Background()
	now := refNow()
	seedMemory(t, st, ctx, now)

	same := `[{"observation":"ты три дня не записывал еду","confidence":0.9,"suggested_action":"note"}]`
	spaced := `[{"observation":"Ты  три дня   не записывал еду","confidence":0.95,"suggested_action":"note"}]`
	f := &fakeLLM{replies: []string{same, same, spaced}}
	ev := NewEvaluator(st, st, f, Config{})

	for i := 0; i < 3; i++ {
		if _, err := ev.Evaluate(ctx, now.Add(time.Duration(i)*time.Hour)); err != nil {
			t.Fatalf("Evaluate %d: %v", i, err)
		}
	}

	notes, err := st.RecentNotes(ctx, 50)
	if err != nil {
		t.Fatalf("RecentNotes: %v", err)
	}
	n := 0
	for _, nt := range notes {
		if nt.Source == EvalNoteSource {
			n++
		}
	}
	if n != 1 {
		t.Fatalf("eval notes after three identical evaluations = %d, want 1", n)
	}
}

// TestEvaluateIgnoresOwnNotes — her own observations must not become input.
// Otherwise "я заметила X" is evidence for noticing X again, three evaluations
// deep. With nothing but eval notes in the store there is no new memory, so the
// model is not asked at all.
func TestEvaluateIgnoresOwnNotes(t *testing.T) {
	st := newTestStore(t)
	ctx := context.Background()
	now := refNow()
	if _, err := st.WriteNote(ctx, now.Add(-time.Hour), "я заметила, что ты мало пьёшь [note]", nil, EvalNoteSource); err != nil {
		t.Fatalf("write note: %v", err)
	}

	f := &fakeLLM{}
	ev := NewEvaluator(st, st, f, Config{})
	obs, err := ev.Evaluate(ctx, now)
	if err != nil {
		t.Fatalf("Evaluate: %v", err)
	}
	if len(obs) != 0 || len(f.calls) != 0 {
		t.Fatalf("observations=%d llm calls=%d, want 0/0 — own notes are not memory to evaluate", len(obs), len(f.calls))
	}
}

// TestEvaluateEmptyArrayIsNotAnError — "nothing to say" is the expected outcome
// most of the time and must not be logged as a failure.
func TestEvaluateEmptyArrayIsNotAnError(t *testing.T) {
	st := newTestStore(t)
	ctx := context.Background()
	now := refNow()
	seedMemory(t, st, ctx, now)

	ev := NewEvaluator(st, st, &fakeLLM{replies: []string{"[]"}}, Config{})
	obs, err := ev.Evaluate(ctx, now)
	if err != nil {
		t.Fatalf("Evaluate: %v", err)
	}
	if len(obs) != 0 {
		t.Fatalf("observations = %d, want 0", len(obs))
	}
}

// TestEvaluateLLMErrorIsReported — a broken llama-server is an error the caller
// logs; it must not silently write anything.
func TestEvaluateLLMErrorIsReported(t *testing.T) {
	st := newTestStore(t)
	ctx := context.Background()
	now := refNow()
	seedMemory(t, st, ctx, now)

	ev := NewEvaluator(st, st, &fakeLLM{err: errors.New("connection refused")}, Config{})
	if _, err := ev.Evaluate(ctx, now); err == nil {
		t.Fatal("want an error when the model is unreachable")
	}
	notes, err := st.RecentNotes(ctx, 50)
	if err != nil {
		t.Fatalf("RecentNotes: %v", err)
	}
	for _, n := range notes {
		if n.Source == EvalNoteSource {
			t.Fatalf("wrote a note despite an LLM failure: %q", n.Text)
		}
	}
}

// TestParseObservationsTolerantAndBounded — Thinking models wrap JSON in prose,
// and no reply may exceed MaxObservations even if the grammar is bypassed.
func TestParseObservationsTolerantAndBounded(t *testing.T) {
	obs, err := parseObservations(`<think>hmm</think> вот: [{"observation":"a","confidence":0.9,"suggested_action":"note"}] всё`)
	if err != nil {
		t.Fatalf("parse: %v", err)
	}
	if len(obs) != 1 || obs[0].Text != "a" {
		t.Fatalf("got %+v, want one observation 'a'", obs)
	}

	var b strings.Builder
	b.WriteString("[")
	for i := 0; i < MaxObservations+3; i++ {
		if i > 0 {
			b.WriteString(",")
		}
		b.WriteString(`{"observation":"x","confidence":0.5,"suggested_action":"note"}`)
	}
	b.WriteString("]")
	obs, err = parseObservations(b.String())
	if err != nil {
		t.Fatalf("parse: %v", err)
	}
	if len(obs) != MaxObservations {
		t.Fatalf("parsed %d observations, want the %d cap", len(obs), MaxObservations)
	}
}

// TestDedupeSurvivesOrdinaryNotes — the dedupe window is a count of HER notes,
// not of all notes. With a plain recent-notes read, DedupeWindow ordinary notes
// written after an observation pushed it out of sight and the same sentence was
// written again on the next evaluation.
func TestDedupeSurvivesOrdinaryNotes(t *testing.T) {
	st := newTestStore(t)
	ctx := context.Background()
	now := refNow()
	seedMemory(t, st, ctx, now)

	same := `[{"observation":"ты три дня не записывал еду","confidence":0.9,"suggested_action":"note"}]`
	f := &fakeLLM{replies: []string{same, same}}
	ev := NewEvaluator(st, st, f, Config{})

	if _, err := ev.Evaluate(ctx, now); err != nil {
		t.Fatalf("Evaluate: %v", err)
	}
	// A few months of ordinary use between the two evaluations.
	for i := 0; i < DedupeWindow*2; i++ {
		if _, err := st.WriteNote(ctx, now.Add(time.Duration(i+1)*time.Minute),
			fmt.Sprintf("обычная заметка %d", i), nil, "tap:voice"); err != nil {
			t.Fatalf("write note: %v", err)
		}
	}
	if _, err := ev.Evaluate(ctx, now.Add(24*time.Hour)); err != nil {
		t.Fatalf("Evaluate: %v", err)
	}

	own, err := st.RecentNotesBySource(ctx, EvalNoteSource, 100)
	if err != nil {
		t.Fatalf("RecentNotesBySource: %v", err)
	}
	if len(own) != 1 {
		t.Fatalf("eval notes = %d, want 1 — the repeat was not deduped", len(own))
	}
}

// TestSnapshotBudgetIsNotEatenByOwnNotes — MaxItems notes must be MaxItems of
// HIS notes. Reading MaxItems rows and then discarding hers left the model a
// handful of real notes once hourly evaluation had run for a few weeks.
func TestSnapshotBudgetIsNotEatenByOwnNotes(t *testing.T) {
	st := newTestStore(t)
	ctx := context.Background()
	now := refNow()

	// His notes first, then a long run of hers on top of them.
	for i := 0; i < 5; i++ {
		if _, err := st.WriteNote(ctx, now.Add(-time.Duration(100-i)*time.Hour),
			fmt.Sprintf("его заметка %d", i), nil, "tap:voice"); err != nil {
			t.Fatalf("write note: %v", err)
		}
	}
	for i := 0; i < 50; i++ {
		if _, err := st.WriteNote(ctx, now.Add(-time.Duration(50-i)*time.Hour),
			fmt.Sprintf("я заметила кое-что %d [note]", i), nil, EvalNoteSource); err != nil {
			t.Fatalf("write note: %v", err)
		}
	}

	f := &fakeLLM{replies: []string{"[]"}}
	ev := NewEvaluator(st, st, f, Config{MaxItems: 5})
	if _, err := ev.Evaluate(ctx, now); err != nil {
		t.Fatalf("Evaluate: %v", err)
	}
	if len(f.calls) != 1 {
		t.Fatalf("LLM calls = %d, want 1", len(f.calls))
	}
	prompt := f.calls[0].User
	if strings.Contains(prompt, "я заметила") {
		t.Errorf("her own observations reached the prompt:\n%s", prompt)
	}
	for i := 0; i < 5; i++ {
		if !strings.Contains(prompt, fmt.Sprintf("его заметка %d", i)) {
			t.Errorf("his note %d missing from the prompt:\n%s", i, prompt)
		}
	}
}

// TestStripActionKeepsBracketedTail — the dedupe key strips the recorded
// action and nothing else. Cutting at the last " [" ate the end of an
// observation that itself ends on a bracketed clause, so the same sentence
// hashed two ways.
func TestStripActionKeepsBracketedTail(t *testing.T) {
	text := "ты не пил воду [со вторника]"
	if got := stripAction(formatNote(Observation{Text: text, Action: "note"})); got != text {
		t.Errorf("stripAction = %q, want %q", got, text)
	}
	if got := stripAction(text); got != text {
		t.Errorf("stripAction = %q, want it untouched", got)
	}
}