Maven/internal/memeval/eval.go

// Package memeval is background memory evaluation (Vikunja #248,
// docs/plans/03-memory-evaluation.md).
//
// It lives beside internal/memory rather than inside it because
// internal/store imports internal/memory for the vector-store backend, and an
// evaluator has to read store.Fact / store.Note / store.Nudge — putting it in
// internal/memory would close that import cycle.
//
// Every so often Maven reads back her own recent memory — facts, notes, the
// nudges she sent — and asks the resident model what it notices: a habit that
// stopped, a gap, something worth saying later. What comes back is written as
// notes with source EvalNoteSource and nothing else happens. That restraint is
// the design, not an unfinished edge:
//
//   - She does not speak here. There is no dispatcher, no channel, no nudge.
//     An observation is a thought she wrote down; he reads it on /dash when he
//     wants to. "Not a nag, not autonomous" (CLAUDE.md) is easy to violate with
//     exactly this feature — an hourly loop with an LLM in it and permission to
//     talk is a machine for generating interruptions — so the loop has no way
//     to reach him at all. Turning observations into nudges is a separate
//     decision with a separate opt-in, and it is deliberately NOT in this file.
//   - She does not act. No reminder is created, no routine proposed, no fact
//     written. The model's suggested_action is recorded as text inside the note
//     and interpreted by nobody.
//   - She says nothing about an empty store. No memory ⇒ no LLM call ⇒ no
//     "observations" invented out of two facts. A 1.7B asked to find a pattern
//     will always find one; the defence is not asking.
//
// Everything the evaluator writes is attributable: source is EvalNoteSource, so
// an inferred observation can never be mistaken for something he said, and the
// whole batch is one SQL delete away if the output turns out to be noise.
package memeval

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/kami/maven/internal/llm"
	"github.com/kami/maven/internal/persona"
	"github.com/kami/maven/internal/store"
)

// EvalNoteSource — the source stamped on every note the evaluator writes.
// Same infer:* convention as the rest of the derived facts.
const EvalNoteSource = "infer:memory-eval"

// DefaultMinConfidence — an observation below this is dropped. The model is
// asked for its own confidence and small models are badly calibrated, so this
// is a coarse filter, not a probability: it exists to throw away the guesses
// the model itself hedged on.
const DefaultMinConfidence = 0.7

// DefaultMaxItems — how much recent memory goes into one evaluation, per
// store. 30 facts + 30 notes + 30 nudges is a few thousand tokens of the 4096
// context the resident Thinking model runs with, which leaves room for its
// reasoning tokens. Raising this trades reasoning room for history.
const DefaultMaxItems = 30

// MaxObservations — the model may return at most this many observations per
// evaluation, enforced by the grammar. A cap here is also a noise cap: an
// evaluation that "notices" ten things has noticed nothing.
const MaxObservations = 3

// Observation — one thing the evaluator noticed.
type Observation struct {
	Text string  `json:"observation"`
	Conf float64 `json:"confidence"`
	// Action — what the model thinks should happen with this. Recorded, never
	// executed: see the file comment. One of "note", "propose", "notify".
	Action string `json:"suggested_action"`
}

// Completer — the llama-server seam, same shape router.Completer uses so the
// one resident model serves this caller too.
type Completer interface {
	Complete(ctx context.Context, r llm.Req) (string, error)
}

// Reader — the slice of the store an evaluation reads. Narrow on purpose: the
// evaluator gets recent memory and nothing else. No entity graph, no presence,
// no config facts.
type Reader interface {
	RecentFacts(ctx context.Context, n int) ([]store.Fact, error)
	// RecentNotesExcludingSource feeds the snapshot, RecentNotesBySource feeds
	// the dedupe. Both are source-scoped in SQL rather than filtered here: a
	// plain RecentNotes read makes each window a budget over ALL note writers,
	// so her own hourly observations shrink the snapshot and ordinary notes
	// push old observations out of the dedupe set.
	RecentNotesExcludingSource(ctx context.Context, source string, n int) ([]store.Note, error)
	RecentNotesBySource(ctx context.Context, source string, n int) ([]store.Note, error)
	RecentNudges(ctx context.Context, n int) ([]store.Nudge, error)
}

// NoteWriter — where observations land. Embeddings are passed nil: an
// observation is written for a human to read on /dash, not to be recalled by
// similarity. Feeding LLM-generated text back into the RAG pool it was
// generated from is how a small model starts citing its own guesses as
// evidence.
type NoteWriter interface {
	WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error)
}

// Config — evaluator tuning. Zero values are replaced by the Default*
// constants, so the zero Config is the sane one.
type Config struct {
	MaxItems      int
	MinConfidence float64

	// ContextBlock — the shared persona block (internal/persona), re-evaluated
	// per call so the clock in it is current. Prepended to the system prompt so
	// observations come out in Maven's voice: feminine self-reference, informal
	// "ты". nil is allowed; the base prompt still carries the address rules.
	ContextBlock func() string
}

// Evaluator reads recent memory and records what the model notices.
type Evaluator struct {
	read  Reader
	write NoteWriter
	llm   Completer
	cfg   Config
}

func NewEvaluator(r Reader, w NoteWriter, c Completer, cfg Config) *Evaluator {
	if cfg.MaxItems <= 0 {
		cfg.MaxItems = DefaultMaxItems
	}
	if cfg.MinConfidence <= 0 {
		cfg.MinConfidence = DefaultMinConfidence
	}
	return &Evaluator{read: r, write: w, llm: c, cfg: cfg}
}

// evalGrammar — GBNF pinning the reply to a bounded JSON array of fixed-shape
// observations. Same reasoning as the router's routeGrammar: the enum and the
// length bound are what stop a small model from drifting into free text or
// filling the token budget with one repeated field.
const evalGrammar = `
root   ::= "[" ws (obs ("," ws obs){0,2})? ws "]"
obs    ::= "{" ws "\"observation\"" ws ":" ws text "," ws "\"confidence\"" ws ":" ws conf "," ws "\"suggested_action\"" ws ":" ws act ws "}"
text   ::= "\"" ([^"\\] | "\\" .){1,200} "\""
conf   ::= "0" "." [0-9]{1,2} | "1" ("." "0")?
act    ::= "\"note\"" | "\"propose\"" | "\"notify\""
ws     ::= [ \t\n]*
`

// evalSystem — the evaluation prompt. Two things it insists on, both learned
// from the phraser: state the observation as something she noticed rather than
// an instruction, and say nothing when there is nothing (the model is given an
// explicit way to return an empty array, because a model with no exit returns
// filler).
const evalSystem = `Ты просматриваешь свою собственную память: недавние факты, заметки и напоминания, которые ты отправляла.
Найди то, что действительно заметно: привычка, которая прервалась; пробел в записях; повторяющаяся закономерность.

Правила:
- Отвечай ТОЛЬКО массивом JSON. Каждый элемент: {"observation": "...", "confidence": 0.0-1.0, "suggested_action": "note"|"propose"|"notify"}.
- observation — короткая фраза по-русски о том, что ты заметила. О себе — в женском роде ("я заметила"). К нему — на "ты".
- Не выдумывай. Если в памяти нет ничего заметного, верни пустой массив [].
- Не давай советов и не приказывай. Ты замечаешь, а не требуешь.
- confidence — насколько ты уверена, что это настоящая закономерность, а не совпадение.
- Максимум три наблюдения. Лучше одно точное, чем три общих.`

// Evaluate runs one evaluation and returns the observations it recorded.
//
// Returns (nil, nil) — not an error — for every ordinary "nothing to say"
// outcome: an empty store, an empty array from the model, everything below the
// confidence floor, or every observation already recorded earlier. Only a real
// read/LLM/write failure is an error, and the caller (a background ticker) logs
// it and waits for the next interval.
func (e *Evaluator) Evaluate(ctx context.Context, now time.Time) ([]Observation, error) {
	snap, err := e.snapshot(ctx)
	if err != nil {
		return nil, err
	}
	if snap == "" {
		return nil, nil // nothing recorded ⇒ nothing to notice, and no LLM call
	}

	raw, err := e.llm.Complete(ctx, llm.Req{
		System:        persona.Prepend(e.cfg.ContextBlock, evalSystem),
		User:          snap,
		Grammar:       evalGrammar,
		MaxTokens:     512,
		RepeatPenalty: 1.1,
	})
	if err != nil {
		return nil, fmt.Errorf("memory eval: complete: %w", err)
	}
	obs, err := parseObservations(raw)
	if err != nil {
		return nil, fmt.Errorf("memory eval: parse %q: %w", truncate(raw, 120), err)
	}

	// Dedupe against what earlier evaluations already wrote. Without this an
	// hourly loop over a slowly-changing store writes the same sentence every
	// hour until /dash is nothing but the evaluator talking to itself.
	seen, err := e.recordedTexts(ctx)
	if err != nil {
		return nil, err
	}

	var kept []Observation
	for _, o := range obs {
		o.Text = strings.TrimSpace(o.Text)
		if o.Text == "" || o.Conf < e.cfg.MinConfidence {
			continue
		}
		norm := normalizeObservation(o.Text)
		if seen[norm] {
			continue
		}
		seen[norm] = true
		if _, err := e.write.WriteNote(ctx, now, formatNote(o), nil, EvalNoteSource); err != nil {
			return kept, fmt.Errorf("memory eval: write note: %w", err)
		}
		kept = append(kept, o)
	}
	return kept, nil
}

// formatNote — the stored text. The suggested action is kept as a visible
// suffix rather than a column: it is the model's opinion about what to do next,
// and the only consumer is a human reading /dash.
func formatNote(o Observation) string {
	if o.Action == "" {
		return o.Text
	}
	return fmt.Sprintf("%s [%s]", o.Text, o.Action)
}

// DedupeWindow — how many of her OWN past observations the dedupe looks back
// over. Wider than MaxItems because the point is to remember saying it, not to
// summarize it. Counted in eval notes only: when this was a plain recent-notes
// read the window was really a budget over every note writer, so a few hundred
// ordinary notes pushed an observation out of sight and the next evaluation was
// free to write the same sentence again.
const DedupeWindow = 200

// recordedTexts — the normalized text of every observation earlier evaluations
// wrote, for dedupe.
func (e *Evaluator) recordedTexts(ctx context.Context) (map[string]bool, error) {
	notes, err := e.read.RecentNotesBySource(ctx, EvalNoteSource, DedupeWindow)
	if err != nil {
		return nil, fmt.Errorf("memory eval: recent notes: %w", err)
	}
	seen := make(map[string]bool, len(notes))
	for _, n := range notes {
		seen[normalizeObservation(stripAction(n.Text))] = true
	}
	return seen, nil
}

// actions — the suggested_action enum, as the grammar constrains it.
var actions = []string{"note", "propose", "notify"}

// stripAction removes the "[action]" suffix formatNote appended, and only that.
// Matching any bracketed tail would eat the end of an observation that happens
// to finish on a bracketed clause, which changes its dedupe key and lets the
// same sentence through twice.
func stripAction(text string) string {
	for _, a := range actions {
		if s, ok := strings.CutSuffix(text, " ["+a+"]"); ok {
			return s
		}
	}
	return text
}

// normalizeObservation — dedupe key. Case- and whitespace-insensitive, which
// catches the realistic repeat (the model re-emitting the same sentence with a
// different comma) without pretending to do semantic dedupe.
func normalizeObservation(s string) string {
	return strings.Join(strings.Fields(strings.ToLower(s)), " ")
}

// snapshot renders recent memory as the user turn. Returns "" when there is
// nothing in any store — the caller treats that as "do not ask the model".
//
// Notes written by earlier evaluations are excluded, in SQL. Feeding her own
// observations back in is how "я заметила, что ты не записывал еду" becomes
// evidence for noticing it again, three evaluations deep. Excluding them after
// the read was worse than not excluding them: hourly evaluation makes her own
// notes the majority of the newest rows within weeks, so asking for MaxItems
// and dropping hers left the model a handful of real notes to look at.
func (e *Evaluator) snapshot(ctx context.Context) (string, error) {
	n := e.cfg.MaxItems
	facts, err := e.read.RecentFacts(ctx, n)
	if err != nil {
		return "", fmt.Errorf("memory eval: recent facts: %w", err)
	}
	notes, err := e.read.RecentNotesExcludingSource(ctx, EvalNoteSource, n)
	if err != nil {
		return "", fmt.Errorf("memory eval: recent notes: %w", err)
	}
	nudges, err := e.read.RecentNudges(ctx, n)
	if err != nil {
		return "", fmt.Errorf("memory eval: recent nudges: %w", err)
	}

	var b strings.Builder
	wrote := false
	if len(facts) > 0 {
		b.WriteString("Факты:\n")
		for _, f := range facts {
			fmt.Fprintf(&b, "- %s %s=%s (%s)\n", f.Ts.Format("2006-01-02 15:04"), f.Key, truncate(f.Value, 80), f.Source)
			wrote = true
		}
	}
	if len(notes) > 0 {
		b.WriteString("\nЗаметки:\n")
		for _, nt := range notes {
			fmt.Fprintf(&b, "- %s %s\n", nt.Ts.Format("2006-01-02 15:04"), truncate(nt.Text, 160))
			wrote = true
		}
	}
	if len(nudges) > 0 {
		b.WriteString("\nНапоминания, которые ты отправляла:\n")
		for _, nd := range nudges {
			outcome := nd.Outcome
			if outcome == "" {
				outcome = "?"
			}
			fmt.Fprintf(&b, "- %s %s → %s (%s)\n", nd.Ts.Format("2006-01-02 15:04"), nd.Rule, outcome, nd.Channel)
			wrote = true
		}
	}
	if !wrote {
		// Only her own past observations, or nothing at all. Either way there is
		// no new memory to evaluate.
		return "", nil
	}
	b.WriteString("\nЧто ты замечаешь?")
	return b.String(), nil
}

// parseObservations reads the model's array. Tolerates the leading/trailing
// prose a Thinking model sometimes emits around JSON by taking the outermost
// bracketed span, the same tolerance the router's parser has.
func parseObservations(raw string) ([]Observation, error) {
	s := strings.TrimSpace(raw)
	if i := strings.Index(s, "["); i >= 0 {
		if j := strings.LastIndex(s, "]"); j > i {
			s = s[i : j+1]
		}
	}
	if s == "" {
		return nil, nil
	}
	var obs []Observation
	if err := json.Unmarshal([]byte(s), &obs); err != nil {
		return nil, err
	}
	if len(obs) > MaxObservations {
		// The grammar bounds this; a grammar-less server or a future prompt
		// change must not be able to flood /dash.
		sort.SliceStable(obs, func(i, j int) bool { return obs[i].Conf > obs[j].Conf })
		obs = obs[:MaxObservations]
	}
	return obs, nil
}

func truncate(s string, n int) string {
	r := []rune(s)
	if len(r) <= n {
		return s
	}
	return string(r[:n]) + "…"
}