// Package memeval is background memory evaluation (Vikunja #248, // docs/plans/03-memory-evaluation.md). // // It lives beside internal/memory rather than inside it because // internal/store imports internal/memory for the vector-store backend, and an // evaluator has to read store.Fact / store.Note / store.Nudge — putting it in // internal/memory would close that import cycle. // // Every so often Maven reads back her own recent memory — facts, notes, the // nudges she sent — and asks the resident model what it notices: a habit that // stopped, a gap, something worth saying later. What comes back is written as // notes with source EvalNoteSource and nothing else happens. That restraint is // the design, not an unfinished edge: // // - She does not speak here. There is no dispatcher, no channel, no nudge. // An observation is a thought she wrote down; he reads it on /dash when he // wants to. "Not a nag, not autonomous" (CLAUDE.md) is easy to violate with // exactly this feature — an hourly loop with an LLM in it and permission to // talk is a machine for generating interruptions — so the loop has no way // to reach him at all. Turning observations into nudges is a separate // decision with a separate opt-in, and it is deliberately NOT in this file. // - She does not act. No reminder is created, no routine proposed, no fact // written. The model's suggested_action is recorded as text inside the note // and interpreted by nobody. // - She says nothing about an empty store. No memory ⇒ no LLM call ⇒ no // "observations" invented out of two facts. A 1.7B asked to find a pattern // will always find one; the defence is not asking. // // Everything the evaluator writes is attributable: source is EvalNoteSource, so // an inferred observation can never be mistaken for something he said, and the // whole batch is one SQL delete away if the output turns out to be noise. package memeval import ( "context" "encoding/json" "fmt" "sort" "strings" "time" "github.com/kami/maven/internal/llm" "github.com/kami/maven/internal/persona" "github.com/kami/maven/internal/store" ) // EvalNoteSource — the source stamped on every note the evaluator writes. // Same infer:* convention as the rest of the derived facts. const EvalNoteSource = "infer:memory-eval" // DefaultMinConfidence — an observation below this is dropped. The model is // asked for its own confidence and small models are badly calibrated, so this // is a coarse filter, not a probability: it exists to throw away the guesses // the model itself hedged on. const DefaultMinConfidence = 0.7 // DefaultMaxItems — how much recent memory goes into one evaluation, per // store. 30 facts + 30 notes + 30 nudges is a few thousand tokens of the 4096 // context the resident Thinking model runs with, which leaves room for its // reasoning tokens. Raising this trades reasoning room for history. const DefaultMaxItems = 30 // MaxObservations — the model may return at most this many observations per // evaluation, enforced by the grammar. A cap here is also a noise cap: an // evaluation that "notices" ten things has noticed nothing. const MaxObservations = 3 // Observation — one thing the evaluator noticed. type Observation struct { Text string `json:"observation"` Conf float64 `json:"confidence"` // Action — what the model thinks should happen with this. Recorded, never // executed: see the file comment. One of "note", "propose", "notify". Action string `json:"suggested_action"` } // Completer — the llama-server seam, same shape router.Completer uses so the // one resident model serves this caller too. type Completer interface { Complete(ctx context.Context, r llm.Req) (string, error) } // Reader — the slice of the store an evaluation reads. Narrow on purpose: the // evaluator gets recent memory and nothing else. No entity graph, no presence, // no config facts. type Reader interface { RecentFacts(ctx context.Context, n int) ([]store.Fact, error) // RecentNotesExcludingSource feeds the snapshot, RecentNotesBySource feeds // the dedupe. Both are source-scoped in SQL rather than filtered here: a // plain RecentNotes read makes each window a budget over ALL note writers, // so her own hourly observations shrink the snapshot and ordinary notes // push old observations out of the dedupe set. RecentNotesExcludingSource(ctx context.Context, source string, n int) ([]store.Note, error) RecentNotesBySource(ctx context.Context, source string, n int) ([]store.Note, error) RecentNudges(ctx context.Context, n int) ([]store.Nudge, error) } // NoteWriter — where observations land. Embeddings are passed nil: an // observation is written for a human to read on /dash, not to be recalled by // similarity. Feeding LLM-generated text back into the RAG pool it was // generated from is how a small model starts citing its own guesses as // evidence. type NoteWriter interface { WriteNote(ctx context.Context, ts time.Time, text string, embedding []float32, source string) (int64, error) } // Config — evaluator tuning. Zero values are replaced by the Default* // constants, so the zero Config is the sane one. type Config struct { MaxItems int MinConfidence float64 // ContextBlock — the shared persona block (internal/persona), re-evaluated // per call so the clock in it is current. Prepended to the system prompt so // observations come out in Maven's voice: feminine self-reference, informal // "ты". nil is allowed; the base prompt still carries the address rules. ContextBlock func() string } // Evaluator reads recent memory and records what the model notices. type Evaluator struct { read Reader write NoteWriter llm Completer cfg Config } func NewEvaluator(r Reader, w NoteWriter, c Completer, cfg Config) *Evaluator { if cfg.MaxItems <= 0 { cfg.MaxItems = DefaultMaxItems } if cfg.MinConfidence <= 0 { cfg.MinConfidence = DefaultMinConfidence } return &Evaluator{read: r, write: w, llm: c, cfg: cfg} } // evalGrammar — GBNF pinning the reply to a bounded JSON array of fixed-shape // observations. Same reasoning as the router's routeGrammar: the enum and the // length bound are what stop a small model from drifting into free text or // filling the token budget with one repeated field. const evalGrammar = ` root ::= "[" ws (obs ("," ws obs){0,2})? ws "]" obs ::= "{" ws "\"observation\"" ws ":" ws text "," ws "\"confidence\"" ws ":" ws conf "," ws "\"suggested_action\"" ws ":" ws act ws "}" text ::= "\"" ([^"\\] | "\\" .){1,200} "\"" conf ::= "0" "." [0-9]{1,2} | "1" ("." "0")? act ::= "\"note\"" | "\"propose\"" | "\"notify\"" ws ::= [ \t\n]* ` // evalSystem — the evaluation prompt. Two things it insists on, both learned // from the phraser: state the observation as something she noticed rather than // an instruction, and say nothing when there is nothing (the model is given an // explicit way to return an empty array, because a model with no exit returns // filler). const evalSystem = `Ты просматриваешь свою собственную память: недавние факты, заметки и напоминания, которые ты отправляла. Найди то, что действительно заметно: привычка, которая прервалась; пробел в записях; повторяющаяся закономерность. Правила: - Отвечай ТОЛЬКО массивом JSON. Каждый элемент: {"observation": "...", "confidence": 0.0-1.0, "suggested_action": "note"|"propose"|"notify"}. - observation — короткая фраза по-русски о том, что ты заметила. О себе — в женском роде ("я заметила"). К нему — на "ты". - Не выдумывай. Если в памяти нет ничего заметного, верни пустой массив []. - Не давай советов и не приказывай. Ты замечаешь, а не требуешь. - confidence — насколько ты уверена, что это настоящая закономерность, а не совпадение. - Максимум три наблюдения. Лучше одно точное, чем три общих.` // Evaluate runs one evaluation and returns the observations it recorded. // // Returns (nil, nil) — not an error — for every ordinary "nothing to say" // outcome: an empty store, an empty array from the model, everything below the // confidence floor, or every observation already recorded earlier. Only a real // read/LLM/write failure is an error, and the caller (a background ticker) logs // it and waits for the next interval. func (e *Evaluator) Evaluate(ctx context.Context, now time.Time) ([]Observation, error) { snap, err := e.snapshot(ctx) if err != nil { return nil, err } if snap == "" { return nil, nil // nothing recorded ⇒ nothing to notice, and no LLM call } raw, err := e.llm.Complete(ctx, llm.Req{ System: persona.Prepend(e.cfg.ContextBlock, evalSystem), User: snap, Grammar: evalGrammar, MaxTokens: 512, RepeatPenalty: 1.1, }) if err != nil { return nil, fmt.Errorf("memory eval: complete: %w", err) } obs, err := parseObservations(raw) if err != nil { return nil, fmt.Errorf("memory eval: parse %q: %w", truncate(raw, 120), err) } // Dedupe against what earlier evaluations already wrote. Without this an // hourly loop over a slowly-changing store writes the same sentence every // hour until /dash is nothing but the evaluator talking to itself. seen, err := e.recordedTexts(ctx) if err != nil { return nil, err } var kept []Observation for _, o := range obs { o.Text = strings.TrimSpace(o.Text) if o.Text == "" || o.Conf < e.cfg.MinConfidence { continue } norm := normalizeObservation(o.Text) if seen[norm] { continue } seen[norm] = true if _, err := e.write.WriteNote(ctx, now, formatNote(o), nil, EvalNoteSource); err != nil { return kept, fmt.Errorf("memory eval: write note: %w", err) } kept = append(kept, o) } return kept, nil } // formatNote — the stored text. The suggested action is kept as a visible // suffix rather than a column: it is the model's opinion about what to do next, // and the only consumer is a human reading /dash. func formatNote(o Observation) string { if o.Action == "" { return o.Text } return fmt.Sprintf("%s [%s]", o.Text, o.Action) } // DedupeWindow — how many of her OWN past observations the dedupe looks back // over. Wider than MaxItems because the point is to remember saying it, not to // summarize it. Counted in eval notes only: when this was a plain recent-notes // read the window was really a budget over every note writer, so a few hundred // ordinary notes pushed an observation out of sight and the next evaluation was // free to write the same sentence again. const DedupeWindow = 200 // recordedTexts — the normalized text of every observation earlier evaluations // wrote, for dedupe. func (e *Evaluator) recordedTexts(ctx context.Context) (map[string]bool, error) { notes, err := e.read.RecentNotesBySource(ctx, EvalNoteSource, DedupeWindow) if err != nil { return nil, fmt.Errorf("memory eval: recent notes: %w", err) } seen := make(map[string]bool, len(notes)) for _, n := range notes { seen[normalizeObservation(stripAction(n.Text))] = true } return seen, nil } // actions — the suggested_action enum, as the grammar constrains it. var actions = []string{"note", "propose", "notify"} // stripAction removes the "[action]" suffix formatNote appended, and only that. // Matching any bracketed tail would eat the end of an observation that happens // to finish on a bracketed clause, which changes its dedupe key and lets the // same sentence through twice. func stripAction(text string) string { for _, a := range actions { if s, ok := strings.CutSuffix(text, " ["+a+"]"); ok { return s } } return text } // normalizeObservation — dedupe key. Case- and whitespace-insensitive, which // catches the realistic repeat (the model re-emitting the same sentence with a // different comma) without pretending to do semantic dedupe. func normalizeObservation(s string) string { return strings.Join(strings.Fields(strings.ToLower(s)), " ") } // snapshot renders recent memory as the user turn. Returns "" when there is // nothing in any store — the caller treats that as "do not ask the model". // // Notes written by earlier evaluations are excluded, in SQL. Feeding her own // observations back in is how "я заметила, что ты не записывал еду" becomes // evidence for noticing it again, three evaluations deep. Excluding them after // the read was worse than not excluding them: hourly evaluation makes her own // notes the majority of the newest rows within weeks, so asking for MaxItems // and dropping hers left the model a handful of real notes to look at. func (e *Evaluator) snapshot(ctx context.Context) (string, error) { n := e.cfg.MaxItems facts, err := e.read.RecentFacts(ctx, n) if err != nil { return "", fmt.Errorf("memory eval: recent facts: %w", err) } notes, err := e.read.RecentNotesExcludingSource(ctx, EvalNoteSource, n) if err != nil { return "", fmt.Errorf("memory eval: recent notes: %w", err) } nudges, err := e.read.RecentNudges(ctx, n) if err != nil { return "", fmt.Errorf("memory eval: recent nudges: %w", err) } var b strings.Builder wrote := false if len(facts) > 0 { b.WriteString("Факты:\n") for _, f := range facts { fmt.Fprintf(&b, "- %s %s=%s (%s)\n", f.Ts.Format("2006-01-02 15:04"), f.Key, truncate(f.Value, 80), f.Source) wrote = true } } if len(notes) > 0 { b.WriteString("\nЗаметки:\n") for _, nt := range notes { fmt.Fprintf(&b, "- %s %s\n", nt.Ts.Format("2006-01-02 15:04"), truncate(nt.Text, 160)) wrote = true } } if len(nudges) > 0 { b.WriteString("\nНапоминания, которые ты отправляла:\n") for _, nd := range nudges { outcome := nd.Outcome if outcome == "" { outcome = "?" } fmt.Fprintf(&b, "- %s %s → %s (%s)\n", nd.Ts.Format("2006-01-02 15:04"), nd.Rule, outcome, nd.Channel) wrote = true } } if !wrote { // Only her own past observations, or nothing at all. Either way there is // no new memory to evaluate. return "", nil } b.WriteString("\nЧто ты замечаешь?") return b.String(), nil } // parseObservations reads the model's array. Tolerates the leading/trailing // prose a Thinking model sometimes emits around JSON by taking the outermost // bracketed span, the same tolerance the router's parser has. func parseObservations(raw string) ([]Observation, error) { s := strings.TrimSpace(raw) if i := strings.Index(s, "["); i >= 0 { if j := strings.LastIndex(s, "]"); j > i { s = s[i : j+1] } } if s == "" { return nil, nil } var obs []Observation if err := json.Unmarshal([]byte(s), &obs); err != nil { return nil, err } if len(obs) > MaxObservations { // The grammar bounds this; a grammar-less server or a future prompt // change must not be able to flood /dash. sort.SliceStable(obs, func(i, j int) bool { return obs[i].Conf > obs[j].Conf }) obs = obs[:MaxObservations] } return obs, nil } func truncate(s string, n int) string { r := []rune(s) if len(r) <= n { return s } return string(r[:n]) + "…" }