Maven/internal/memory/recalleval/recalleval.go

// Package recalleval is the held-out contract for note recall: can Maven find
// the right note again when the user asks for it weeks later?
//
// Why it sits beside internal/memory rather than inside it: the thing under
// test is a whole path, not one function — an embedder (internal/router), a
// vector store (internal/memory or internal/store) and the confidence gate the
// daemon applies on top (cmd/mavend/recall.go's bestRecall, config's
// query_min_score). A _test.go file inside internal/memory could not reach the
// persistent store without an import cycle, and testdata is not reachable from
// another package's working directory — so the fixture is embedded here and the
// scorer takes the store as a factory. Same layout and same reasons as
// internal/router/eval.
//
// The fixture is HELD OUT the same way the routing fixture is: a query never
// repeats its note's wording verbatim beyond ordinary shared vocabulary, and
// TestFixtureIsParaphrased enforces a floor on how little the two overlap.
// Scoring recall on a query that is a copy of the note measures string
// matching, not recall.
package recalleval

import (
	"context"
	_ "embed"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/kami/maven/internal/memory"
	"github.com/kami/maven/internal/router"
)

//go:embed ru_recall_v1.json
var fixtureJSON []byte

// SchemaVersion — the version this package understands. The loader refuses any
// other version rather than misreading a fixture and reporting a number.
const SchemaVersion = 1

// daemonTopK — how many candidates the read path asks for, mirroring
// memoryRecallWidth in cmd/mavend/actions_query.go. Kept in step so recall@3
// measures the same top-3 the daemon actually reads, not an arbitrary width.
const daemonTopK = 3

// StoredNote — one thing the user said once, as it lands in the semantic store.
// Kind is "note" or "fact"; both share the vector index (see
// cmd/mavend/recall.go), so a fact can legitimately win a recall.
type StoredNote struct {
	ID   string `json:"id"`
	Text string `json:"text"`
	Kind string `json:"kind"`
}

// Case — a small set of notes, one query, and the note that must come back
// first. Want is empty exactly when the query should recall NOTHING: that lane
// measures false recall, which is the direction the spec cares about ("a
// confident wrong fact is worse than a known gap").
type Case struct {
	ID    string       `json:"id"`
	Lang  string       `json:"lang"`
	Notes []StoredNote `json:"notes"`
	Query string       `json:"query"`
	Want  string       `json:"want"`
	Tags  []string     `json:"tags"`
	Note  string       `json:"note"`
}

// Answerable reports whether the case expects a recall at all.
func (c Case) Answerable() bool { return c.Want != "" }

// Fixture — the versioned envelope, same shape as the routing fixture.
//
// Filler is inserted into EVERY case's store on top of that case's own notes.
// Without it a case with three notes scores recall@3 = 100% by construction,
// which measures nothing. A real store holds months of unrelated notes, and the
// wanted note has to beat all of them.
type Fixture struct {
	SchemaVersion int          `json:"schema_version"`
	Name          string       `json:"name"`
	Notes         []string     `json:"notes"`
	Filler        []StoredNote `json:"filler"`
	Cases         []Case       `json:"cases"`
}

// Load returns the embedded fixture.
func Load() (Fixture, error) {
	var f Fixture
	if err := json.Unmarshal(fixtureJSON, &f); err != nil {
		return Fixture{}, fmt.Errorf("parse fixture: %w", err)
	}
	if f.SchemaVersion != SchemaVersion {
		return Fixture{}, fmt.Errorf("fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion)
	}
	if len(f.Cases) == 0 {
		return Fixture{}, fmt.Errorf("fixture has no cases")
	}
	if err := checkIDs(f); err != nil {
		return Fixture{}, err
	}
	return f, nil
}

// checkIDs refuses a fixture where a case note and a filler note share an id.
//
// Every case is scored over its own notes plus the whole filler set, and the
// two stores disagree about what a repeated id means: the sqlite store upserts
// on it, the in-memory store appends. So one collision makes a case score
// differently on the two backends, and it reads as an embedder or gate
// difference, which is the one thing this harness exists to measure (Vikunja
// #386). It was dodged once by hand during #373 by renaming two ids.
//
// Checked in Load rather than in the test, so every caller of the fixture is
// covered and not only the one that remembers to look.
func checkIDs(f Fixture) error {
	filler := make(map[string]bool, len(f.Filler))
	for _, n := range f.Filler {
		if n.ID == "" {
			return fmt.Errorf("filler note with an empty id")
		}
		if filler[n.ID] {
			return fmt.Errorf("duplicate filler note id %q", n.ID)
		}
		filler[n.ID] = true
	}
	for _, c := range f.Cases {
		for _, n := range c.Notes {
			if filler[n.ID] {
				return fmt.Errorf("case %s: note id %q collides with a filler note", c.ID, n.ID)
			}
		}
	}
	return nil
}

// NewStore builds an empty store for one case, plus a function to release it.
// A factory rather than a store because every case needs a clean index — notes
// from case A must not be visible to case B's query.
type NewStore func() (memory.Store, func(), error)

// InMemory is the NewStore for memory.InMemoryStore — the fallback the daemon
// uses when there is no database (cmd/mavend/voice.go:224).
func InMemory() (memory.Store, func(), error) {
	return memory.NewInMemoryStore(), func() {}, nil
}

// Cache wraps an embedder so repeated text is embedded once. The gate sweep
// scores the same fixture at nine thresholds, and every case re-inserts the
// filler notes — without this the ONNX run spends minutes re-embedding
// identical strings. Latency numbers come from the uncached run.
func Cache(inner router.Embedder) router.Embedder {
	return &cachingEmbedder{inner: inner, seen: map[string][]float32{}}
}

type cachingEmbedder struct {
	inner router.Embedder
	seen  map[string][]float32
}

var _ router.AsymmetricEmbedder = (*cachingEmbedder)(nil)

func (c *cachingEmbedder) Dim() int     { return c.inner.Dim() }
func (c *cachingEmbedder) Close() error { return nil } // the caller owns inner

func (c *cachingEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
	return c.cached(ctx, "embed:"+text, func() ([]float32, error) {
		return c.inner.Embed(ctx, text)
	})
}

// The two sides of an asymmetric embedder give different vectors for the same
// string, so the cache key has to say which side asked.
func (c *cachingEmbedder) EmbedQuery(ctx context.Context, text string) ([]float32, error) {
	return c.cached(ctx, "query:"+text, func() ([]float32, error) {
		return router.EmbedQuery(ctx, c.inner, text)
	})
}

func (c *cachingEmbedder) EmbedPassage(ctx context.Context, text string) ([]float32, error) {
	return c.cached(ctx, "passage:"+text, func() ([]float32, error) {
		return router.EmbedPassage(ctx, c.inner, text)
	})
}

func (c *cachingEmbedder) cached(_ context.Context, key string, embed func() ([]float32, error)) ([]float32, error) {
	if v, ok := c.seen[key]; ok {
		return v, nil
	}
	v, err := embed()
	if err != nil {
		return nil, err
	}
	c.seen[key] = v
	return v, nil
}

// Outcome — one scored case.
type Outcome struct {
	Case Case
	Hits []memory.Result
	Err  error
	// Latency is the read path only: embed the query, then Search. Insert time
	// is excluded because it happens once, weeks earlier.
	Latency time.Duration
	// Rank1/Rank3 — the wanted note came back first / in the top three,
	// ignoring the confidence gate. Ranking is the store's job.
	Rank1 bool
	Rank3 bool
	// Recalled — what the daemon would actually say back: the top hit's text
	// when it clears the gate. Mirrors bestRecall in cmd/mavend/recall.go.
	Recalled string
	// Pass — the wanted note was recalled AND survived the gate; or, for a
	// no-answer case, nothing was recalled.
	Pass bool
	// Tied — the wanted note is on top but shares its score with the next hit,
	// so the sort decided it, not the embedder. Counted apart from a real hit.
	Tied    bool
	TopID   string
	TopScor float64
	// Margin — top1 − top2. 0 when fewer than two hits came back.
	Margin  float64
	Reasons []string
}

// Report — the aggregate. Rank and gate are kept apart on purpose: a note that
// ranks first but is silenced by query_min_score is a threshold problem, and a
// note that never ranks first is an embedder problem. Those are different fixes.
type Report struct {
	Name     string
	MinScore float64
	// MinMargin — how far the top hit must beat the runner-up. 0 ⇒ off.
	MinMargin  float64
	Total      int
	Answerable int
	Rank1      int
	Rank3      int
	// Gated — ranked first but the score was under MinScore, so the daemon
	// stays silent and answers "не знаю".
	Gated int
	// WrongTop — a different note outranked the right one.
	WrongTop int
	// Tied — the right note was on top only because of sort order. Not credited
	// as recall; tracked because it is a distinct failure (the embedder scored
	// the query and the note the same as everything else).
	Tied int
	// NoAnswer / FalseRecall — the cases that must recall nothing, and how many
	// of them the daemon would answer anyway.
	NoAnswer    int
	FalseRecall int
	Errors      int
	Passed      int
	Outcomes    []Outcome
	ByTag       map[string]TagStat
	ByLang      map[string]TagStat
	// CorrectTop / NoAnswerTop — sorted top-1 scores for the answerable cases
	// where the right note ranked first, and for the no-answer cases. The gap
	// between these two distributions is what a defensible query_min_score
	// would have to sit inside; if they overlap, no threshold separates them.
	CorrectTop  []float64
	NoAnswerTop []float64
	// CorrectMargin / NoAnswerMargin — the same two groups, but top1 − top2
	// instead of top1. This is the pair the margin gate has to separate, and
	// unlike the absolute scores it is what the sweep reads.
	CorrectMargin  []float64
	NoAnswerMargin []float64
	P50, P95, Max  time.Duration
}

// TagStat — passed/total for one slice of the fixture.
type TagStat struct{ Passed, Total int }

// Recall1 — fraction of answerable cases whose wanted note ranked first.
func (r Report) Recall1() float64 { return ratio(r.Rank1, r.Answerable) }

// Recall3 — same, in the top three. The daemon asks for daemonTopK candidates
// (mirroring memoryRecallWidth in voice.go's caller), so this is the ceiling a
// better gate or a reranker could reach.
func (r Report) Recall3() float64 { return ratio(r.Rank3, r.Answerable) }

// Answered — fraction of answerable cases the daemon would actually answer
// correctly, gate included. This is the number the operator experiences.
func (r Report) Answered() float64 { return ratio(r.Rank1-r.Gated, r.Answerable) }

// FalseRecallRate — fraction of the no-answer cases the daemon answers anyway.
func (r Report) FalseRecallRate() float64 { return ratio(r.FalseRecall, r.NoAnswer) }

func ratio(n, d int) float64 {
	if d == 0 {
		return 0
	}
	return float64(n) / float64(d)
}

// Score runs every case against a fresh store and aggregates. It never fails
// the run on an embed or search error: an erroring case scores as a miss and is
// counted in Errors, because "the embedder was down" and "the embedder was
// wrong" are different numbers.
func Score(ctx context.Context, name string, emb router.Embedder, newStore NewStore, minScore, minMargin float64, f Fixture) (Report, error) {
	rep := Report{
		Name:      name,
		MinScore:  minScore,
		MinMargin: minMargin,
		Total:     len(f.Cases),
		ByTag:     map[string]TagStat{},
		ByLang:    map[string]TagStat{},
	}
	lat := make([]time.Duration, 0, len(f.Cases))

	for _, c := range f.Cases {
		if c.Answerable() {
			rep.Answerable++
		} else {
			rep.NoAnswer++
		}
		o, err := scoreCase(ctx, emb, newStore, minScore, minMargin, c, f.Filler)
		if err != nil {
			return Report{}, err
		}
		lat = append(lat, o.Latency)

		switch {
		case o.Err != nil:
			rep.Errors++
		case c.Answerable():
			if o.Rank1 {
				rep.Rank1++
			}
			if o.Rank3 {
				rep.Rank3++
			}
			if o.Rank1 && o.Recalled == "" {
				rep.Gated++
			}
			if o.Tied {
				rep.Tied++
			} else if !o.Rank1 {
				rep.WrongTop++
			}
			if o.Rank1 {
				// Margins are collected on rank, not on the gate, so the
				// distribution does not move as the sweep changes the gate.
				rep.CorrectMargin = append(rep.CorrectMargin, o.Margin)
			}
			if o.Rank1 && o.Recalled != "" {
				rep.CorrectTop = append(rep.CorrectTop, o.TopScor)
			}
		default:
			if o.Recalled != "" {
				rep.FalseRecall++
			}
			rep.NoAnswerTop = append(rep.NoAnswerTop, o.TopScor)
			rep.NoAnswerMargin = append(rep.NoAnswerMargin, o.Margin)
		}

		if o.Pass {
			rep.Passed++
		}
		bump(rep.ByLang, c.Lang, o.Pass)
		for _, tag := range c.Tags {
			bump(rep.ByTag, tag, o.Pass)
		}
		rep.Outcomes = append(rep.Outcomes, o)
	}

	sort.Float64s(rep.CorrectTop)
	sort.Float64s(rep.NoAnswerTop)
	sort.Float64s(rep.CorrectMargin)
	sort.Float64s(rep.NoAnswerMargin)
	sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] })
	rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95)
	if len(lat) > 0 {
		rep.Max = lat[len(lat)-1]
	}
	return rep, nil
}

// scoreCase inserts the case's notes into a fresh store, then runs the read
// path the daemon runs. The returned error is fatal (the harness is broken);
// an embedder or store failure on the query lands in Outcome.Err instead.
func scoreCase(ctx context.Context, emb router.Embedder, newStore NewStore, minScore, minMargin float64, c Case, filler []StoredNote) (Outcome, error) {
	st, release, err := newStore()
	if err != nil {
		return Outcome{}, fmt.Errorf("%s: new store: %w", c.ID, err)
	}
	defer release()

	all := append(append([]StoredNote(nil), c.Notes...), filler...)
	for _, n := range all {
		vec, err := router.EmbedPassage(ctx, emb, n.Text)
		if err != nil {
			return Outcome{}, fmt.Errorf("%s: embed note %s: %w", c.ID, n.ID, err)
		}
		meta := map[string]string{"text": n.Text, "type": n.Kind}
		if err := st.Insert(ctx, n.ID, vec, meta); err != nil {
			return Outcome{}, fmt.Errorf("%s: insert %s: %w", c.ID, n.ID, err)
		}
	}

	o := Outcome{Case: c}
	start := time.Now()
	qvec, err := router.EmbedQuery(ctx, emb, c.Query)
	if err != nil {
		o.Latency = time.Since(start)
		o.Err = err
		o.Reasons = []string{fmt.Sprintf("embed query: %v", err)}
		return o, nil
	}
	hits, err := st.Search(ctx, qvec, daemonTopK)
	o.Latency = time.Since(start)
	if err != nil {
		o.Err = err
		o.Reasons = []string{fmt.Sprintf("search: %v", err)}
		return o, nil
	}
	o.Hits = hits

	if len(hits) > 0 {
		o.TopID, o.TopScor = hits[0].ID, hits[0].Score
		if len(hits) > 1 {
			o.Margin = hits[0].Score - hits[1].Score
		}
		o.Recalled = bestRecall(c.Query, hits, minScore, minMargin)
	}
	for i, h := range hits {
		if h.ID != c.Want {
			continue
		}
		o.Rank3 = true
		// A tie is not a hit. With a lexical embedder several notes score
		// exactly 0 against a paraphrased query, and whichever one the sort
		// happens to leave on top would otherwise be credited as recall.
		if i == 0 && (len(hits) < 2 || hits[0].Score > hits[1].Score) {
			o.Rank1 = true
		}
		if i == 0 && !o.Rank1 {
			o.Tied = true
		}
	}

	switch {
	case !c.Answerable():
		if o.Recalled != "" {
			o.Reasons = append(o.Reasons, fmt.Sprintf("false recall: %q at %.3f (margin %.3f), want silence", o.TopID, o.TopScor, o.Margin))
		}
	case o.Tied:
		o.Reasons = append(o.Reasons, fmt.Sprintf("tie at %.3f — the right note is on top only by sort order", o.TopScor))
	case !o.Rank1:
		o.Reasons = append(o.Reasons, fmt.Sprintf("top hit %q (%.3f), want %q%s", o.TopID, o.TopScor, c.Want, rankNote(o.Rank3)))
	case o.Recalled == "":
		o.Reasons = append(o.Reasons, fmt.Sprintf("right note ranked first at %.3f (margin %.3f) but the gate silenced it — daemon says \"не знаю\"", o.TopScor, o.Margin))
	}
	o.Pass = len(o.Reasons) == 0
	return o, nil
}

func rankNote(inTop3 bool) string {
	if inTop3 {
		return " (wanted note is in the top 3)"
	}
	return " (wanted note is not in the top 3)"
}

// bestRecall mirrors cmd/mavend/recall.go — the gate the daemon actually
// applies to a memory hit, including the topic veto added for #470: a score
// that clears the gate still has to be about what he asked. Duplicated rather
// than imported because package main is not importable; recalleval_test.go
// asserts the two agree in behaviour, so a weaker gate here than the daemon
// runs cannot flatter the score. The daemon returns the whole hit (a note and
// a fact are said differently); the harness only scores what came back, so it
// keeps returning the text.
func bestRecall(query string, results []memory.Result, minScore, minMargin float64) string {
	if !memory.Confident(results, minScore, minMargin) {
		return ""
	}
	text := results[0].Meta["text"]
	if !memory.RecallAllowed(query, text,
		router.IsOpenQuestionShaped(query), router.IsLocativeQuestionShaped(query)) {
		return ""
	}
	return text
}

func bump(m map[string]TagStat, key string, pass bool) {
	if key == "" {
		return
	}
	s := m[key]
	s.Total++
	if pass {
		s.Passed++
	}
	m[key] = s
}

// percentile — nearest-rank on a pre-sorted slice. No interpolation: with ~30
// samples an interpolated p95 invents a latency no query actually took.
func percentile(sorted []time.Duration, p float64) time.Duration {
	if len(sorted) == 0 {
		return 0
	}
	i := int(p * float64(len(sorted)))
	if i >= len(sorted) {
		i = len(sorted) - 1
	}
	return sorted[i]
}

// String renders the report in the routing eval's style — headline first, then
// the slices that name where the path is weak.
func (r Report) String() string {
	var b strings.Builder
	fmt.Fprintf(&b, "%s: %d/%d cases pass (gate %.2f, margin %.3f)\n", r.Name, r.Passed, r.Total, r.MinScore, r.MinMargin)
	fmt.Fprintf(&b, "  recall@1 %.1f%% (%d/%d)  recall@3 %.1f%% (%d/%d)  answered after gate %.1f%% (%d/%d)\n",
		100*r.Recall1(), r.Rank1, r.Answerable,
		100*r.Recall3(), r.Rank3, r.Answerable,
		100*r.Answered(), r.Rank1-r.Gated, r.Answerable)
	fmt.Fprintf(&b, "  wrong note on top: %d | tie on top (sort order, not recall): %d | silenced by gate: %d | errors: %d\n",
		r.WrongTop, r.Tied, r.Gated, r.Errors)
	fmt.Fprintf(&b, "  false recall %.1f%% (%d/%d must-be-silent cases answered anyway)\n",
		100*r.FalseRecallRate(), r.FalseRecall, r.NoAnswer)
	fmt.Fprintf(&b, "  top-1 score, right note first: %s\n", spread(r.CorrectTop))
	fmt.Fprintf(&b, "  top-1 score, must be silent:   %s\n", spread(r.NoAnswerTop))
	fmt.Fprintf(&b, "  margin top1-top2, right note first: %s\n", spread(r.CorrectMargin))
	fmt.Fprintf(&b, "  margin top1-top2, must be silent:  %s\n", spread(r.NoAnswerMargin))
	fmt.Fprintf(&b, "  latency: p50 %s  p95 %s  max %s\n", r.P50, r.P95, r.Max)
	fmt.Fprintf(&b, "  by lang: %s\n", renderStats(r.ByLang))
	fmt.Fprintf(&b, "  by tag:  %s\n", renderStats(r.ByTag))
	return b.String()
}

// Failures — per-case detail, sorted by ID so two runs diff cleanly.
func (r Report) Failures() string {
	var b strings.Builder
	out := append([]Outcome(nil), r.Outcomes...)
	sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID })
	for _, o := range out {
		if o.Pass {
			continue
		}
		fmt.Fprintf(&b, "  %s %q: %s\n", o.Case.ID, o.Case.Query, strings.Join(o.Reasons, "; "))
	}
	return b.String()
}

// spread — min / median / max of a sorted score list. Three numbers is enough
// to see whether two distributions overlap, which is the only question a
// threshold can answer.
func spread(sorted []float64) string {
	if len(sorted) == 0 {
		return "n/a"
	}
	return fmt.Sprintf("min %.3f  median %.3f  max %.3f  (n=%d)",
		sorted[0], sorted[len(sorted)/2], sorted[len(sorted)-1], len(sorted))
}

func renderStats(m map[string]TagStat) string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	parts := make([]string, 0, len(keys))
	for _, k := range keys {
		s := m[k]
		parts = append(parts, fmt.Sprintf("%s %d/%d", k, s.Passed, s.Total))
	}
	return strings.Join(parts, "  ")
}