package store import ( "context" "database/sql" "errors" "fmt" ) // metaKeyEmbedderID names the embedder that wrote the stored vectors. // // Why one value for the whole DB and not a column on every vector row: the // vectors are only ever rewritten all at once (one backfill re-embeds every // note and fact together), so a per-row marker would hold the same string in // every row and cost a column on two tables for nothing. const metaKeyEmbedderID = "embedder_id" // Meta reads a single value from the meta table. Missing key ⇒ empty string. func (s *Store) Meta(ctx context.Context, key string) (string, error) { var v string err := s.db.QueryRowContext(ctx, `SELECT value FROM meta WHERE key = ?`, key).Scan(&v) if errors.Is(err, sql.ErrNoRows) { return "", nil } if err != nil { return "", fmt.Errorf("read meta %s: %w", key, err) } return v, nil } // SetMeta writes (or overwrites) a single meta value. func (s *Store) SetMeta(ctx context.Context, key, value string) error { _, err := s.db.ExecContext(ctx, `INSERT INTO meta (key, value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, key, value) if err != nil { return fmt.Errorf("write meta %s: %w", key, err) } return nil } // EmbedderUnknown is the stored id reported for a DB that already holds // vectors but never recorded who wrote them. const EmbedderUnknown = "unknown (written before this marker existed)" // CheckEmbedder compares the embedder now configured against the one that // wrote the stored vectors. Returns the stored id and whether it differs. // // Vectors from two different models live in different spaces, so cosine // between them is noise rather than a low score — and both of our models are // 384-dimensional, so nothing else catches it. // // Three cases, and the middle one is the one that actually matters: // // - marker present ⇒ compare the two ids. // - marker absent but vectors already stored ⇒ this is a DB from before the // marker, so we cannot know who wrote them. Report a mismatch. This is the // real case on the deployed box: those vectors came from the old embedder, // and claiming them for the current one would hide the exact problem the // marker was added to catch. // - marker absent and no vectors ⇒ fresh DB, claim it, nothing to fix. // // On a mismatch the fix is ReembedAll (backfill.go), run explicitly with // `mavend -reembed`. Nothing is re-embedded here: that work is minutes of CPU // on the laptop and must not stall a normal start. func (s *Store) CheckEmbedder(ctx context.Context, currentID string) (stored string, mismatch bool, err error) { stored, err = s.Meta(ctx, metaKeyEmbedderID) if err != nil { return "", false, err } if stored != "" { return stored, stored != currentID, nil } n, err := s.countVectors(ctx) if err != nil { return "", false, err } if n > 0 { return EmbedderUnknown, true, nil } return currentID, false, s.SetMeta(ctx, metaKeyEmbedderID, currentID) } // countVectors — how many stored rows carry an embedding. Used only to tell a // fresh DB apart from one that predates the marker. func (s *Store) countVectors(ctx context.Context) (int, error) { var notes, vecs int if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM notes WHERE embedding IS NOT NULL`).Scan(¬es); err != nil { return 0, fmt.Errorf("count note vectors: %w", err) } if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM memory_vectors`).Scan(&vecs); err != nil { return 0, fmt.Errorf("count memory vectors: %w", err) } return notes + vecs, nil }