kami
1e47eaca5a
The embedder moved from paraphrase-multilingual-MiniLM-L12-v2 to multilingual-e5-small. Both are 384-dimensional, so nothing in the code noticed: cosine between an old stored vector and a new query vector is noise, and recall degrades silently. So the DB now records the embedder that wrote its vectors. One value for the whole DB (migration #11, a small `meta` key/value table) rather than a column on every vector row: the backfill re-embeds every note and fact in one pass, so a per-row marker would hold the same string in every row and cost a column on two tables for nothing. The identity comes from the embedder itself via a new optional ID() method ("multilingual-e5-small@384", model file name plus dimension), so pointing the config at another model changes the string without anyone editing a constant. mavend logs a loud WARNING at startup naming both the stored and the configured embedder when they differ. Detection only — recall behaviour is unchanged. TODO(#378) in store.CheckEmbedder marks where the backfill will hook in. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
66 lines
2.3 KiB
Go
66 lines
2.3 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
)
|
|
|
|
// metaKeyEmbedderID names the embedder that wrote the stored vectors.
|
|
//
|
|
// Why one value for the whole DB and not a column on every vector row: the
|
|
// vectors are only ever rewritten all at once (one backfill re-embeds every
|
|
// note and fact together), so a per-row marker would hold the same string in
|
|
// every row and cost a column on two tables for nothing.
|
|
const metaKeyEmbedderID = "embedder_id"
|
|
|
|
// Meta reads a single value from the meta table. Missing key ⇒ empty string.
|
|
func (s *Store) Meta(ctx context.Context, key string) (string, error) {
|
|
var v string
|
|
err := s.db.QueryRowContext(ctx, `SELECT value FROM meta WHERE key = ?`, key).Scan(&v)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return "", nil
|
|
}
|
|
if err != nil {
|
|
return "", fmt.Errorf("read meta %s: %w", key, err)
|
|
}
|
|
return v, nil
|
|
}
|
|
|
|
// SetMeta writes (or overwrites) a single meta value.
|
|
func (s *Store) SetMeta(ctx context.Context, key, value string) error {
|
|
_, err := s.db.ExecContext(ctx,
|
|
`INSERT INTO meta (key, value) VALUES (?,?)
|
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value`, key, value)
|
|
if err != nil {
|
|
return fmt.Errorf("write meta %s: %w", key, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CheckEmbedder compares the embedder now configured against the one that
|
|
// wrote the stored vectors.
|
|
//
|
|
// A DB that has never recorded one is claimed for the current embedder: either
|
|
// it is fresh (nothing stored yet, nothing to fix) or it predates this marker.
|
|
// Returns the stored id and whether it differs from the current one.
|
|
//
|
|
// Vectors from two different models live in different spaces, so cosine
|
|
// between them is noise rather than a low score — and both of our models are
|
|
// 384-dimensional, so nothing else catches it.
|
|
//
|
|
// TODO(#378): on a mismatch, run the one-shot backfill here — re-embed every
|
|
// stored note and fact text with the current embedder (EmbedPassage side),
|
|
// write the vectors back, then SetMeta the current id.
|
|
func (s *Store) CheckEmbedder(ctx context.Context, currentID string) (stored string, mismatch bool, err error) {
|
|
stored, err = s.Meta(ctx, metaKeyEmbedderID)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
if stored == "" {
|
|
return currentID, false, s.SetMeta(ctx, metaKeyEmbedderID, currentID)
|
|
}
|
|
return stored, stored != currentID, nil
|
|
}
|