Warn about vectors written before the marker existed

This commit is contained in:
kami
2026-07-31 13:44:29 +04:00
parent 7bb9f9be06
commit 4282f6b9a9
2 changed files with 73 additions and 8 deletions
+40 -8
View File
@@ -39,17 +39,27 @@ func (s *Store) SetMeta(ctx context.Context, key, value string) error {
return nil
}
// EmbedderUnknown is the stored id reported for a DB that already holds
// vectors but never recorded who wrote them.
const EmbedderUnknown = "unknown (written before this marker existed)"
// CheckEmbedder compares the embedder now configured against the one that
// wrote the stored vectors.
//
// A DB that has never recorded one is claimed for the current embedder: either
// it is fresh (nothing stored yet, nothing to fix) or it predates this marker.
// Returns the stored id and whether it differs from the current one.
// wrote the stored vectors. Returns the stored id and whether it differs.
//
// Vectors from two different models live in different spaces, so cosine
// between them is noise rather than a low score — and both of our models are
// 384-dimensional, so nothing else catches it.
//
// Three cases, and the middle one is the one that actually matters:
//
// - marker present ⇒ compare the two ids.
// - marker absent but vectors already stored ⇒ this is a DB from before the
// marker, so we cannot know who wrote them. Report a mismatch. This is the
// real case on the deployed box: those vectors came from the old embedder,
// and claiming them for the current one would hide the exact problem the
// marker was added to catch.
// - marker absent and no vectors ⇒ fresh DB, claim it, nothing to fix.
//
// TODO(#378): on a mismatch, run the one-shot backfill here — re-embed every
// stored note and fact text with the current embedder (EmbedPassage side),
// write the vectors back, then SetMeta the current id.
@@ -58,8 +68,30 @@ func (s *Store) CheckEmbedder(ctx context.Context, currentID string) (stored str
if err != nil {
return "", false, err
}
if stored == "" {
return currentID, false, s.SetMeta(ctx, metaKeyEmbedderID, currentID)
if stored != "" {
return stored, stored != currentID, nil
}
return stored, stored != currentID, nil
n, err := s.countVectors(ctx)
if err != nil {
return "", false, err
}
if n > 0 {
return EmbedderUnknown, true, nil
}
return currentID, false, s.SetMeta(ctx, metaKeyEmbedderID, currentID)
}
// countVectors — how many stored rows carry an embedding. Used only to tell a
// fresh DB apart from one that predates the marker.
func (s *Store) countVectors(ctx context.Context) (int, error) {
var notes, vecs int
if err := s.db.QueryRowContext(ctx,
`SELECT count(*) FROM notes WHERE embedding IS NOT NULL`).Scan(&notes); err != nil {
return 0, fmt.Errorf("count note vectors: %w", err)
}
if err := s.db.QueryRowContext(ctx,
`SELECT count(*) FROM memory_vectors`).Scan(&vecs); err != nil {
return 0, fmt.Errorf("count memory vectors: %w", err)
}
return notes + vecs, nil
}