Maven/internal/store/memory.go

package store

import (
	"context"
	"database/sql"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"math"
	"sort"
	"strings"
	"time"

	"github.com/kami/maven/internal/memory"
)

// MemoryStore is the persistent backend for internal/memory's vector Store,
// sharing the main encrypted sqlite database so recall text (note/fact bodies
// carried in the meta blob) inherits at-rest encryption — a plaintext sidecar
// file would undercut store.OpenEncrypted. It survives daemon restarts, which
// the InMemoryStore does not: that was the last gap keeping long-term memory
// from being real.
//
// Search is brute-force cosine over every row loaded into memory — the same
// algorithm as InMemoryStore, just sourced from disk. At the single-user note+
// fact scale (thousands of rows, not millions) a full scan per query is well
// under a millisecond; an ANN index is the swap for later, behind this same
// interface. Vectors are assumed L2-normalized by the embedder, so cosine is a
// dot product.
type MemoryStore struct {
	db *sql.DB
}

// VectorMemory returns a persistent memory.Store backed by this store's db.
// The returned store shares the db handle (single writer — the daemon), so it
// participates in the same encrypted tmpfs working copy and is sealed on Close.
func (s *Store) VectorMemory() *MemoryStore {
	return &MemoryStore{db: s.db}
}

// compile-time check: MemoryStore satisfies the memory.Store interface, and the
// wider Catalog that speaker profiles need (enumerate by prefix, delete by id).
var _ memory.Store = (*MemoryStore)(nil)
var _ memory.Catalog = (*MemoryStore)(nil)

// Insert upserts a vector by id: a repeated id replaces the prior row rather
// than accumulating duplicates (the note/fact ids are stable and unique, so a
// re-index is an update, not a second copy — an improvement on InMemoryStore's
// append-always). meta is stored as a JSON object.
func (m *MemoryStore) Insert(ctx context.Context, id string, vec []float32, meta map[string]string) error {
	metaJSON, err := json.Marshal(meta)
	if err != nil {
		return fmt.Errorf("memory: marshal meta: %w", err)
	}
	_, err = m.db.ExecContext(ctx,
		`INSERT INTO memory_vectors (id, vec, meta, created_ts) VALUES (?,?,?,?)
		 ON CONFLICT(id) DO UPDATE SET vec = excluded.vec, meta = excluded.meta, created_ts = excluded.created_ts`,
		id, encodeVec(vec), string(metaJSON), time.Now().UnixMilli())
	if err != nil {
		return fmt.Errorf("memory: insert %q: %w", id, err)
	}
	return nil
}

// Search returns the topK nearest rows by cosine similarity. A full scan; see
// the type doc for why that's fine at this scale.
//
// Rows under memory.NonRecallPrefix are excluded in SQL. They are speaker
// voiceprints sharing this table, and note recall must not rank them; see that
// constant for why the previous arrangement only appeared to do this.
func (m *MemoryStore) Search(ctx context.Context, vec []float32, topK int) ([]memory.Result, error) {
	if topK <= 0 {
		topK = 10
	}
	rows, err := m.db.QueryContext(ctx,
		`SELECT id, vec, meta FROM memory_vectors WHERE id NOT LIKE ? ESCAPE '\'`,
		escapeLike(memory.NonRecallPrefix)+"%")
	if err != nil {
		return nil, fmt.Errorf("memory: scan: %w", err)
	}
	defer rows.Close()

	var out []memory.Result
	for rows.Next() {
		var id, metaJSON string
		var blob []byte
		if err := rows.Scan(&id, &blob, &metaJSON); err != nil {
			return nil, fmt.Errorf("memory: row: %w", err)
		}
		meta := map[string]string{}
		if err := json.Unmarshal([]byte(metaJSON), &meta); err != nil {
			return nil, fmt.Errorf("memory: unmarshal meta for %q: %w", id, err)
		}
		out = append(out, memory.Result{ID: id, Score: dot(vec, decodeVec(blob)), Meta: meta})
	}
	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("memory: rows: %w", err)
	}

	sort.Slice(out, func(i, j int) bool { return out[i].Score > out[j].Score })
	if topK < len(out) {
		out = out[:topK]
	}
	return out, nil
}

// ByPrefix returns every row whose id starts with prefix, vectors included.
//
// This is not a similarity query and deliberately does not score anything:
// listing the enrolled voices is a question about which rows exist, and asking
// it through Search would mean inventing a query vector to rank them by. The
// prefix is matched with LIKE against an escaped pattern, so a profile id
// containing % or _ cannot widen the match.
func (m *MemoryStore) ByPrefix(ctx context.Context, prefix string) ([]memory.Record, error) {
	pattern := escapeLike(prefix) + "%"
	rows, err := m.db.QueryContext(ctx,
		`SELECT id, vec, meta FROM memory_vectors WHERE id LIKE ? ESCAPE '\'`, pattern)
	if err != nil {
		return nil, fmt.Errorf("memory: by prefix %q: %w", prefix, err)
	}
	defer rows.Close()

	var out []memory.Record
	for rows.Next() {
		var id, metaJSON string
		var blob []byte
		if err := rows.Scan(&id, &blob, &metaJSON); err != nil {
			return nil, fmt.Errorf("memory: row: %w", err)
		}
		meta := map[string]string{}
		if err := json.Unmarshal([]byte(metaJSON), &meta); err != nil {
			return nil, fmt.Errorf("memory: unmarshal meta for %q: %w", id, err)
		}
		out = append(out, memory.Record{ID: id, Vec: decodeVec(blob), Meta: meta})
	}
	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("memory: rows: %w", err)
	}
	return out, nil
}

// Delete removes one vector by id. A row that is not there is not an error —
// "forget this voice" is satisfied either way.
func (m *MemoryStore) Delete(ctx context.Context, id string) error {
	if _, err := m.db.ExecContext(ctx, `DELETE FROM memory_vectors WHERE id = ?`, id); err != nil {
		return fmt.Errorf("memory: delete %q: %w", id, err)
	}
	return nil
}

// escapeLike neutralises the LIKE wildcards in a literal prefix.
func escapeLike(s string) string {
	r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
	return r.Replace(s)
}

// encodeVec serializes a float32 slice as little-endian IEEE-754 bytes (4 bytes
// per element) for the BLOB column.
func encodeVec(v []float32) []byte {
	b := make([]byte, 4*len(v))
	for i, f := range v {
		binary.LittleEndian.PutUint32(b[4*i:], math.Float32bits(f))
	}
	return b
}

// decodeVec reverses encodeVec. A blob whose length isn't a multiple of 4 is
// truncated to the whole-element prefix (defensive — a well-formed row can't
// produce that).
func decodeVec(b []byte) []float32 {
	n := len(b) / 4
	v := make([]float32, n)
	for i := 0; i < n; i++ {
		v[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[4*i:]))
	}
	return v
}

// dot is the cosine similarity for L2-normalized vectors (mismatched lengths ⇒
// 0, matching internal/memory's cosine).
func dot(a, b []float32) float64 {
	if len(a) != len(b) || len(a) == 0 {
		return 0
	}
	var sum float64
	for i := range a {
		sum += float64(a[i]) * float64(b[i])
	}
	return sum
}