1e47eaca5a
The embedder moved from paraphrase-multilingual-MiniLM-L12-v2 to multilingual-e5-small. Both are 384-dimensional, so nothing in the code noticed: cosine between an old stored vector and a new query vector is noise, and recall degrades silently. So the DB now records the embedder that wrote its vectors. One value for the whole DB (migration #11, a small `meta` key/value table) rather than a column on every vector row: the backfill re-embeds every note and fact in one pass, so a per-row marker would hold the same string in every row and cost a column on two tables for nothing. The identity comes from the embedder itself via a new optional ID() method ("multilingual-e5-small@384", model file name plus dimension), so pointing the config at another model changes the string without anyone editing a constant. mavend logs a loud WARNING at startup naming both the stored and the configured embedder when they differ. Detection only — recall behaviour is unchanged. TODO(#378) in store.CheckEmbedder marks where the backfill will hook in. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
122 lines
5.3 KiB
Go
122 lines
5.3 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
)
|
|
|
|
// migrations are ordered, forward-only schema steps applied after schema.sql.
|
|
// Index i (1-based) is the user_version the step at migrations[i-1] brings the
|
|
// DB TO; there is no step 0 — schema.sql is the idempotent baseline (version 0).
|
|
// An empty slice is a clean no-op that leaves user_version at 0.
|
|
//
|
|
// To add migration #1 (e.g. the sqlcipher rekey), append its SQL:
|
|
//
|
|
// var migrations = []string{
|
|
// `ALTER TABLE ...;`, // #1
|
|
// }
|
|
var migrations = []string{
|
|
`ALTER TABLE tools ADD COLUMN scope TEXT NOT NULL DEFAULT 'homelab';`, // #1
|
|
`ALTER TABLE reminders ADD COLUMN cron TEXT;
|
|
ALTER TABLE reminders ADD COLUMN next_fire_ts INTEGER;`, // #2
|
|
`CREATE TABLE memory_vectors (
|
|
id TEXT PRIMARY KEY,
|
|
vec BLOB NOT NULL,
|
|
meta TEXT NOT NULL DEFAULT '{}',
|
|
created_ts INTEGER NOT NULL
|
|
);`, // #3 — long-term vector memory (persistent backend for internal/memory)
|
|
`CREATE TABLE IF NOT EXISTS events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
fact_id INTEGER NOT NULL REFERENCES facts(id),
|
|
action TEXT NOT NULL,
|
|
object TEXT NOT NULL,
|
|
ts INTEGER NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_events_action_object ON events (action, object, ts DESC);
|
|
CREATE TABLE IF NOT EXISTS proposed_routines (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
action TEXT NOT NULL,
|
|
object TEXT NOT NULL,
|
|
interval_days REAL NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'proposed' CHECK (status IN ('proposed','accepted','dismissed')),
|
|
created_ts INTEGER NOT NULL,
|
|
reminder_id INTEGER REFERENCES reminders(id),
|
|
UNIQUE(action, object)
|
|
);`, // #4 — event extraction + pattern inference
|
|
|
|
`CREATE TABLE IF NOT EXISTS ack_sends (
|
|
rule TEXT NOT NULL,
|
|
sent_at INTEGER NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_ack_sends_rule ON ack_sends (rule, sent_at DESC);`, // #5 — sev4 telegram repeat-til-ack tracking
|
|
|
|
`CREATE TABLE IF NOT EXISTS delivery_attempts (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
kind TEXT NOT NULL CHECK (kind IN ('nudge','reminder')),
|
|
rule TEXT NOT NULL DEFAULT '',
|
|
reminder_id INTEGER NOT NULL DEFAULT 0,
|
|
channel TEXT NOT NULL,
|
|
body_hash TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','sent','failed','unknown')),
|
|
created_ts INTEGER NOT NULL,
|
|
completed_ts INTEGER
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_delivery_attempts_status ON delivery_attempts (status);`, // #6 — durable delivery outbox
|
|
|
|
`ALTER TABLE facts ADD COLUMN subject TEXT NOT NULL DEFAULT '';
|
|
ALTER TABLE facts ADD COLUMN entity_id TEXT;
|
|
ALTER TABLE facts ADD COLUMN resolution_state TEXT NOT NULL DEFAULT 'none'
|
|
CHECK (resolution_state IN ('none','pending','resolved','ambiguous','not_found'));
|
|
CREATE INDEX IF NOT EXISTS idx_facts_entity_id ON facts (entity_id) WHERE entity_id IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS idx_facts_resolution_pending ON facts (resolution_state) WHERE resolution_state = 'pending';`, // #7 — entity-aware memory (Vikunja #279): facts about a subject get resolved to a Nexus entity_id async
|
|
|
|
`CREATE INDEX IF NOT EXISTS idx_nudges_snoozed ON nudges (outcome_ts) WHERE outcome = 'snoozed';`, // #8 — SnoozedUntil runs every tick; keep it off a full scan (Vikunja #364)
|
|
`ALTER TABLE proposed_routines ADD COLUMN accepted_ts INTEGER;
|
|
ALTER TABLE proposed_routines ADD COLUMN last_fired_ts INTEGER;`, // #9 — accepted routines keep firing (Vikunja #366): the tick loop needs to know when a routine was accepted and when it last nudged
|
|
|
|
`CREATE TABLE IF NOT EXISTS dialogue_sessions (
|
|
id TEXT PRIMARY KEY,
|
|
data BLOB NOT NULL,
|
|
ts INTEGER NOT NULL,
|
|
ttl_ms INTEGER NOT NULL,
|
|
expires_ts INTEGER NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_dialogue_sessions_expires ON dialogue_sessions (expires_ts);`, // #10 — the follow-up session survives a restart (Vikunja #363); small, TTL-pruned table, not a history log
|
|
|
|
`CREATE TABLE IF NOT EXISTS meta (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL
|
|
);`, // #11 — small key/value table for facts about the DB itself; first key is embedder_id (Vikunja #378)
|
|
}
|
|
|
|
// migrate applies every migration with a number greater than the DB's current
|
|
// user_version, each in its own transaction that also bumps user_version. Fails
|
|
// closed: the first erroring step aborts and leaves prior steps committed.
|
|
func migrate(ctx context.Context, db *sql.DB) error {
|
|
var current int
|
|
if err := db.QueryRowContext(ctx, "PRAGMA user_version").Scan(¤t); err != nil {
|
|
return fmt.Errorf("read user_version: %w", err)
|
|
}
|
|
for i := current; i < len(migrations); i++ {
|
|
version := i + 1 // 1-based: migrations[i] brings DB to `version`
|
|
tx, err := db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("migration %d begin: %w", version, err)
|
|
}
|
|
if _, err := tx.ExecContext(ctx, migrations[i]); err != nil {
|
|
_ = tx.Rollback()
|
|
return fmt.Errorf("migration %d: %w", version, err)
|
|
}
|
|
// PRAGMA user_version can't be parameterized; version is our own int.
|
|
if _, err := tx.ExecContext(ctx, fmt.Sprintf("PRAGMA user_version = %d", version)); err != nil {
|
|
_ = tx.Rollback()
|
|
return fmt.Errorf("migration %d bump: %w", version, err)
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("migration %d commit: %w", version, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|