From 3de1cfb4ca8fc46bb4dddddac1199d5ee0925eec Mon Sep 17 00:00:00 2001 From: kami Date: Thu, 30 Jul 2026 23:43:46 +0400 Subject: [PATCH] fix: dedup claims and make the unique constraint NULLS NOT DISTINCT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit claims declared UNIQUE (..., source, user_id). Every objective claim has user_id IS NULL, and under default NULLS DISTINCT semantics Postgres treats those rows as unique, so the ON CONFLICT DO UPDATE / DO NOTHING clauses in db.service and mb-spine-writer never fired. Re-enrichment inserted a fresh duplicate every run instead of reinforcing. Live data: 236 duplicate groups, 2110 excess rows, worst single claim 86 copies, ~15% of 13,910 claims. claim_fusion is SUM(trust * confidence * recency), so one edge could carry 86x its intended weight — the likely cause of repetitive recommendations, and almost certainly the root of d497588 (claim_fusion MV duplicate-key failure). Migration 20260730_claims_dedup_nulls_not_distinct, two phases in one transaction. Dedup MUST precede the constraint or adding it fails. Phase 1 collapses each group into its most recently reinforced row, carrying forward MAX(last_reinforced_at), MAX(evidence_at) and MAX(confidence) — reinforcement recency would otherwise be lost by simply deleting extras. The MAX(...) OVER grp and ROW_NUMBER() OVER ordered windows are deliberately separate: an ORDER BY inside the window makes the default frame UNBOUNDED PRECEDING TO CURRENT ROW, which turns MAX() into a running maximum and would silently keep the wrong confidence. Phase 2 drops the old constraint by matching its definition rather than its name, because the live DB has drifted and its autogenerated name is truncated at 63 characters. Verified on a scratch PG16 seeded with the old schema plus a 3-row duplicate group, a distinct-source singleton and a real-user_id row: UPDATE 3 / DELETE 2, keeper retained the group max of each field from three different rows, re-run is a no-op, and a subsequent ON CONFLICT DO UPDATE with user_id = NULL fires correctly. REVIEW-2026-07-30.md finding 4. Co-Authored-By: Claude Opus 5 --- backend/src/db/schema.sql | 8 ++- backend/src/services/db.service.ts | 84 ++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/backend/src/db/schema.sql b/backend/src/db/schema.sql index dd49a26..0a40563 100644 --- a/backend/src/db/schema.sql +++ b/backend/src/db/schema.sql @@ -340,7 +340,13 @@ CREATE TABLE IF NOT EXISTS claims ( last_reinforced_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), raw JSONB, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - UNIQUE (subject_type, subject_id, predicate, object_type, object_id, source, user_id) + -- NULLS NOT DISTINCT (PG15+) is load-bearing: user_id is NULL for every + -- objective claim, and with default NULLS DISTINCT semantics the + -- ON CONFLICT clauses in upsertClaim() / MbSpineWriter never fire, so + -- re-enrichment inserts duplicates instead of reinforcing. + CONSTRAINT claims_edge_source_user_key + UNIQUE NULLS NOT DISTINCT + (subject_type, subject_id, predicate, object_type, object_id, source, user_id) ); CREATE INDEX IF NOT EXISTS idx_claims_subject ON claims (subject_type, subject_id, predicate); diff --git a/backend/src/services/db.service.ts b/backend/src/services/db.service.ts index f7b2c3f..0da2109 100644 --- a/backend/src/services/db.service.ts +++ b/backend/src/services/db.service.ts @@ -504,6 +504,90 @@ const MIGRATIONS: { id: string; sql: string }[] = [ JOIN artists a ON a.id = cf.object_id; `, }, + { + id: '20260730_claims_dedup_nulls_not_distinct', + sql: ` + -- The claims uniqueness constraint was declared as a plain + -- UNIQUE (subject_type, subject_id, predicate, object_type, object_id, + -- source, user_id). Every objective claim (MB, Discogs, tags) has + -- user_id IS NULL, and Postgres treats NULLs as distinct, so none of the + -- ON CONFLICT clauses in upsertClaim() / MbSpineWriter ever fired: + -- re-enrichment inserted a fresh duplicate row every time instead of + -- reinforcing. claim_fusion is SUM(trust * confidence * recency), so a + -- duplicated edge carried N times its intended weight. + -- + -- Two phases, in this order (the constraint cannot be added while + -- duplicates exist): + -- 1. collapse each duplicate group into its most recently reinforced + -- row, carrying forward MAX(last_reinforced_at) / MAX(evidence_at) / + -- MAX(confidence) so reinforcement recency is not lost; + -- 2. replace the constraint with a NULLS NOT DISTINCT version (PG15+). + + -- Phase 1: dedup. + CREATE TEMP TABLE claims_dedup ON COMMIT DROP AS + SELECT + id, + ROW_NUMBER() OVER ordered AS rn, + -- These MUST use the unordered window: an ORDER BY in the window spec + -- makes the default frame "UNBOUNDED PRECEDING TO CURRENT ROW", turning + -- MAX() into a running maximum rather than a per-group one. + MAX(last_reinforced_at) OVER grp AS max_last_reinforced_at, + MAX(evidence_at) OVER grp AS max_evidence_at, + MAX(confidence) OVER grp AS max_confidence + FROM claims + WINDOW + grp AS ( + PARTITION BY subject_type, subject_id, predicate, object_type, object_id, + source, + COALESCE(user_id, '00000000-0000-0000-0000-000000000000'::uuid) + ), + ordered AS ( + grp ORDER BY last_reinforced_at DESC, evidence_at DESC, id + ); + + -- Keeper of each group absorbs the group's best values. + UPDATE claims c + SET last_reinforced_at = d.max_last_reinforced_at, + evidence_at = d.max_evidence_at, + confidence = d.max_confidence + FROM claims_dedup d + WHERE c.id = d.id + AND d.rn = 1; + + DELETE FROM claims c + USING claims_dedup d + WHERE c.id = d.id + AND d.rn > 1; + + -- Phase 2: replace the constraint. The live DB has drifted from + -- schema.sql, so find the existing constraint by its definition rather + -- than assuming Postgres' auto-generated name. + DO $mig$ + DECLARE + cname TEXT; + BEGIN + FOR cname IN + SELECT con.conname + FROM pg_constraint con + WHERE con.conrelid = 'claims'::regclass + AND con.contype = 'u' + AND pg_get_constraintdef(con.oid) LIKE '%subject_type%' + AND pg_get_constraintdef(con.oid) NOT LIKE '%NULLS NOT DISTINCT%' + LOOP + EXECUTE format('ALTER TABLE claims DROP CONSTRAINT %I', cname); + END LOOP; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'claims'::regclass AND conname = 'claims_edge_source_user_key' + ) THEN + ALTER TABLE claims ADD CONSTRAINT claims_edge_source_user_key + UNIQUE NULLS NOT DISTINCT + (subject_type, subject_id, predicate, object_type, object_id, source, user_id); + END IF; + END $mig$; + `, + }, ]; export class DbService {