feat: enhance discovery, vibe sessions, and library enrichment
Typecheck / typecheck (backend) (push) Has been cancelled
Typecheck / typecheck (workers) (push) Has been cancelled

This commit is contained in:
kami
2026-08-01 14:40:48 +04:00
parent a0c9f42a89
commit 4c48d11e9d
54 changed files with 4136 additions and 521 deletions
+1 -1
View File
@@ -160,7 +160,7 @@ export async function buildApp(config: AppConfig) {
const sessionDirector = new SessionDirector(dbService);
fastify.register(v2Routes, { prefix: '/api', dbService, sessionDirector });
fastify.register(discoveryRoutes, { prefix: '/api', dbService });
fastify.register(discoveryRoutes, { prefix: '/api', dbService, jobService });
// ponytail: /api/test/enqueue-job (manual job-enqueue test endpoint) removed —
// nothing in the deployed app or its tests called it, and deployment never
// sets NODE_ENV so an env gate would've stayed live in prod anyway.
+22
View File
@@ -0,0 +1,22 @@
import { describe, expect, it } from 'vitest';
import { MIGRATIONS } from './migrations.js';
describe('track release-date migration', () => {
const migration = MIGRATIONS.find(
({ id }) => id === '20260801_track_release_dates_from_albums',
);
it('backfills from albums with a null-safe, deterministic conflict rule', () => {
expect(migration).toBeDefined();
expect(migration!.sql).toContain('ALTER TABLE albums ADD COLUMN IF NOT EXISTS release_date DATE');
expect(migration!.sql).toContain('UPDATE tracks AS t');
expect(migration!.sql).toContain('SET release_date = al.release_date');
expect(migration!.sql).toContain('t.release_date IS DISTINCT FROM al.release_date');
});
it('keeps new tracks and album metadata updates synchronized', () => {
expect(migration!.sql).toContain('BEFORE INSERT OR UPDATE OF album_id, release_date ON tracks');
expect(migration!.sql).toContain('AFTER UPDATE OF release_date ON albums');
expect(migration!.sql).toContain('WHEN (OLD.release_date IS DISTINCT FROM NEW.release_date)');
});
});
+134
View File
@@ -510,4 +510,138 @@ export const MIGRATIONS: Migration[] = [
ON pending_file_deletions(requested_at);
`,
},
{
// System E originally wrote graph_exploration claims without registering it
// in source_trust, so the FK rejected every graph walk. It also lacked a
// durable link from an acquired candidate to the track the scanner created.
id: '20260801_acquisition_pipeline',
sql: `
INSERT INTO source_trust (key, trust, description)
VALUES ('graph_exploration', 0.40,
'Muzick graph traversal provenance for discovery candidates.')
ON CONFLICT (key) DO NOTHING;
ALTER TABLE discovery_candidates
ADD COLUMN IF NOT EXISTS acquired_track_id UUID
REFERENCES tracks(id) ON DELETE SET NULL;
ALTER TABLE discovery_candidates
ADD COLUMN IF NOT EXISTS acquired_at TIMESTAMPTZ;
ALTER TABLE discovery_candidates
ADD COLUMN IF NOT EXISTS acquisition_attempts INTEGER NOT NULL DEFAULT 0;
ALTER TABLE discovery_candidates
ADD COLUMN IF NOT EXISTS last_error TEXT;
CREATE INDEX IF NOT EXISTS idx_discovery_candidates_status
ON discovery_candidates (status, first_seen_at);
`,
},
{
// Separate portrait fetching from structural metadata so an operator can
// re-enrich missing artist images without implicitly enabling MB writes.
id: '20260801_artist_image_enrichment_toggle',
sql: `
INSERT INTO settings (key, value)
VALUES ('enrich_artist_images', 'false')
ON CONFLICT (key) DO NOTHING;
`,
},
{
// Audio analysis v1 stored an arbitrary linear energy scale, which made
// nearly every mastered track look maximally energetic. Track the analysis
// contract and source hash so bounded background jobs can safely refresh
// stale results without repeatedly decoding unchanged files.
id: '20260801_audio_analysis_v2',
sql: `
ALTER TABLE track_audio_features
ADD COLUMN IF NOT EXISTS analysis_version SMALLINT NOT NULL DEFAULT 0;
ALTER TABLE track_audio_features
ADD COLUMN IF NOT EXISTS source_hash TEXT;
ALTER TABLE track_audio_features
ADD COLUMN IF NOT EXISTS analyzed_at TIMESTAMPTZ;
DO $mig$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conrelid = 'track_audio_features'::regclass
AND conname = 'track_audio_features_bpm_range'
) THEN
ALTER TABLE track_audio_features
ADD CONSTRAINT track_audio_features_bpm_range
CHECK (bpm IS NULL OR (bpm >= 30 AND bpm <= 300)) NOT VALID;
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conrelid = 'track_audio_features'::regclass
AND conname = 'track_audio_features_energy_range'
) THEN
ALTER TABLE track_audio_features
ADD CONSTRAINT track_audio_features_energy_range
CHECK (energy IS NULL OR (energy >= 0 AND energy <= 1)) NOT VALID;
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_constraint
WHERE conrelid = 'track_audio_features'::regclass
AND conname = 'track_audio_features_danceability_range'
) THEN
ALTER TABLE track_audio_features
ADD CONSTRAINT track_audio_features_danceability_range
CHECK (danceability IS NULL OR (danceability >= 0 AND danceability <= 1)) NOT VALID;
END IF;
END $mig$;
`,
},
{
// tracks.release_date is a denormalised copy of the canonical
// MusicBrainz release-group date on albums. Before this migration the
// worker populated albums only, leaving every historical track invisible
// to the novelty generator. The backfill and triggers make the canonical
// album value win deterministically for both old and future rows.
id: '20260801_track_release_dates_from_albums',
sql: `
-- The worker used to create this column lazily, but migration execution
-- must not rely on worker startup order.
ALTER TABLE albums ADD COLUMN IF NOT EXISTS release_date DATE;
-- Backfill only mismatches. IS DISTINCT FROM is null-safe and makes the
-- statement safe to rerun manually for diagnostics/recovery.
UPDATE tracks AS t
SET release_date = al.release_date
FROM albums AS al
WHERE t.album_id = al.id
AND t.release_date IS DISTINCT FROM al.release_date;
CREATE OR REPLACE FUNCTION sync_track_release_date_from_album()
RETURNS TRIGGER AS $mig$
BEGIN
SELECT release_date INTO NEW.release_date
FROM albums
WHERE id = NEW.album_id;
RETURN NEW;
END;
$mig$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS trg_tracks_sync_release_date ON tracks;
CREATE TRIGGER trg_tracks_sync_release_date
BEFORE INSERT OR UPDATE OF album_id, release_date ON tracks
FOR EACH ROW EXECUTE FUNCTION sync_track_release_date_from_album();
CREATE OR REPLACE FUNCTION propagate_album_release_date_to_tracks()
RETURNS TRIGGER AS $mig$
BEGIN
UPDATE tracks
SET release_date = NEW.release_date
WHERE album_id = NEW.id
AND release_date IS DISTINCT FROM NEW.release_date;
RETURN NEW;
END;
$mig$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS trg_albums_propagate_release_date ON albums;
CREATE TRIGGER trg_albums_propagate_release_date
AFTER UPDATE OF release_date ON albums
FOR EACH ROW
WHEN (OLD.release_date IS DISTINCT FROM NEW.release_date)
EXECUTE FUNCTION propagate_album_release_date_to_tracks();
`,
},
];
+81 -1
View File
@@ -130,6 +130,17 @@ CREATE TABLE IF NOT EXISTS tracks (
-- Add normalized columns as generated columns for existing databases where the
-- CREATE TABLE IF NOT EXISTS above was a no-op (column didn't exist before).
-- `albums.release_date` was introduced after the original albums table. It
-- must exist before the track synchronisation trigger below is compiled.
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'albums' AND column_name = 'release_date'
) THEN
ALTER TABLE albums ADD COLUMN release_date DATE;
END IF;
END $$;
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
@@ -140,6 +151,49 @@ DO $$ BEGIN
END $$;
CREATE INDEX IF NOT EXISTS idx_tracks_release_date ON tracks (release_date) WHERE release_date IS NOT NULL;
-- An album's MusicBrainz first-release-date is the canonical date for every
-- track on that album. Keep the denormalised tracks.release_date column in
-- lockstep so date-based recommendation queries stay indexable and never have
-- to guess which of two conflicting values is authoritative.
--
-- The trigger intentionally also overwrites a direct tracks.release_date
-- update. There is no per-recording release-date provenance in this schema;
-- accepting an independent track value would silently make novelty results
-- depend on write order. A future per-recording metadata source needs its own
-- canonical/provenance column before changing this rule.
CREATE OR REPLACE FUNCTION sync_track_release_date_from_album()
RETURNS TRIGGER AS $$
BEGIN
SELECT release_date INTO NEW.release_date
FROM albums
WHERE id = NEW.album_id;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS trg_tracks_sync_release_date ON tracks;
CREATE TRIGGER trg_tracks_sync_release_date
BEFORE INSERT OR UPDATE OF album_id, release_date ON tracks
FOR EACH ROW EXECUTE FUNCTION sync_track_release_date_from_album();
CREATE OR REPLACE FUNCTION propagate_album_release_date_to_tracks()
RETURNS TRIGGER AS $$
BEGIN
UPDATE tracks
SET release_date = NEW.release_date
WHERE album_id = NEW.id
AND release_date IS DISTINCT FROM NEW.release_date;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS trg_albums_propagate_release_date ON albums;
CREATE TRIGGER trg_albums_propagate_release_date
AFTER UPDATE OF release_date ON albums
FOR EACH ROW
WHEN (OLD.release_date IS DISTINCT FROM NEW.release_date)
EXECUTE FUNCTION propagate_album_release_date_to_tracks();
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
@@ -306,12 +360,20 @@ CREATE TABLE IF NOT EXISTS track_audio_features (
key TEXT,
energy REAL,
danceability REAL,
-- Reserved legacy columns: Vibe readers tolerate these as NULL. The local
-- analyzer intentionally does not claim to infer them.
valence REAL,
acousticness REAL,
instrumentalness REAL,
liveness REAL,
valence_score REAL,
tempo REAL
tempo REAL,
analysis_version SMALLINT NOT NULL DEFAULT 0,
source_hash TEXT,
analyzed_at TIMESTAMPTZ,
CONSTRAINT track_audio_features_bpm_range CHECK (bpm IS NULL OR (bpm >= 30 AND bpm <= 300)),
CONSTRAINT track_audio_features_energy_range CHECK (energy IS NULL OR (energy >= 0 AND energy <= 1)),
CONSTRAINT track_audio_features_danceability_range CHECK (danceability IS NULL OR (danceability >= 0 AND danceability <= 1))
);
CREATE TABLE IF NOT EXISTS track_lyrics (
@@ -331,6 +393,10 @@ CREATE TABLE IF NOT EXISTS settings (
);
INSERT INTO settings (key, value) VALUES ('enrich_metadata', 'true') ON CONFLICT (key) DO NOTHING;
-- Artist portraits are independent from structural metadata. Keeping this
-- separate lets an operator re-fill artwork without re-running MusicBrainz
-- canonicalisation across the entire library.
INSERT INTO settings (key, value) VALUES ('enrich_artist_images', 'true') ON CONFLICT (key) DO NOTHING;
INSERT INTO settings (key, value) VALUES ('enrich_cover_art', 'true') ON CONFLICT (key) DO NOTHING;
INSERT INTO settings (key, value) VALUES ('enrich_genres', 'true') ON CONFLICT (key) DO NOTHING;
INSERT INTO settings (key, value) VALUES ('enrich_lyrics', 'true') ON CONFLICT (key) DO NOTHING;
@@ -355,6 +421,10 @@ INSERT INTO source_trust (key, trust, description) VALUES
('cover_art_archive', 0.85, 'Cover Art Archive, MB-backed.'),
('discogs', 0.75, 'Discogs release/artist credits.'),
('lastfm', 0.50, 'Last.fm tags + similar. Noisy; used as weak signal.'),
-- A first-party record of how a candidate was reached through the graph.
-- This is deliberately separate from Last.fm/MB: it describes the
-- traversal strategy, not a claim made by an external provider.
('graph_exploration', 0.40, 'Muzick graph traversal provenance for discovery candidates.'),
('listener_behavior', 0.40, 'Derived from observed play patterns. User-keyed.'),
('tag', 0.30, 'File-tag-derived via scanner heuristic. Lowest trust.')
ON CONFLICT (key) DO NOTHING;
@@ -487,9 +557,19 @@ CREATE TABLE IF NOT EXISTS discovery_candidates (
first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_eval_at TIMESTAMPTZ,
status TEXT NOT NULL DEFAULT 'candidate',
-- Filled only after a worker scanned a successfully acquired file. Keeping
-- this FK makes candidate -> local-track provenance auditable and avoids
-- guessing from filename metadata later.
acquired_track_id UUID REFERENCES tracks(id) ON DELETE SET NULL,
acquired_at TIMESTAMPTZ,
acquisition_attempts INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
UNIQUE (source, external_id)
);
CREATE INDEX IF NOT EXISTS idx_discovery_candidates_status
ON discovery_candidates (status, first_seen_at);
-- Probation status for acquired tracks.
DO $$ BEGIN
IF NOT EXISTS (
+28
View File
@@ -16,6 +16,8 @@ export interface Album {
artist_id: string;
title: string;
year?: number | null;
/** Canonical MusicBrainz release-group first-release-date (YYYY-MM-DD). */
release_date?: string | null;
artwork_id?: string | null;
}
@@ -38,6 +40,8 @@ export interface Track {
skip_count: number;
dislike_count: number;
last_played_at?: Date | null;
/** Denormalised from albums.release_date by a database trigger. */
release_date?: string | null;
mtime?: number | null;
source_type: string;
artists?: TrackArtist[];
@@ -136,6 +140,30 @@ export interface ListenerBelief {
last_decayed_at: Date;
}
/**
* Stable pseudo-entity IDs for audio preference buckets. listener_beliefs uses
* UUID entity IDs for every entity type, while audio dimensions are values rather
* than rows in their own table. Keeping the IDs fixed makes these beliefs usable
* by the planner without introducing a second, unbounded vocabulary.
*/
export const AUDIO_PREFERENCE_BUCKETS = {
energy: {
low: '10000000-0000-0000-0000-000000000001',
medium: '10000000-0000-0000-0000-000000000002',
high: '10000000-0000-0000-0000-000000000003',
},
bpm: {
slow: '10000000-0000-0000-0000-000000000011',
medium: '10000000-0000-0000-0000-000000000012',
fast: '10000000-0000-0000-0000-000000000013',
},
valence: {
low: '10000000-0000-0000-0000-000000000021',
neutral: '10000000-0000-0000-0000-000000000022',
high: '10000000-0000-0000-0000-000000000023',
},
} as const;
export interface ClaimEdge {
subjectType: string;
subjectId: string;
+260 -10
View File
@@ -2,6 +2,87 @@ import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { JobService } from '../services/job.service.js';
import { DbService } from '../services/db.service.js';
const ENRICHMENT_SETTING_KEYS = [
'enrich_metadata',
'enrich_artist_images',
'enrich_cover_art',
'enrich_genres',
'enrich_lyrics',
'enrich_artist_similarity',
'enrich_audio_analysis',
] as const;
type ReenrichmentScope = {
metadata?: boolean;
artistImages?: boolean;
albumCovers?: boolean;
};
type ReenrichmentRequest = {
/** Preview by default. Jobs are only added with an explicit confirmation. */
confirm?: boolean;
/** Missing-only is the safe default; false means refresh the selected scope. */
missingOnly?: boolean;
/** Per-scope batch cap. Defaults to 250 and never exceeds 1,000. */
limit?: number;
scope?: ReenrichmentScope;
};
function getReenrichmentOptions(body: ReenrichmentRequest = {}) {
const requestedScope = body.scope ?? {};
const scope = {
metadata: requestedScope.metadata ?? true,
artistImages: requestedScope.artistImages ?? true,
albumCovers: requestedScope.albumCovers ?? true,
};
const rawLimit = Number(body.limit ?? 250);
const limit = Number.isInteger(rawLimit) ? Math.max(1, Math.min(rawLimit, 1000)) : 250;
return { confirm: body.confirm === true, missingOnly: body.missingOnly !== false, limit, scope };
}
async function getReenrichmentStatus(dbService: DbService, jobService: JobService) {
const db = dbService.pgClient;
const [settingsRes, coverageRes, queue] = await Promise.all([
db.query<{ key: string; value: string }>(
`SELECT key, value FROM settings WHERE key = ANY($1::text[])`,
[ENRICHMENT_SETTING_KEYS],
),
db.query<{
library_tracks: number;
tracks_without_release_date: number;
tracks_without_album_mbid: number;
artists_total: number;
artists_without_image: number;
artists_without_mbid: number;
albums_total: number;
albums_without_artwork: number;
albums_without_release_date: number;
}>(`
SELECT
COUNT(*) FILTER (WHERE t.state = 'LIBRARY')::int AS library_tracks,
COUNT(*) FILTER (WHERE t.state = 'LIBRARY' AND t.release_date IS NULL)::int AS tracks_without_release_date,
COUNT(*) FILTER (WHERE t.state = 'LIBRARY' AND al.mbid IS NULL)::int AS tracks_without_album_mbid,
(SELECT COUNT(*)::int FROM artists) AS artists_total,
(SELECT COUNT(*)::int FROM artists WHERE image_path IS NULL OR image_path = '') AS artists_without_image,
(SELECT COUNT(*)::int FROM artists WHERE mbid IS NULL) AS artists_without_mbid,
(SELECT COUNT(*)::int FROM albums) AS albums_total,
(SELECT COUNT(*)::int FROM albums WHERE artwork_id IS NULL OR artwork_id = '') AS albums_without_artwork,
(SELECT COUNT(*)::int FROM albums WHERE release_date IS NULL) AS albums_without_release_date
FROM tracks t
LEFT JOIN albums al ON al.id = t.album_id
`),
jobService.getEnrichmentQueueDiagnostics(),
]);
const settings = Object.fromEntries(
ENRICHMENT_SETTING_KEYS.map((key) => [
key,
settingsRes.rows.find((row) => row.key === key)?.value === 'true',
]),
) as Record<(typeof ENRICHMENT_SETTING_KEYS)[number], boolean>;
return { settings, coverage: coverageRes.rows[0], queue };
}
export default async function adminRoutes(fastify: FastifyInstance, options: { jobService: JobService; dbService: DbService }) {
const { jobService, dbService } = options;
@@ -24,6 +105,60 @@ export default async function adminRoutes(fastify: FastifyInstance, options: { j
return { status: 'Artist reprocessing job enqueued' };
});
/** Rebuild artist/genre/audio beliefs from durable history after a model upgrade. */
fastify.post('/vibe/rebuild-beliefs', async (request: FastifyRequest, reply: FastifyReply) => {
const { userId } = (request.body ?? {}) as { userId?: string };
const resolvedUserId = userId || (request.headers['x-user-id'] as string) || '00000000-0000-0000-0000-000000000000';
const result = await dbService.rebuildDerivedListenerBeliefs(resolvedUserId);
return reply.send({ status: 'rebuilt', userId: resolvedUserId, ...result });
});
/**
* Attach a human/resolver-vetted source to a System E candidate. This is an
* admin-only hand-off: graph traversal identifies an artist/path, but must
* never turn that into an arbitrary web search and download. The worker still
* applies its own host allow-list immediately before invoking yt-dlp.
*/
fastify.post('/discovery/candidates/:id/acquisition-source', async (request, reply) => {
const { id } = request.params as { id: string };
const body = request.body as { url?: string; expectedTitle?: string; expectedArtist?: string };
if (!body?.url || typeof body.url !== 'string' || body.url.length > 4000) {
return reply.code(400).send({ error: 'HTTPS url is required' });
}
let url: URL;
try {
url = new URL(body.url);
} catch {
return reply.code(400).send({ error: 'url is invalid' });
}
if (url.protocol !== 'https:' || url.username || url.password) {
return reply.code(400).send({ error: 'url must be credential-free HTTPS' });
}
const source = {
url: url.toString(),
...(typeof body.expectedTitle === 'string' ? { expectedTitle: body.expectedTitle.slice(0, 500) } : {}),
...(typeof body.expectedArtist === 'string' ? { expectedArtist: body.expectedArtist.slice(0, 500) } : {}),
};
try {
const result = await dbService.pgClient.query(
`UPDATE discovery_candidates
SET notes = jsonb_set(COALESCE(notes, '{}'::jsonb), '{acquisition}', $2::jsonb, true),
status = 'candidate', last_eval_at = NULL, last_error = NULL
WHERE id = $1::uuid
AND status IN ('candidate', 'awaiting_resolution', 'acquisition_disabled', 'failed')
RETURNING id, status`,
[id, JSON.stringify(source)]
);
if (result.rows.length === 0) {
return reply.code(404).send({ error: 'candidate does not exist or cannot be re-queued' });
}
return reply.send({ candidate: result.rows[0] });
} catch (err) {
const message = err instanceof Error ? err.message : 'could not attach acquisition source';
return reply.code(400).send({ error: message });
}
});
fastify.post('/dedup-albums', async (_request: FastifyRequest, reply: FastifyReply) => {
// Merge duplicate album rows directly (synchronous — it's just SQL, no
// external API calls). Returns the number of albums merged away.
@@ -92,17 +227,132 @@ export default async function adminRoutes(fastify: FastifyInstance, options: { j
return { status: 'Albums deduplicated', merged: res.rows[0]?.count ?? 0 };
});
/**
* Return coverage, active toggles, queued work, and recent worker failures.
* This is deliberately separate from enqueueing so an operator can diagnose
* a disabled provider or a failing worker before launching another batch.
*/
fastify.get('/reenrichment/status', async () => {
return getReenrichmentStatus(dbService, jobService);
});
/**
* Safe re-enrichment control plane. A call is a preview unless confirm=true;
* the default is a 250-entity, missing-only batch. The three job types remain
* independent so artwork work is not hidden behind track metadata work.
*/
fastify.post<{ Body: ReenrichmentRequest }>('/reenrichment', async (request, reply) => {
const options = getReenrichmentOptions(request.body);
const status = await getReenrichmentStatus(dbService, jobService);
const blockedBySettings: Partial<Record<keyof ReenrichmentScope, string>> = {};
const metadataEnabled = status.settings.enrich_metadata
|| status.settings.enrich_genres
|| status.settings.enrich_lyrics;
if (options.scope.metadata && !metadataEnabled) {
blockedBySettings.metadata = 'All track enrichment toggles are disabled.';
}
if (options.scope.artistImages && !status.settings.enrich_artist_images) {
blockedBySettings.artistImages = 'enrich_artist_images is disabled.';
}
if (options.scope.albumCovers && !status.settings.enrich_cover_art) {
blockedBySettings.albumCovers = 'enrich_cover_art is disabled.';
}
const db = dbService.pgClient;
const [tracks, artists, albums] = await Promise.all([
options.scope.metadata && !blockedBySettings.metadata
? db.query<{ id: string }>(
`SELECT t.id
FROM tracks t
LEFT JOIN albums al ON al.id = t.album_id
LEFT JOIN track_artists ta ON ta.track_id = t.id AND ta.role = 'main'
LEFT JOIN artists ar ON ar.id = ta.artist_id
WHERE t.state = 'LIBRARY'
AND ($1::boolean = false OR t.release_date IS NULL OR al.mbid IS NULL OR ar.mbid IS NULL)
ORDER BY t.id
LIMIT $2`,
[options.missingOnly, options.limit],
)
: Promise.resolve({ rows: [] as Array<{ id: string }> }),
options.scope.artistImages && !blockedBySettings.artistImages
? db.query<{ id: string }>(
`SELECT id FROM artists
WHERE $1::boolean = false OR image_path IS NULL OR image_path = ''
ORDER BY id
LIMIT $2`,
[options.missingOnly, options.limit],
)
: Promise.resolve({ rows: [] as Array<{ id: string }> }),
options.scope.albumCovers && !blockedBySettings.albumCovers
? db.query<{ id: string }>(
`SELECT id FROM albums
WHERE $1::boolean = false OR artwork_id IS NULL OR artwork_id = ''
ORDER BY id
LIMIT $2`,
[options.missingOnly, options.limit],
)
: Promise.resolve({ rows: [] as Array<{ id: string }> }),
]);
const plan = {
metadataTrackIds: tracks.rows.map((row) => row.id),
artistIds: artists.rows.map((row) => row.id),
albumIds: albums.rows.map((row) => row.id),
};
const wouldQueue = {
metadata: plan.metadataTrackIds.length,
artistImages: plan.artistIds.length,
albumCovers: plan.albumIds.length,
};
if (!options.confirm) {
return {
status: 'preview',
message: 'No jobs were queued. Repeat with confirm=true to enqueue this bounded plan.',
options,
blockedBySettings,
wouldQueue,
diagnostics: status,
};
}
if (Object.keys(blockedBySettings).length === 3 ||
(wouldQueue.metadata + wouldQueue.artistImages + wouldQueue.albumCovers === 0)) {
return reply.code(409).send({
status: 'not_queued',
message: Object.keys(blockedBySettings).length === 3
? 'Every selected scope is disabled by enrichment settings.'
: 'No eligible entities matched this re-enrichment batch.',
options,
blockedBySettings,
wouldQueue,
diagnostics: status,
});
}
const queued = await jobService.enqueueReenrichment(plan);
const enqueueFailures = queued.metadata.failed.length
+ queued.artistImages.failed.length
+ queued.albumCovers.failed.length;
return {
status: enqueueFailures === 0 ? 'queued' : 'partially_queued',
message: enqueueFailures === 0
? 'Re-enrichment jobs were queued. Check /reenrichment/status and job history for outcomes.'
: 'Some jobs could not be queued; see queued.*.failed for exact errors.',
options,
blockedBySettings,
queued,
diagnostics: await getReenrichmentStatus(dbService, jobService),
};
});
// The prior endpoint queued every track and implied cover/image work that it
// never scheduled. Keep the failure explicit instead of silently claiming a
// successful library-wide refresh.
fastify.post('/reenrich-tracks', async (_request: FastifyRequest, reply: FastifyReply) => {
// Re-enqueue metadata_refresh for every LIBRARY track without re-reading
// files from disk. This re-runs the MusicBrainz canonicalisation (artist
// names, album titles, MBIDs) and re-triggers album_cover jobs — much
// faster than a full scan when only metadata needs refreshing.
const res = await dbService.pgClient.query<{ id: string }>(
`SELECT id FROM tracks WHERE state = 'LIBRARY' ORDER BY id`
);
const trackIds = res.rows.map((r) => r.id);
const enqueued = await jobService.enqueueMetadataRefreshBatch(trackIds);
return { status: 'Re-enrich enqueued', trackCount: enqueued };
return reply.code(410).send({
error: 'Deprecated endpoint. Use POST /admin/reenrichment (preview first; confirm=true to queue).',
});
});
fastify.get('/queue-stats', async () => {
+14 -1
View File
@@ -2,9 +2,11 @@ import { FastifyInstance } from 'fastify';
import { DbService } from '../services/db.service.js';
import { DiscoveryService } from '../services/discovery.service.js';
import { ImageEnrichmentService } from '../services/image-enrichment.service.js';
import { JobService } from '../services/job.service.js';
export default async function discoveryRoutes(fastify: FastifyInstance, options: { dbService: DbService }) {
export default async function discoveryRoutes(fastify: FastifyInstance, options: { dbService: DbService; jobService: JobService }) {
const { dbService } = options;
const { jobService } = options;
const discovery = new DiscoveryService(dbService);
const images = new ImageEnrichmentService(dbService);
@@ -39,6 +41,17 @@ export default async function discoveryRoutes(fastify: FastifyInstance, options:
fastify.post('/discovery/eval', async (request, reply) => {
const userId = (request.headers['x-user-id'] as string) || '00000000-0000-0000-0000-000000000000';
const results = await discovery.evalCandidates(userId);
for (const result of results) {
if (!result.shouldAcquire) continue;
try {
await jobService.enqueueDiscoveryAcquisition(result.candidateId);
} catch (err) {
const reason = err instanceof Error ? err.message : 'failed to enqueue acquisition';
await discovery.markEnqueueFailed(result.candidateId, reason);
result.shouldAcquire = false;
result.reason = `queue unavailable: ${reason}`;
}
}
return reply.send({ evaluated: results.length, results });
});
+1
View File
@@ -3,6 +3,7 @@ import { DbService } from '../services/db.service.js';
const SETTING_KEYS = [
'enrich_metadata',
'enrich_artist_images',
'enrich_cover_art',
'enrich_genres',
'enrich_lyrics',
+114 -12
View File
@@ -9,14 +9,42 @@ interface ActivePlan {
sessionId: string;
plan: Candidate[];
seedTrackId: string | null;
/** Tracks handed to the player during this Redis-backed session. */
servedTrackIds?: string[];
/** Explicit feedback targets (skip, dislike, completion, promotion). */
excludedTrackIds?: string[];
/** Main artists served or explicitly rejected in this session. */
excludedArtistIds?: string[];
}
const PLAN_TTL_SEC = 2 * 3600;
// Retain enough history for long listening sessions without allowing an
// unbounded Redis value if a client leaves a session running for days.
const MAX_SESSION_EXCLUSIONS = 1000;
function planKey(userId: string): string {
return `v2:plan:${userId}`;
}
function appendUniqueTrackId(ids: string[] | undefined, trackId: string): string[] {
const next = ids ? [...ids] : [];
if (!next.includes(trackId)) next.push(trackId);
return next.length > MAX_SESSION_EXCLUSIONS
? next.slice(next.length - MAX_SESSION_EXCLUSIONS)
: next;
}
function sessionExclusions(active: ActivePlan): string[] {
return [...new Set([
...(active.servedTrackIds ?? []),
...(active.excludedTrackIds ?? []),
])];
}
function sessionArtistExclusions(active: ActivePlan): string[] {
return [...new Set(active.excludedArtistIds ?? [])];
}
export default async function v2Routes(fastify: FastifyInstance, options: { dbService: DbService; sessionDirector: SessionDirector }) {
const { dbService, sessionDirector: director } = options;
@@ -47,6 +75,7 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
// span multiple keys/services atomically, swap this for a WATCH/MULTI transaction
// or move the plan into a single Lua script instead of an app-level lock.
const RELEASE_LOCK_LUA = `if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end`;
const RENEW_LOCK_LUA = `if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end`;
async function withPlanLock<T>(userId: string, fn: () => Promise<T>): Promise<T> {
const lockKey = `v2:planlock:${userId}`;
@@ -54,16 +83,20 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
const deadline = Date.now() + 5000;
let acquired = false;
while (Date.now() < deadline) {
const res = await redisClient.set(lockKey, token, { NX: true, PX: 3000 });
const res = await redisClient.set(lockKey, token, { NX: true, PX: 5000 });
if (res) { acquired = true; break; }
await new Promise((r) => setTimeout(r, 20 + Math.random() * 30));
}
if (!acquired) {
throw new Error('Timed out waiting for active-plan lock');
}
const renewal = setInterval(() => {
void redisClient.eval(RENEW_LOCK_LUA, { keys: [lockKey], arguments: [token, '5000'] });
}, 1500);
try {
return await fn();
} finally {
clearInterval(renewal);
await redisClient.eval(RELEASE_LOCK_LUA, { keys: [lockKey], arguments: [token] });
}
}
@@ -78,9 +111,18 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
const { seedTrackId } = request.body as { seedTrackId?: string };
const sessionId = await dbService.createSessionState(userId, undefined, { energy: 0.5, novelty_hunger: 0.3 });
const plan = await director.buildPlan(userId, sessionId, seedTrackId);
const initialExclusions = seedTrackId ? [seedTrackId] : [];
const plan = await director.buildPlan(userId, sessionId, seedTrackId, {
excludedTrackIds: initialExclusions,
});
await setActivePlan(userId, { sessionId, plan, seedTrackId: seedTrackId ?? null });
await setActivePlan(userId, {
sessionId,
plan,
seedTrackId: seedTrackId ?? null,
servedTrackIds: [],
excludedTrackIds: initialExclusions,
});
return reply.send({ sessionId, plan: plan.slice(0, 10) });
});
@@ -90,34 +132,71 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
*/
fastify.get('/v2/vibe/next', async (request, reply) => {
const userId = (request.headers['x-user-id'] as string) || '00000000-0000-0000-0000-000000000000';
const { sessionId } = request.query as { sessionId?: string };
const result = await withPlanLock(userId, async () => {
const active = await getActivePlan(userId);
if (!active || active.plan.length === 0) {
return null;
if (!active) {
return { kind: 'missing' as const };
}
if (!sessionId || active.sessionId !== sessionId) {
return { kind: 'replaced' as const };
}
if (active.plan.length === 0) {
return { kind: 'exhausted' as const };
}
const next = active.plan.shift()!;
active.servedTrackIds = appendUniqueTrackId(active.servedTrackIds, next.trackId);
const artistResult = await dbService.pgClient.query<{ artist_id: string }>(
`SELECT artist_id FROM track_artists_v2
WHERE track_id = $1 AND role = 'main'
ORDER BY confidence DESC NULLS LAST LIMIT 1`,
[next.trackId]
);
if (artistResult.rows[0]?.artist_id) {
active.excludedArtistIds = appendUniqueTrackId(active.excludedArtistIds, artistResult.rows[0].artist_id);
}
// Enrich with track details
const track = await dbService.getTrackById(next.trackId);
// Replan if running low
if (active.plan.length < 5) {
const refill = await director.replan(userId, active.sessionId, active.plan, [next.trackId], active.seedTrackId ?? undefined);
const refill = await director.replan(
userId,
active.sessionId,
active.plan,
[next.trackId],
active.seedTrackId ?? undefined,
{ excludedTrackIds: sessionExclusions(active), excludedArtistIds: sessionArtistExclusions(active) }
);
active.plan = refill;
}
await setActivePlan(userId, active);
return { track, explanation: next.explanation, planRemaining: active.plan.length };
return { kind: 'track' as const, track, explanation: next.explanation, planRemaining: active.plan.length };
});
if (!result) {
if (result.kind === 'missing') {
return reply.code(404).send({ error: 'No active plan. POST /api/v2/vibe/start first.' });
}
if (result.kind === 'replaced') {
return reply.code(409).send({ error: 'This Vibe session was replaced by a newer session.', code: 'VIBE_SESSION_REPLACED' });
}
if (result.kind === 'exhausted') {
return reply.code(409).send({
error: 'Vibe plan exhausted: no eligible unserved tracks remain for this session.',
code: 'VIBE_PLAN_EXHAUSTED',
});
}
return reply.send(result);
return reply.send({
track: result.track,
explanation: result.explanation,
planRemaining: result.planRemaining,
});
});
/**
@@ -126,11 +205,14 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
*/
fastify.post('/v2/vibe/feedback', async (request, reply) => {
const userId = (request.headers['x-user-id'] as string) || '00000000-0000-0000-0000-000000000000';
const { trackId, action } = request.body as { trackId: string; action: string };
const { trackId, action, sessionId } = request.body as { trackId: string; action: string; sessionId?: string };
if (!trackId || !action) {
return reply.code(400).send({ error: 'trackId and action are required' });
}
if (!['completed', 'skipped', 'promoted', 'disliked'].includes(action)) {
return reply.code(400).send({ error: 'Unsupported Vibe feedback action' });
}
// Route to existing handlers for evidence wiring
if (action === 'completed') {
@@ -148,8 +230,28 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
const planRemaining = await withPlanLock(userId, async () => {
const active = await getActivePlan(userId);
if (!active) return 0;
const playedTrackIds = [trackId];
const refill = await director.replan(userId, active.sessionId, active.plan, playedTrackIds, active.seedTrackId ?? undefined);
if (!sessionId || active.sessionId !== sessionId) return 0;
// Feedback may race /next or arrive after a client-side prefetch. In all
// cases its track becomes ineligible for the rest of this session.
active.excludedTrackIds = appendUniqueTrackId(active.excludedTrackIds, trackId);
const artistResult = await dbService.pgClient.query<{ artist_id: string }>(
`SELECT artist_id FROM track_artists_v2
WHERE track_id = $1 AND role = 'main'
ORDER BY confidence DESC NULLS LAST LIMIT 1`,
[trackId]
);
if (artistResult.rows[0]?.artist_id) {
active.excludedArtistIds = appendUniqueTrackId(active.excludedArtistIds, artistResult.rows[0].artist_id);
}
const sessionTrackIds = sessionExclusions(active);
const refill = await director.replan(
userId,
active.sessionId,
active.plan,
[trackId],
active.seedTrackId ?? undefined,
{ excludedTrackIds: sessionTrackIds, excludedArtistIds: sessionArtistExclusions(active) }
);
active.plan = refill;
await setActivePlan(userId, active);
return active.plan.length;
+85
View File
@@ -8,6 +8,22 @@ function makeService(): { service: DbService; mockQuery: ReturnType<typeof vi.fn
}
describe('DbService v2 methods', () => {
describe('createAlbum', () => {
it('persists the canonical release date instead of discarding it', async () => {
const { service, mockQuery } = makeService();
mockQuery.mockResolvedValue({ rows: [{ id: 'album-1' }] });
await service.createAlbum({
id: 'album-1', artist_id: 'artist-1', title: 'Album',
year: 2026, release_date: '2026-07-15', artwork_id: null,
});
const [sql, params] = mockQuery.mock.calls[0];
expect(sql).toContain('release_date');
expect(params[3]).toBe('2026-07-15');
});
});
describe('upsertClaim', () => {
it('calls INSERT ... ON CONFLICT with correct parameters', async () => {
const { service, mockQuery } = makeService();
@@ -168,6 +184,75 @@ describe('DbService v2 methods', () => {
});
});
describe('track evidence propagation', () => {
it('projects a favourite onto artist, genre, and present audio dimensions', async () => {
const { service, mockQuery } = makeService();
let evidenceNumber = 0;
mockQuery.mockImplementation((sql: string) => {
if (sql.includes('INSERT INTO evidence')) return Promise.resolve({ rows: [{ id: `ev-${++evidenceNumber}` }] });
if (sql.includes('WITH artist_ids')) {
return Promise.resolve({ rows: [
{ entity_type: 'artist', entity_id: 'artist-1' },
{ entity_type: 'genre', entity_id: 'genre-1' },
] });
}
if (sql.includes('SELECT energy, bpm, valence')) {
return Promise.resolve({ rows: [{ energy: 0.81, bpm: 128, valence: 0.22 }] });
}
return Promise.resolve({ rowCount: 1, rows: [] });
});
await service.recordTrackEvidence({
user_id: 'user-1', track_id: 'track-1', signal: 'add_to_favorites',
profile: 'longterm', weight: 0.60,
});
const evidenceWrites = mockQuery.mock.calls
.filter(([sql]) => (sql as string).includes('INSERT INTO evidence'))
.map(([, params]) => params as unknown[]);
expect(evidenceWrites.map(params => params[1])).toEqual([
'track', 'artist', 'genre', 'audio', 'audio', 'audio',
]);
const beliefWrites = mockQuery.mock.calls
.filter(([sql]) => (sql as string).includes('INSERT INTO listener_beliefs'))
.map(([, params]) => params as unknown[]);
const artistBelief = beliefWrites.find(params => params[2] === 'artist');
const genreBelief = beliefWrites.find(params => params[2] === 'genre');
expect(artistBelief?.[3]).toBe('artist-1');
expect(artistBelief?.[5]).toBeCloseTo(0.54); // one favourite is a usable comfort signal
expect(genreBelief?.[5]).toBeCloseTo(0.27);
expect(beliefWrites.filter(params => params[2] === 'audio')).toHaveLength(3);
});
it('rebuilds shared beliefs from local completed plays and feedback without appending evidence', async () => {
const clientQuery = vi.fn((sql: string) => {
if (sql === 'BEGIN' || sql === 'COMMIT' || sql === 'ROLLBACK') return Promise.resolve({ rows: [] });
if (sql.includes('DELETE FROM listener_beliefs')) return Promise.resolve({ rowCount: 0, rows: [] });
if (sql.includes('FROM play_history ph')) {
return Promise.resolve({ rows: [{
track_id: 'track-1', signal: 'playback_completed', profile: 'longterm', weight: 0.10,
}] });
}
if (sql.includes('WITH artist_ids')) return Promise.resolve({ rows: [{ entity_type: 'artist', entity_id: 'artist-1' }] });
if (sql.includes('SELECT energy, bpm, valence')) return Promise.resolve({ rows: [] });
return Promise.resolve({ rowCount: 1, rows: [] });
});
const connect = vi.fn().mockResolvedValue({ query: clientQuery, release: vi.fn() });
const service = new DbService({ query: vi.fn(), connect } as any);
const result = await service.rebuildDerivedListenerBeliefs('user-1');
expect(result).toEqual({ interactions: 1, beliefs: 1 });
expect(clientQuery.mock.calls.some(([sql]) => (sql as string).includes('FROM feedback f'))).toBe(true);
expect(clientQuery.mock.calls.some(([sql]) => (sql as string).includes('INSERT INTO evidence'))).toBe(false);
const beliefCall = (clientQuery.mock.calls as unknown as Array<[string, unknown[]]>).
find(([sql]) => sql.includes('INSERT INTO listener_beliefs'));
const beliefParams = beliefCall?.[1] ?? [];
expect(beliefParams.slice(1, 5)).toEqual(['longterm', 'artist', 'artist-1', 'affinity']);
});
});
describe('getFusedTrackArtists', () => {
it('reads from claim_fusion view', async () => {
const { service, mockQuery } = makeService();
+251 -27
View File
@@ -34,7 +34,7 @@ import type {
DiversityBudget,
RepetitionRule,
} from '../db/types.js';
import { FEEDBACK_ACTIONS } from '../db/types.js';
import { AUDIO_PREFERENCE_BUCKETS, FEEDBACK_ACTIONS } from '../db/types.js';
export * from '../db/types.js';
export class DbService {
@@ -372,11 +372,11 @@ export class DbService {
);
});
// Write evidence: hidden → negative profile (only on success)
await this.recordEvidence({
// Write evidence: hidden → negative profile (only on success), then carry
// that signal through the track's artist/genre/audio identities.
await this.recordTrackEvidence({
user_id: userId,
entity_type: 'track',
entity_id: trackId,
track_id: trackId,
signal: 'hidden',
profile: 'negative',
weight: -0.60,
@@ -462,10 +462,9 @@ export class DbService {
);
// 3. Write evidence: playback_completed → longterm affinity
await this.recordEvidence({
await this.recordTrackEvidence({
user_id: userId,
entity_type: 'track',
entity_id: trackId,
track_id: trackId,
signal: 'playback_completed',
profile: 'longterm',
weight: 0.10,
@@ -480,18 +479,16 @@ export class DbService {
[userId, trackId]
);
if ((recentPlays.rows[0]?.cnt as number) > 1) {
await this.recordEvidence({
await this.recordTrackEvidence({
user_id: userId,
entity_type: 'track',
entity_id: trackId,
track_id: trackId,
signal: 'replay_within_24h',
profile: 'longterm',
weight: 0.25,
}, client);
await this.recordEvidence({
await this.recordTrackEvidence({
user_id: userId,
entity_type: 'track',
entity_id: trackId,
track_id: trackId,
signal: 'replay_within_24h',
profile: 'obsession',
weight: 0.40,
@@ -591,10 +588,9 @@ export class DbService {
[userId, trackId]
);
// Write evidence: skip_quick → negative profile
await this.recordEvidence({
await this.recordTrackEvidence({
user_id: userId,
entity_type: 'track',
entity_id: trackId,
track_id: trackId,
signal: 'skip_quick',
profile: 'negative',
weight: -0.20,
@@ -613,19 +609,17 @@ export class DbService {
// Also write evidence for promoted/disliked signals
if (action === 'promoted') {
await this.recordEvidence({
await this.recordTrackEvidence({
user_id: userId,
entity_type: 'track',
entity_id: trackId,
track_id: trackId,
signal: 'add_to_favorites',
profile: 'longterm',
weight: 0.60,
});
} else if (action === 'disliked') {
await this.recordEvidence({
await this.recordTrackEvidence({
user_id: userId,
entity_type: 'track',
entity_id: trackId,
track_id: trackId,
signal: 'hidden',
profile: 'negative',
weight: -0.60,
@@ -688,8 +682,9 @@ export class DbService {
async createAlbum(data: Album): Promise<Album> {
const res = await this.pgClient.query(
'INSERT INTO albums (artist_id, title, year, artwork_id) VALUES ($1, $2, $3, $4) RETURNING *',
[data.artist_id, data.title, data.year, data.artwork_id]
`INSERT INTO albums (artist_id, title, year, release_date, artwork_id)
VALUES ($1, $2, $3, $4::date, $5) RETURNING *`,
[data.artist_id, data.title, data.year, data.release_date ?? null, data.artwork_id]
);
return res.rows[0];
}
@@ -1102,6 +1097,123 @@ export class DbService {
// v2 — System B: Listener Model
// =========================================================================
private beliefDimensionForSignal(signal: string): string {
return signal === 'play_of_never_seen' ? 'novelty_tolerance' : 'affinity';
}
/**
* Resolve the durable identities represented by a track. Artist credits use
* the fusion-backed view (with the legacy table as a fallback during an
* enrichment transition); genres and audio features are direct metadata.
*/
private async getTrackBeliefTargets(trackId: string, client?: Queryable): Promise<Array<{
entity_type: 'artist' | 'genre' | 'audio';
entity_id: string;
factor: number;
context: Record<string, unknown>;
}>> {
const queryable = client ?? this.pgClient;
const targets: Array<{
entity_type: 'artist' | 'genre' | 'audio';
entity_id: string;
factor: number;
context: Record<string, unknown>;
}> = [];
const identities = await queryable.query(
`WITH artist_ids AS (
SELECT artist_id FROM track_artists_v2 WHERE track_id = $1
UNION
SELECT artist_id FROM track_artists WHERE track_id = $1
)
SELECT 'artist' AS entity_type, artist_id AS entity_id FROM artist_ids
UNION ALL
SELECT 'genre' AS entity_type, genre_id AS entity_id
FROM track_genre WHERE track_id = $1`,
[trackId]
);
for (const row of identities.rows as Array<{ entity_type: 'artist' | 'genre'; entity_id: string }>) {
targets.push({
entity_type: row.entity_type,
entity_id: row.entity_id,
// An explicit favourite should be enough to form a usable comfort
// artist belief; ordinary completed plays still accumulate gradually.
factor: row.entity_type === 'artist' ? 0.90 : 0.45,
context: { source_track_id: trackId, association: row.entity_type },
});
}
const featuresRes = await queryable.query(
`SELECT energy, bpm, valence
FROM track_audio_features
WHERE track_id = $1`,
[trackId]
);
const features = featuresRes.rows[0] as { energy?: number | null; bpm?: number | null; valence?: number | null } | undefined;
if (!features) return targets;
const addAudioBucket = (dimension: 'energy' | 'bpm' | 'valence', bucket: string) => {
targets.push({
entity_type: 'audio',
entity_id: bucket,
factor: 0.30,
context: { source_track_id: trackId, association: 'audio', dimension },
});
};
if (typeof features.energy === 'number' && Number.isFinite(features.energy)) {
addAudioBucket('energy', features.energy < 0.34
? AUDIO_PREFERENCE_BUCKETS.energy.low
: features.energy < 0.67 ? AUDIO_PREFERENCE_BUCKETS.energy.medium : AUDIO_PREFERENCE_BUCKETS.energy.high);
}
if (typeof features.bpm === 'number' && Number.isFinite(features.bpm) && features.bpm > 0) {
addAudioBucket('bpm', features.bpm < 90
? AUDIO_PREFERENCE_BUCKETS.bpm.slow
: features.bpm <= 140 ? AUDIO_PREFERENCE_BUCKETS.bpm.medium : AUDIO_PREFERENCE_BUCKETS.bpm.fast);
}
if (typeof features.valence === 'number' && Number.isFinite(features.valence)) {
addAudioBucket('valence', features.valence < 0.34
? AUDIO_PREFERENCE_BUCKETS.valence.low
: features.valence < 0.67 ? AUDIO_PREFERENCE_BUCKETS.valence.neutral : AUDIO_PREFERENCE_BUCKETS.valence.high);
}
return targets;
}
/**
* Append the track-level event, then project it onto the track's meaningful
* shared identities. The original event remains the canonical audit record;
* projected evidence makes artist/genre/audio affinity directly queryable by
* Vibe generators. Callers pass their transaction client to keep the event
* and every derived belief atomic.
*/
async recordTrackEvidence(evidence: {
user_id: string;
track_id: string;
signal: string;
profile: string;
weight: number;
context?: Record<string, unknown>;
}, client?: Queryable): Promise<string> {
const { track_id: trackId, ...event } = evidence;
const id = await this.recordEvidence({
...event,
entity_type: 'track',
entity_id: trackId,
}, client);
const targets = await this.getTrackBeliefTargets(trackId, client);
for (const target of targets) {
await this.recordEvidence({
user_id: event.user_id,
entity_type: target.entity_type,
entity_id: target.entity_id,
signal: event.signal,
profile: event.profile,
weight: event.weight * target.factor,
context: { ...event.context, ...target.context },
}, client);
}
return id;
}
/**
* Record evidence (append-only). Writes a signal into the evidence stream.
*/
@@ -1134,7 +1246,7 @@ export class DbService {
// which feeds 'novelty_tolerance'. Each new evidence row must also
// upsert the matching listener_belief (spec §B.4) — otherwise evidence
// accumulates but beliefs never materialise.
const dimension = evidence.signal === 'play_of_never_seen' ? 'novelty_tolerance' : 'affinity';
const dimension = this.beliefDimensionForSignal(evidence.signal);
await this.updateListenerBelief({
user_id: evidence.user_id,
profile: evidence.profile,
@@ -1163,6 +1275,119 @@ export class DbService {
});
}
/**
* Rebuild only the derived shared-preference layer from durable local
* interaction history. This intentionally does not append new evidence (the
* evidence log is an audit stream) and does not replace track beliefs. It is
* safe to run repeatedly after deploying propagation or after enrichment has
* added artist/genre/audio metadata to old tracks.
*/
async rebuildDerivedListenerBeliefs(userId: string): Promise<{ interactions: number; beliefs: number }> {
return this.withTransaction(async (client) => {
await client.query(
`DELETE FROM listener_beliefs
WHERE user_id = $1 AND entity_type IN ('artist', 'genre', 'audio')`,
[userId]
);
const interactionRes = await client.query(
`SELECT track_id, signal, profile, weight
FROM (
SELECT ph.track_id, ph.played_at AS occurred_at,
'playback_completed'::text AS signal,
'longterm'::text AS profile,
0.10::real AS weight
FROM play_history ph
WHERE ph.user_id = $1 AND ph.completed = true
UNION ALL
SELECT replay.track_id, replay.played_at AS occurred_at,
'replay_within_24h'::text AS signal,
'longterm'::text AS profile,
0.25::real AS weight
FROM (
SELECT track_id, played_at,
COUNT(*) OVER (
PARTITION BY track_id ORDER BY played_at
RANGE BETWEEN INTERVAL '24 hours' PRECEDING AND CURRENT ROW
) AS recent_plays
FROM play_history WHERE user_id = $1 AND completed = true
) replay
WHERE replay.recent_plays > 1
UNION ALL
SELECT replay.track_id, replay.played_at AS occurred_at,
'replay_within_24h'::text AS signal,
'obsession'::text AS profile,
0.40::real AS weight
FROM (
SELECT track_id, played_at,
COUNT(*) OVER (
PARTITION BY track_id ORDER BY played_at
RANGE BETWEEN INTERVAL '24 hours' PRECEDING AND CURRENT ROW
) AS recent_plays
FROM play_history WHERE user_id = $1 AND completed = true
) replay
WHERE replay.recent_plays > 1
UNION ALL
SELECT fav.track_id, fav.created_at AS occurred_at,
'add_to_favorites'::text AS signal,
'longterm'::text AS profile,
0.60::real AS weight
FROM favorites fav WHERE fav.user_id = $1
UNION ALL
SELECT f.track_id, f.created_at AS occurred_at,
CASE f.action
WHEN 'promoted' THEN 'add_to_favorites'
WHEN 'disliked' THEN 'hidden'
WHEN 'skipped' THEN 'skip_quick'
END AS signal,
CASE WHEN f.action = 'promoted' THEN 'longterm' ELSE 'negative' END AS profile,
CASE f.action
WHEN 'promoted' THEN 0.60::real
WHEN 'disliked' THEN -0.60::real
WHEN 'skipped' THEN -0.20::real
END AS weight
FROM feedback f
WHERE f.user_id = $1
AND f.track_id IS NOT NULL
AND f.action IN ('promoted', 'disliked', 'skipped')
) interactions
ORDER BY occurred_at ASC`,
[userId]
);
let beliefs = 0;
for (const interaction of interactionRes.rows as Array<{
track_id: string;
signal: string;
profile: string;
weight: number;
}>) {
const targets = await this.getTrackBeliefTargets(interaction.track_id, client);
for (const target of targets) {
await this.updateListenerBelief({
user_id: userId,
profile: interaction.profile,
entity_type: target.entity_type,
entity_id: target.entity_id,
dimension: this.beliefDimensionForSignal(interaction.signal),
value_delta: interaction.weight * target.factor,
confidence_delta: 0.05,
}, client);
beliefs++;
}
}
return { interactions: interactionRes.rows.length, beliefs };
});
}
/**
* Get listener beliefs for a user, optionally filtered by profile/entity.
*/
@@ -1395,4 +1620,3 @@ export class DbService {
return res.rowCount ?? 0;
}
}
@@ -0,0 +1,47 @@
import { describe, expect, it, vi } from 'vitest';
import { DiscoveryService } from './discovery.service.js';
import type { DbService } from './db.service.js';
function makeDb() {
const query = vi.fn();
return {
pgClient: { query },
getListenerBeliefs: vi.fn(),
upsertClaim: vi.fn().mockResolvedValue('claim-id'),
} as unknown as DbService;
}
describe('DiscoveryService System E provenance', () => {
it('writes graph-walk claims against discovery_candidate using registered graph provenance', async () => {
const db = makeDb();
(db.getListenerBeliefs as any).mockResolvedValue([
{ entity_id: '11111111-1111-1111-1111-111111111111', value: 0.8 },
]);
(db.pgClient.query as any)
.mockResolvedValueOnce({ rows: [{ candidate_artist_id: '22222222-2222-2222-2222-222222222222' }] })
.mockResolvedValueOnce({ rows: [{ id: '33333333-3333-3333-3333-333333333333' }] });
await new DiscoveryService(db).walkGraphForDiscovery('00000000-0000-0000-0000-000000000000');
expect(db.upsertClaim).toHaveBeenCalledWith(expect.objectContaining({
subject_type: 'discovery_candidate',
source: 'graph_exploration',
}));
});
it('does not queue artist-only graph candidates without a vetted acquisition URL', async () => {
const db = makeDb();
(db.pgClient.query as any)
.mockResolvedValueOnce({ rows: [{ id: 'candidate-1', notes: { discovery_source: 'graph_exploration' } }] })
.mockResolvedValueOnce({ rows: [{ cnt: 0 }] })
.mockResolvedValueOnce({ rows: [{ object_id: 'artist-1', fused_value: 0.7 }] })
.mockResolvedValueOnce({ rows: [] });
const result = await new DiscoveryService(db).evalCandidates('00000000-0000-0000-0000-000000000000');
expect(result).toEqual([{
candidateId: 'candidate-1', shouldAcquire: false, reason: 'awaiting resolved acquisition source',
}]);
expect((db.pgClient.query as any).mock.calls.at(-1)[0]).toContain("status = 'awaiting_resolution'");
});
});
+55 -6
View File
@@ -83,12 +83,17 @@ export class DiscoveryService {
const dcId = dcRes.rows[0].id;
const relevance = Math.min(belief.value, 0.8);
// A candidate is not a track. Claims intentionally support generic
// entity types, so keeping this distinction prevents graph reads from
// treating a random candidate UUID as a library track UUID.
await this.db.upsertClaim({
subject_type: 'track',
subject_type: 'discovery_candidate',
subject_id: dcId,
predicate: 'discovery_candidate',
object_type: 'artist',
object_id: row.candidate_artist_id,
// Registered in source_trust by the System E migration. Previously
// this unregistered value violated claims.source's foreign key.
source: 'graph_exploration',
confidence: relevance,
raw: {
@@ -134,7 +139,7 @@ export class DiscoveryService {
const claimRes = await this.db.pgClient.query<{ object_id: string; fused_value: number }>(
`SELECT object_id, fused_value
FROM claim_fusion
WHERE subject_type = 'track' AND subject_id = $1::uuid
WHERE subject_type = 'discovery_candidate' AND subject_id = $1::uuid
AND predicate = 'discovery_candidate'
LIMIT 1`,
[row.id]
@@ -143,6 +148,24 @@ export class DiscoveryService {
const relevance = claimRes.rows[0]?.fused_value ?? 0;
const candidateArtistId = claimRes.rows[0]?.object_id;
// Graph walks identify artists, not a legal/downloadable recording. A
// resolver (or a human) must attach a vetted HTTPS source before the
// worker can acquire anything. Do not guess a search query and download
// an arbitrary track under an artist's name.
const notes = typeof row.notes === 'string' ? safeJson(row.notes) : row.notes;
const resolvedUrl = (notes as { acquisition?: { url?: unknown } } | null)?.acquisition?.url;
if (typeof resolvedUrl !== 'string' || resolvedUrl.trim() === '') {
await this.db.pgClient.query(
`UPDATE discovery_candidates
SET status = 'awaiting_resolution', last_eval_at = NOW(),
last_error = 'no vetted acquisition source attached'
WHERE id = $1`,
[row.id]
);
results.push({ candidateId: row.id, shouldAcquire: false, reason: 'awaiting resolved acquisition source' });
continue;
}
const noveltyBeliefs = await this.db.getListenerBeliefs({
userId,
profile: 'discovery',
@@ -160,6 +183,7 @@ export class DiscoveryService {
FROM discovery_candidates dc
JOIN claims c ON c.subject_id = dc.id
WHERE dc.status = 'acquiring'
AND c.subject_type = 'discovery_candidate'
AND c.predicate = 'discovery_candidate'
AND c.object_id = $1::uuid`,
[candidateArtistId]
@@ -200,6 +224,16 @@ export class DiscoveryService {
return results;
}
/** Undo the state transition when Redis rejected an acquisition enqueue. */
async markEnqueueFailed(candidateId: string, reason: string): Promise<void> {
await this.db.pgClient.query(
`UPDATE discovery_candidates
SET status = 'candidate', last_eval_at = NOW(), last_error = $2
WHERE id = $1 AND status = 'acquiring'`,
[candidateId, reason.slice(0, 2000)]
);
}
// ---------------------------------------------------------------
// E.4 — Probation lifecycle
// ---------------------------------------------------------------
@@ -226,13 +260,18 @@ export class DiscoveryService {
if (completedPlays >= 3) {
await this.db.pgClient.query(
`UPDATE tracks SET probation_status = 'retained' WHERE id = $1`,
// Retained recommendations become normal library candidates. Their
// source_type remains RECOMMENDATION for provenance, while state=LIBRARY
// is the explicit promotion gate consumed by the existing Vibe queries.
`UPDATE tracks
SET probation_status = 'retained', state = 'LIBRARY'
WHERE id = $1`,
[trackId]
);
const claimRes = await this.db.pgClient.query<{ source: string }>(
`SELECT source FROM claims
WHERE subject_type = 'track' AND subject_id = $1 AND predicate = 'discovery_candidate'
WHERE subject_type = 'track' AND subject_id = $1 AND predicate = 'acquired_from'
LIMIT 1`,
[trackId]
);
@@ -252,7 +291,9 @@ export class DiscoveryService {
if (completedPlays === 0 && skips >= 3 && daysSinceProbation > 7) {
await this.db.pgClient.query(
`UPDATE tracks SET probation_status = 'retired' WHERE id = $1`,
// Retirement is reversible and never unlinks a file. The existing
// cleanup hard-delete workflow remains separately gated.
`UPDATE tracks SET probation_status = 'retired', state = 'HIDDEN' WHERE id = $1`,
[trackId]
);
return 'retired';
@@ -286,7 +327,7 @@ export class DiscoveryService {
`SELECT c.source, COUNT(*)::int AS cnt
FROM claims c
JOIN tracks t ON t.id = c.subject_id
WHERE c.predicate = 'discovery_candidate'
WHERE c.predicate = 'acquired_from'
AND t.probation_status = 'retained'
GROUP BY c.source
ORDER BY cnt DESC`
@@ -295,3 +336,11 @@ export class DiscoveryService {
console.log('[MetaLearning] Discovery source retention counts:', JSON.stringify(res.rows));
}
}
function safeJson(value: string): unknown {
try {
return JSON.parse(value);
} catch {
return null;
}
}
+8 -8
View File
@@ -68,7 +68,7 @@ async function comfortGenerator(db: DbService, ctx: GeneratorContext): Promise<C
AND cf.object_type = 'artist'
AND cf.object_id = ANY($1::uuid[])
AND (cf.user_id = $2 OR cf.user_id = $3)
WHERE t.state = 'LIBRARY'
WHERE (t.state = 'LIBRARY' OR (t.state = 'RECOMMENDED' AND t.probation_status = 'probation'))
AND NOT (t.id = ANY($4::uuid[]))
) sub
WHERE sub.rn <= 2
@@ -141,7 +141,7 @@ async function adjacentGenerator(db: DbService, ctx: GeneratorContext): Promise<
AND cf.object_type = 'artist'
AND cf.object_id = ANY($1::uuid[])
AND (cf.user_id = $2 OR cf.user_id = $3)
WHERE t.state = 'LIBRARY'
WHERE (t.state = 'LIBRARY' OR (t.state = 'RECOMMENDED' AND t.probation_status = 'probation'))
AND NOT (t.id = ANY($4::uuid[]))
) sub
ORDER BY RANDOM()
@@ -209,7 +209,7 @@ async function discoveryGenerator(db: DbService, ctx: GeneratorContext): Promise
AND cf.predicate IN ('credited_main_on', 'featured_on')
AND cf.object_type = 'artist'
AND cf.object_id = ANY($1::uuid[])
WHERE t.state = 'LIBRARY'
WHERE (t.state = 'LIBRARY' OR (t.state = 'RECOMMENDED' AND t.probation_status = 'probation'))
AND NOT (t.id = ANY($2::uuid[]))
ORDER BY t.id, cf.fused_value DESC NULLS LAST
) sub
@@ -273,7 +273,7 @@ async function deepDiveGenerator(db: DbService, ctx: GeneratorContext): Promise<
ROW_NUMBER() OVER (PARTITION BY t.album_id ORDER BY t.title ASC) AS rn
FROM tracks t
WHERE t.album_id = ANY($1::uuid[])
AND t.state = 'LIBRARY'
AND (t.state = 'LIBRARY' OR (t.state = 'RECOMMENDED' AND t.probation_status = 'probation'))
AND NOT (t.id = ANY($2::uuid[]))
) sub
WHERE sub.rn <= 5
@@ -333,7 +333,7 @@ async function revivalGenerator(db: DbService, ctx: GeneratorContext): Promise<C
AND cf.object_type = 'artist'
AND cf.object_id = ANY($1::uuid[])
AND (cf.user_id = $2 OR cf.user_id = $3)
WHERE t.state = 'LIBRARY'
WHERE (t.state = 'LIBRARY' OR (t.state = 'RECOMMENDED' AND t.probation_status = 'probation'))
AND NOT (t.id = ANY($4::uuid[]))
ORDER BY t.id, cf.fused_value DESC NULLS LAST
) sub
@@ -385,7 +385,7 @@ async function noveltyGenerator(db: DbService, ctx: GeneratorContext): Promise<C
AND cf_track.object_id = cf_edge.object_id
WHERE t.release_date IS NOT NULL
AND t.release_date >= NOW() - INTERVAL '60 days'
AND t.state = 'LIBRARY'
AND (t.state = 'LIBRARY' OR (t.state = 'RECOMMENDED' AND t.probation_status = 'probation'))
AND NOT (t.id = ANY($1::uuid[]))
) sub
ORDER BY release_date DESC
@@ -435,7 +435,7 @@ async function experimentalGenerator(db: DbService, ctx: GeneratorContext): Prom
FROM tracks t
JOIN track_genre tg ON tg.track_id = t.id
JOIN unfamiliar_genres ug ON ug.id = tg.genre_id
WHERE t.state = 'LIBRARY'
WHERE (t.state = 'LIBRARY' OR (t.state = 'RECOMMENDED' AND t.probation_status = 'probation'))
AND NOT (t.id = ANY($2::uuid[]))
LIMIT 30
)
@@ -491,7 +491,7 @@ async function contextualGenerator(db: DbService, ctx: GeneratorContext): Promis
AND cf.object_type = 'artist'
AND cf.object_id = ANY($1::uuid[])
AND (cf.user_id = $2 OR cf.user_id = $3)
WHERE t.state = 'LIBRARY'
WHERE (t.state = 'LIBRARY' OR (t.state = 'RECOMMENDED' AND t.probation_status = 'probation'))
AND NOT (t.id = ANY($4::uuid[]))
ORDER BY t.id, cf.fused_value DESC NULLS LAST
) sub
+31 -4
View File
@@ -53,6 +53,19 @@ describe('generators', () => {
expect(results[0].explanation.length).toBeGreaterThanOrEqual(1);
});
it('accepts the fresh artist affinity shape produced by a promoted track', async () => {
const db = makeMockDb();
(db.pgClient.query as any).mockResolvedValue({ rows: [{ id: 'track-1', artist_id: 'artist-1' }] });
// recordTrackEvidence(add_to_favorites) projects 0.60 * 0.90 = 0.54
// onto a fresh longterm artist affinity belief.
const ctx = makeCtx({ beliefs: [
{ entity_type: 'artist', entity_id: 'artist-1', value: 0.54, confidence: 0.05, profile: 'longterm', dimension: 'affinity' } as any,
] });
const results = await generatorByName.comfort(db, ctx);
expect(results).toHaveLength(1);
expect(results[0].generatorId).toBe('comfort');
});
it('returns empty when no high-affinity artists', async () => {
const db = makeMockDb();
const ctx = makeCtx({ beliefs: [ { entity_type: 'artist', entity_id: 'a1', value: 0.3, profile: 'longterm', dimension: 'affinity' } as any ] });
@@ -128,16 +141,30 @@ describe('generators', () => {
});
});
describe('novelty', () => {
it('returns recent tracks', async () => {
it('returns a candidate when canonical release-date coverage makes a recent track eligible', async () => {
const db = makeMockDb();
(db.pgClient.query as any).mockResolvedValue({ rows: [{ id: 't1' }] });
const ctx = makeCtx({
beliefs: [ { entity_type: 'artist', entity_id: 'trusted-1', value: 0.5, profile: 'longterm', dimension: 'affinity' } as any ],
});
const results = await generatorByName.novelty(db, ctx);
if (results.length > 0) {
expect(results[0].generatorId).toBe('novelty');
}
expect(results).toHaveLength(1);
expect(results[0]).toMatchObject({ trackId: 't1', generatorId: 'novelty' });
});
it('queries only dated library tracks from the current novelty window', async () => {
const db = makeMockDb();
(db.pgClient.query as any).mockResolvedValue({ rows: [] });
const ctx = makeCtx({
beliefs: [ { entity_type: 'artist', entity_id: 'trusted-1', value: 0.5, profile: 'longterm', dimension: 'affinity' } as any ],
});
await generatorByName.novelty(db, ctx);
const [sql] = (db.pgClient.query as any).mock.calls[0];
expect(sql).toContain('t.release_date IS NOT NULL');
expect(sql).toContain("t.release_date >= NOW() - INTERVAL '60 days'");
expect(sql).toContain("t.state = 'LIBRARY'");
});
});
+161 -27
View File
@@ -1,5 +1,12 @@
import { Queue, Job } from 'bullmq';
import { MetadataRefreshJob, AudioAnalysisJob, CleanupJob, LibraryScanJob, ReindexTracksJob, ReprocessArtistsJob } from '../types/job.types.js';
import { MetadataRefreshJob, AudioAnalysisJob, CleanupJob, LibraryScanJob, ReindexTracksJob, ReprocessArtistsJob, AcquisitionJob } from '../types/job.types.js';
const AUDIO_ANALYSIS_JOB_OPTIONS = {
attempts: 3,
backoff: { type: 'exponential', delay: 5_000 },
removeOnComplete: { age: 7 * 24 * 60 * 60, count: 10_000 },
removeOnFail: { age: 30 * 24 * 60 * 60, count: 10_000 },
} as const;
export interface JobServiceConfig {
redisUrl: string;
@@ -26,6 +33,37 @@ export interface JobHistoryEntry {
returnvalue?: unknown;
}
export interface ReenrichmentQueuePlan {
metadataTrackIds: string[];
artistIds: string[];
albumIds: string[];
}
export interface EnqueueResult {
requested: number;
enqueued: number;
alreadyQueued: number;
failed: Array<{ id: string; error: string }>;
}
export interface ReenrichmentQueueResult {
metadata: EnqueueResult;
artistImages: EnqueueResult;
albumCovers: EnqueueResult;
}
export interface EnrichmentQueueDiagnostics {
pending: Record<string, number>;
recentFailures: Array<{
id: string;
name: string;
data: Record<string, unknown>;
failedReason?: string;
timestamp: number;
finishedOn?: number;
}>;
}
export class JobService {
private queue: Queue;
@@ -42,9 +80,12 @@ export class JobService {
await this.queue.add('metadata_refresh', payload);
}
async enqueueAudioAnalysis(trackId: string, features: string[]) {
const payload: AudioAnalysisJob = { trackId, features };
await this.queue.add('audio_analysis', payload);
async enqueueAudioAnalysis(trackId: string) {
const payload: AudioAnalysisJob = { trackId };
await this.queue.add('audio_analysis', payload, {
jobId: `audio-${trackId}`,
...AUDIO_ANALYSIS_JOB_OPTIONS,
});
}
async enqueueCleanup(reason: 'expired' | 'manual', targetFiles: string[]) {
@@ -67,30 +108,123 @@ export class JobService {
await this.queue.add('reprocess_artists', payload);
}
/**
* Enqueue metadata_refresh jobs for a batch of track IDs. Used by the
* /admin/reenrich-tracks endpoint to re-canonicalize metadata (artist names,
* album titles, MBIDs, cover art) without re-scanning files from disk.
*
* Each job is deduped by `jobId: meta-<trackId>` so re-running the endpoint
* doesn't stack duplicate jobs. Old completed/failed jobs with the same ID
* are removed first so re-enrichment actually works (BullMQ otherwise treats
* existing jobIds as duplicates and silently skips them).
*/
async enqueueMetadataRefreshBatch(trackIds: string[]): Promise<number> {
let enqueued = 0;
for (const trackId of trackIds) {
const jobId = `meta-${trackId}`;
await this.queue.remove(jobId).catch(() => {});
const payload: MetadataRefreshJob = { trackId, refreshType: 'full' };
await this.queue.add('metadata_refresh', payload, {
jobId,
removeOnComplete: { age: 86400, count: 10000 },
removeOnFail: { age: 86400, count: 10000 },
});
enqueued++;
async enqueueDiscoveryAcquisition(candidateId: string): Promise<void> {
const payload: AcquisitionJob = { candidateId };
const jobId = `acquire-${candidateId}`;
// BullMQ returns a retained completed/failed job for the same id without
// executing it. Remove terminal attempts before retrying so a candidate
// cannot be stranded in `acquiring` for the retention window.
const existing = await this.queue.getJob(jobId);
if (existing) {
const state = await existing.getState();
if (state === 'completed' || state === 'failed') await existing.remove();
}
return enqueued;
await this.queue.add('acquire_discovery_candidate', payload, {
// A candidate may have one active attempt. Failed/completed jobs expire
// so an operator can explicitly re-evaluate it later.
jobId,
removeOnComplete: { age: 86400, count: 1000 },
removeOnFail: { age: 86400, count: 1000 },
});
}
/**
* Queue a bounded re-enrichment plan. Metadata, artist images and album
* covers deliberately have separate jobs: metadata resolution can complete
* even when artwork providers are unavailable, and artwork is deduplicated
* at its natural artist/album boundary.
*
* Existing active/waiting jobs are preserved and reported as alreadyQueued.
* Completed/failed jobs are removed before retrying, so a request never
* claims a retry was enqueued when BullMQ retained an old job id.
*/
async enqueueReenrichment(plan: ReenrichmentQueuePlan): Promise<ReenrichmentQueueResult> {
return {
metadata: await this.enqueueMany(
[...new Set(plan.metadataTrackIds)],
'metadata_refresh',
(trackId) => ({ trackId, refreshType: 'full' }),
(trackId) => `meta-${trackId}`,
),
artistImages: await this.enqueueMany(
[...new Set(plan.artistIds)],
'artist_image',
(artistId) => ({ artistId }),
(artistId) => `artist-image-${artistId}`,
),
albumCovers: await this.enqueueMany(
[...new Set(plan.albumIds)],
'album_cover',
(albumId) => ({ albumId }),
(albumId) => `album-cover-${albumId}`,
),
};
}
private async enqueueMany(
ids: string[],
name: string,
payloadFor: (id: string) => Record<string, unknown>,
jobIdFor: (id: string) => string,
): Promise<EnqueueResult> {
const result: EnqueueResult = { requested: ids.length, enqueued: 0, alreadyQueued: 0, failed: [] };
const keep = {
removeOnComplete: { age: 86400, count: 10000 },
removeOnFail: { age: 86400, count: 10000 },
} as const;
for (const id of ids) {
const jobId = jobIdFor(id);
try {
const existing = await this.queue.getJob(jobId);
if (existing) {
const state = await existing.getState();
if (state === 'active' || state === 'waiting' || state === 'delayed' || state === 'prioritized') {
result.alreadyQueued++;
continue;
}
await existing.remove();
}
await this.queue.add(name, payloadFor(id), { jobId, ...keep });
result.enqueued++;
} catch (err) {
result.failed.push({ id, error: err instanceof Error ? err.message : String(err) });
}
}
return result;
}
async getEnrichmentQueueDiagnostics(limit = 25): Promise<EnrichmentQueueDiagnostics> {
const relevant = new Set(['metadata_refresh', 'artist_image', 'album_cover']);
const [waiting, active, delayed, failed] = await Promise.all([
this.queue.getJobs(['waiting'], 0, 1000),
this.queue.getJobs(['active'], 0, 1000),
this.queue.getJobs(['delayed'], 0, 1000),
this.queue.getJobs(['failed'], 0, Math.max(limit * 4, 100)),
]);
const pending: Record<string, number> = {
metadata_refresh: 0,
artist_image: 0,
album_cover: 0,
};
for (const job of [...waiting, ...active, ...delayed]) {
if (relevant.has(job.name)) pending[job.name]++;
}
const recentFailures = failed
.filter((job) => relevant.has(job.name))
.sort((a, b) => (b.finishedOn ?? b.timestamp) - (a.finishedOn ?? a.timestamp))
.slice(0, limit)
.map((job) => ({
id: String(job.id),
name: job.name,
data: job.data as Record<string, unknown>,
failedReason: job.failedReason,
timestamp: job.timestamp,
finishedOn: job.finishedOn,
}));
return { pending, recentFailures };
}
async getQueueStats(): Promise<QueueStats> {
+125 -12
View File
@@ -1,5 +1,6 @@
import { DbService, ListenerBelief } from './db.service.js';
import { Candidate, GeneratorContext, Generator, ALL_GENERATORS } from './generators.service.js';
import { AUDIO_PREFERENCE_BUCKETS } from '../db/types.js';
export interface FatigueState {
artist: Map<string, number>;
@@ -33,11 +34,43 @@ export interface RepetitionState {
recentArtistIds: Set<string>;
}
export interface PlanBuildOptions {
/** Tracks already exposed during this Vibe session; they are ineligible. */
excludedTrackIds?: Iterable<string>;
/** Main artists already served or rejected in this Vibe session. */
excludedArtistIds?: Iterable<string>;
}
const W_ENJOY = 1.0;
const W_FATIGUE = 0.4;
const W_DIVERSITY = 0.3;
const W_ENTROPY = 0.2;
const W_REPETITION = 0.5;
const PLAN_SIZE = 20;
/**
* Preserve the existing queue, append only genuinely new candidates, and
* never emit a duplicate. This is deliberately pure so the queue boundary is
* testable without a database.
*/
export function mergeUniquePlan(
currentPlan: Candidate[],
additions: Candidate[],
excludedTrackIds: Iterable<string> = [],
targetSize = PLAN_SIZE
): Candidate[] {
const seen = new Set(excludedTrackIds);
const merged: Candidate[] = [];
for (const candidate of [...currentPlan, ...additions]) {
if (seen.has(candidate.trackId)) continue;
seen.add(candidate.trackId);
merged.push(candidate);
if (merged.length >= targetSize) break;
}
return merged;
}
export class SessionDirector {
constructor(private db: DbService) {}
@@ -98,7 +131,22 @@ export class SessionDirector {
WHERE taf.energy IS NOT NULL`,
[userId]
);
const energy = (energyRes.rows[0]?.energy as number) ?? 0.5;
let energy = (energyRes.rows[0]?.energy as number) ?? 0.5;
// Audio beliefs are projected from track feedback. Blend the strongest
// energy preference with the recent-play state so the projection affects
// the session without making a single old preference a hard constraint.
const energyBelief = await this.db.pgClient.query<{ entity_id: string; value: number }>(
`SELECT entity_id, value FROM listener_beliefs
WHERE user_id = $1 AND entity_type = 'audio' AND entity_id = ANY($2::uuid[])
ORDER BY value DESC LIMIT 1`,
[userId, [AUDIO_PREFERENCE_BUCKETS.energy.low, AUDIO_PREFERENCE_BUCKETS.energy.medium, AUDIO_PREFERENCE_BUCKETS.energy.high]]
);
const preferredEnergy = energyBelief.rows[0]?.entity_id === AUDIO_PREFERENCE_BUCKETS.energy.low ? 0.2
: energyBelief.rows[0]?.entity_id === AUDIO_PREFERENCE_BUCKETS.energy.high ? 0.8
: energyBelief.rows[0]?.entity_id === AUDIO_PREFERENCE_BUCKETS.energy.medium ? 0.5 : null;
if (preferredEnergy !== null && (energyBelief.rows[0]?.value ?? 0) > 0) {
energy = energy * 0.65 + preferredEnergy * 0.35;
}
// Read novelty hunger from discovery profile
const noveltyRes = await this.db.pgClient.query(
@@ -783,8 +831,21 @@ export class SessionDirector {
// ---------------------------------------------------------------
// D.9 — Plan + replan loop
// ---------------------------------------------------------------
async buildPlan(userId: string, sessionId: string, seedTrackId?: string): Promise<Candidate[]> {
const allBeliefs = await this.db.getListenerBeliefs({ userId, limit: 200 });
async buildPlan(
userId: string,
sessionId: string,
seedTrackId?: string,
options: PlanBuildOptions = {}
): Promise<Candidate[]> {
// Do not let abundant track-level beliefs crowd out the artist/genre
// affinities required by the discovery generators.
const beliefGroups = await Promise.all([
this.db.getListenerBeliefs({ userId, entityType: 'track', limit: 100 }),
this.db.getListenerBeliefs({ userId, entityType: 'artist', limit: 100 }),
this.db.getListenerBeliefs({ userId, entityType: 'genre', limit: 75 }),
this.db.getListenerBeliefs({ userId, entityType: 'audio', limit: 50 }),
]);
const allBeliefs = beliefGroups.flat();
// Fetch recent completed plays for anti-loop detection
const recentPlaysRes = await this.db.pgClient.query(
@@ -823,7 +884,7 @@ export class SessionDirector {
const budgets = await this.getBudgets(userId);
const arcType = this.pickArc(state);
const planSize = 20;
const planSize = PLAN_SIZE;
const slots = this.getArcSlots(arcType, planSize);
let seedArtistId: string | null = null;
@@ -831,7 +892,16 @@ export class SessionDirector {
seedArtistId = await this.resolveSeedArtistId(seedTrackId) ?? null;
}
const recentExclusions: string[] = recentPlays.map(p => p.trackId);
// Every generator receives this as a SQL exclusion list. It is a hard
// boundary, not a score penalty: a served/skipped track cannot return in
// a replacement plan while its Vibe session is active.
const recentExclusionSet = new Set<string>([
...recentPlays.map(p => p.trackId),
...(options.excludedTrackIds ?? []),
]);
if (seedTrackId) recentExclusionSet.add(seedTrackId);
const recentExclusions = [...recentExclusionSet];
const excludedArtistIds = new Set(options.excludedArtistIds ?? []);
const toleranceMap: Record<string, number> = {};
const discoveryBeliefs = allBeliefs.filter(b => b.profile === 'discovery');
for (const b of discoveryBeliefs) {
@@ -859,8 +929,24 @@ export class SessionDirector {
}
const repetitionState = await this.buildRepetitionState(userId);
const candidateArtistMap = await this.loadArtistMap(
[...new Set(allCandidates.map(c => c.trackId))]
);
// Artist cooldown used to be a 0.1x score multiplier. In a small pool it
// still selected the same artist, so enforce the configured window before
// ranking instead.
const eligibleCandidates = allCandidates.filter(candidate =>
!recentExclusionSet.has(candidate.trackId) &&
!repetitionState.recentTrackIds.has(candidate.trackId) &&
!repetitionState.recentArtistIds.has(candidateArtistMap.get(candidate.trackId) ?? '') &&
!excludedArtistIds.has(candidateArtistMap.get(candidate.trackId) ?? '')
);
if (eligibleCandidates.length === 0) {
return [];
}
const ranked = await this.rankCandidates(
allCandidates, fatigue, budgets, state, repetitionState
eligibleCandidates, fatigue, budgets, state, repetitionState
);
const loopDim = await this.detectAntiLoop(state, fatigue, budgets, recentPlays);
@@ -928,10 +1014,21 @@ export class SessionDirector {
sessionId: string,
currentPlan: Candidate[],
playedTrackIds: string[],
seedTrackId?: string
seedTrackId?: string,
options: PlanBuildOptions = {}
): Promise<Candidate[]> {
const remainingSlots = currentPlan.filter(
c => !playedTrackIds.includes(c.trackId)
const excludedTrackIds = new Set<string>([
...playedTrackIds,
...(options.excludedTrackIds ?? []),
]);
// Corrupt/legacy Redis plans can contain duplicates. Clean those before
// deciding whether the tail needs a refill, and never reintroduce a track
// reported as played/skipped/disliked by the route.
const remainingSlots = mergeUniquePlan(
currentPlan,
[],
excludedTrackIds,
PLAN_SIZE
);
if (remainingSlots.length >= 10 && currentPlan.length > 0) {
@@ -973,19 +1070,35 @@ export class SessionDirector {
const loopDim = await this.detectAntiLoop(state, fatigue, budgets, recentPlays);
if (loopDim) {
return this.buildPlan(userId, sessionId, seedTrackId);
const rebuilt = await this.buildPlan(userId, sessionId, seedTrackId, {
excludedTrackIds: new Set([...excludedTrackIds, ...remainingSlots.map(c => c.trackId)]),
excludedArtistIds: options.excludedArtistIds,
});
return mergeUniquePlan(remainingSlots, rebuilt, excludedTrackIds, PLAN_SIZE);
}
const planArtistMap = await this.loadArtistMap([...new Set(currentPlan.map(c => c.trackId))]);
const entropy = this.computeEntropy(currentPlan, c => planArtistMap.get(c.trackId) ?? 'unknown');
if (Math.abs(entropy - 0.55) > 0.2) {
return this.buildPlan(userId, sessionId, seedTrackId);
const rebuilt = await this.buildPlan(userId, sessionId, seedTrackId, {
excludedTrackIds: new Set([...excludedTrackIds, ...remainingSlots.map(c => c.trackId)]),
excludedArtistIds: options.excludedArtistIds,
});
return mergeUniquePlan(remainingSlots, rebuilt, excludedTrackIds, PLAN_SIZE);
}
return remainingSlots;
}
return this.buildPlan(userId, sessionId, seedTrackId);
// Build replacements against both session history and the queue tail. The
// old code rebuilt from completed plays only, then locally de-duped at the
// route — yielding an empty tail when every returned candidate was already
// queued. Keep the valid tail and append only fresh candidates.
const rebuilt = await this.buildPlan(userId, sessionId, seedTrackId, {
excludedTrackIds: new Set([...excludedTrackIds, ...remainingSlots.map(c => c.trackId)]),
excludedArtistIds: options.excludedArtistIds,
});
return mergeUniquePlan(remainingSlots, rebuilt, excludedTrackIds, PLAN_SIZE);
}
// ---------------------------------------------------------------
+69 -1
View File
@@ -1,5 +1,5 @@
import { describe, it, expect, vi } from 'vitest';
import { SessionDirector } from './session-director.service.js';
import { mergeUniquePlan, SessionDirector } from './session-director.service.js';
import { DbService } from './db.service.js';
function makeMockDb(overrides: Record<string, any> = {}): DbService {
@@ -15,6 +15,74 @@ function makeMockDb(overrides: Record<string, any> = {}): DbService {
}
describe('SessionDirector', () => {
const candidate = (trackId: string) => ({
trackId,
generatorId: 'test',
relevance: 1,
explanation: [],
});
describe('session queue exclusions', () => {
it('keeps more than 100 unique selections without cycling', () => {
const source = Array.from({ length: 125 }, (_, index) => candidate(`track-${index}`));
const plan = mergeUniquePlan([], source, [], 125);
expect(plan).toHaveLength(125);
expect(new Set(plan.map(item => item.trackId)).size).toBe(125);
});
it('drops rapid skips and duplicate-only refill candidates', async () => {
const director = new SessionDirector(makeMockDb());
const buildPlan = vi.spyOn(director, 'buildPlan').mockResolvedValue([
candidate('skipped'),
candidate('already-queued'),
candidate('fresh'),
candidate('fresh'),
]);
const plan = await director.replan(
'user-1',
'session-1',
[candidate('skipped'), candidate('already-queued'), candidate('already-queued')],
['skipped'],
undefined,
{ excludedTrackIds: ['older-skip', 'skipped'] }
);
expect(plan.map(item => item.trackId)).toEqual(['already-queued', 'fresh']);
expect(buildPlan).toHaveBeenCalledWith(
'user-1',
'session-1',
undefined,
expect.objectContaining({
excludedTrackIds: expect.any(Set),
})
);
const refillExclusions = (buildPlan.mock.calls[0][3] as any).excludedTrackIds as Set<string>;
expect(refillExclusions).toEqual(new Set(['older-skip', 'skipped', 'already-queued']));
});
it('does not append anything when a refill contains only queued or excluded tracks', async () => {
const director = new SessionDirector(makeMockDb());
vi.spyOn(director, 'buildPlan').mockResolvedValue([
candidate('queued'),
candidate('skipped'),
candidate('queued'),
]);
const plan = await director.replan(
'user-1',
'session-1',
[candidate('queued')],
['skipped'],
undefined,
{ excludedTrackIds: ['skipped'] }
);
expect(plan.map(item => item.trackId)).toEqual(['queued']);
});
});
describe('pickArc', () => {
const director = new SessionDirector(makeMockDb());
+9 -2
View File
@@ -5,7 +5,6 @@ export interface MetadataRefreshJob {
export interface AudioAnalysisJob {
trackId: string;
features: string[];
}
export interface CleanupJob {
@@ -24,4 +23,12 @@ export interface ReprocessArtistsJob {
offset?: number;
}
export type JobPayload = MetadataRefreshJob | AudioAnalysisJob | CleanupJob | LibraryScanJob | ReindexTracksJob | ReprocessArtistsJob;
/**
* Explicit, opt-in System E hand-off. The worker looks the candidate up again
* from Postgres; no URL or shell arguments travel through Redis.
*/
export interface AcquisitionJob {
candidateId: string;
}
export type JobPayload = MetadataRefreshJob | AudioAnalysisJob | CleanupJob | LibraryScanJob | ReindexTracksJob | ReprocessArtistsJob | AcquisitionJob;