feat: enhance discovery, vibe sessions, and library enrichment
Typecheck / typecheck (backend) (push) Has been cancelled
Typecheck / typecheck (workers) (push) Has been cancelled

This commit is contained in:
kami
2026-08-01 14:40:48 +04:00
parent a0c9f42a89
commit 4c48d11e9d
54 changed files with 4136 additions and 521 deletions
+260 -10
View File
@@ -2,6 +2,87 @@ import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
import { JobService } from '../services/job.service.js';
import { DbService } from '../services/db.service.js';
const ENRICHMENT_SETTING_KEYS = [
'enrich_metadata',
'enrich_artist_images',
'enrich_cover_art',
'enrich_genres',
'enrich_lyrics',
'enrich_artist_similarity',
'enrich_audio_analysis',
] as const;
type ReenrichmentScope = {
metadata?: boolean;
artistImages?: boolean;
albumCovers?: boolean;
};
type ReenrichmentRequest = {
/** Preview by default. Jobs are only added with an explicit confirmation. */
confirm?: boolean;
/** Missing-only is the safe default; false means refresh the selected scope. */
missingOnly?: boolean;
/** Per-scope batch cap. Defaults to 250 and never exceeds 1,000. */
limit?: number;
scope?: ReenrichmentScope;
};
function getReenrichmentOptions(body: ReenrichmentRequest = {}) {
const requestedScope = body.scope ?? {};
const scope = {
metadata: requestedScope.metadata ?? true,
artistImages: requestedScope.artistImages ?? true,
albumCovers: requestedScope.albumCovers ?? true,
};
const rawLimit = Number(body.limit ?? 250);
const limit = Number.isInteger(rawLimit) ? Math.max(1, Math.min(rawLimit, 1000)) : 250;
return { confirm: body.confirm === true, missingOnly: body.missingOnly !== false, limit, scope };
}
async function getReenrichmentStatus(dbService: DbService, jobService: JobService) {
const db = dbService.pgClient;
const [settingsRes, coverageRes, queue] = await Promise.all([
db.query<{ key: string; value: string }>(
`SELECT key, value FROM settings WHERE key = ANY($1::text[])`,
[ENRICHMENT_SETTING_KEYS],
),
db.query<{
library_tracks: number;
tracks_without_release_date: number;
tracks_without_album_mbid: number;
artists_total: number;
artists_without_image: number;
artists_without_mbid: number;
albums_total: number;
albums_without_artwork: number;
albums_without_release_date: number;
}>(`
SELECT
COUNT(*) FILTER (WHERE t.state = 'LIBRARY')::int AS library_tracks,
COUNT(*) FILTER (WHERE t.state = 'LIBRARY' AND t.release_date IS NULL)::int AS tracks_without_release_date,
COUNT(*) FILTER (WHERE t.state = 'LIBRARY' AND al.mbid IS NULL)::int AS tracks_without_album_mbid,
(SELECT COUNT(*)::int FROM artists) AS artists_total,
(SELECT COUNT(*)::int FROM artists WHERE image_path IS NULL OR image_path = '') AS artists_without_image,
(SELECT COUNT(*)::int FROM artists WHERE mbid IS NULL) AS artists_without_mbid,
(SELECT COUNT(*)::int FROM albums) AS albums_total,
(SELECT COUNT(*)::int FROM albums WHERE artwork_id IS NULL OR artwork_id = '') AS albums_without_artwork,
(SELECT COUNT(*)::int FROM albums WHERE release_date IS NULL) AS albums_without_release_date
FROM tracks t
LEFT JOIN albums al ON al.id = t.album_id
`),
jobService.getEnrichmentQueueDiagnostics(),
]);
const settings = Object.fromEntries(
ENRICHMENT_SETTING_KEYS.map((key) => [
key,
settingsRes.rows.find((row) => row.key === key)?.value === 'true',
]),
) as Record<(typeof ENRICHMENT_SETTING_KEYS)[number], boolean>;
return { settings, coverage: coverageRes.rows[0], queue };
}
export default async function adminRoutes(fastify: FastifyInstance, options: { jobService: JobService; dbService: DbService }) {
const { jobService, dbService } = options;
@@ -24,6 +105,60 @@ export default async function adminRoutes(fastify: FastifyInstance, options: { j
return { status: 'Artist reprocessing job enqueued' };
});
/** Rebuild artist/genre/audio beliefs from durable history after a model upgrade. */
fastify.post('/vibe/rebuild-beliefs', async (request: FastifyRequest, reply: FastifyReply) => {
const { userId } = (request.body ?? {}) as { userId?: string };
const resolvedUserId = userId || (request.headers['x-user-id'] as string) || '00000000-0000-0000-0000-000000000000';
const result = await dbService.rebuildDerivedListenerBeliefs(resolvedUserId);
return reply.send({ status: 'rebuilt', userId: resolvedUserId, ...result });
});
/**
* Attach a human/resolver-vetted source to a System E candidate. This is an
* admin-only hand-off: graph traversal identifies an artist/path, but must
* never turn that into an arbitrary web search and download. The worker still
* applies its own host allow-list immediately before invoking yt-dlp.
*/
fastify.post('/discovery/candidates/:id/acquisition-source', async (request, reply) => {
const { id } = request.params as { id: string };
const body = request.body as { url?: string; expectedTitle?: string; expectedArtist?: string };
if (!body?.url || typeof body.url !== 'string' || body.url.length > 4000) {
return reply.code(400).send({ error: 'HTTPS url is required' });
}
let url: URL;
try {
url = new URL(body.url);
} catch {
return reply.code(400).send({ error: 'url is invalid' });
}
if (url.protocol !== 'https:' || url.username || url.password) {
return reply.code(400).send({ error: 'url must be credential-free HTTPS' });
}
const source = {
url: url.toString(),
...(typeof body.expectedTitle === 'string' ? { expectedTitle: body.expectedTitle.slice(0, 500) } : {}),
...(typeof body.expectedArtist === 'string' ? { expectedArtist: body.expectedArtist.slice(0, 500) } : {}),
};
try {
const result = await dbService.pgClient.query(
`UPDATE discovery_candidates
SET notes = jsonb_set(COALESCE(notes, '{}'::jsonb), '{acquisition}', $2::jsonb, true),
status = 'candidate', last_eval_at = NULL, last_error = NULL
WHERE id = $1::uuid
AND status IN ('candidate', 'awaiting_resolution', 'acquisition_disabled', 'failed')
RETURNING id, status`,
[id, JSON.stringify(source)]
);
if (result.rows.length === 0) {
return reply.code(404).send({ error: 'candidate does not exist or cannot be re-queued' });
}
return reply.send({ candidate: result.rows[0] });
} catch (err) {
const message = err instanceof Error ? err.message : 'could not attach acquisition source';
return reply.code(400).send({ error: message });
}
});
fastify.post('/dedup-albums', async (_request: FastifyRequest, reply: FastifyReply) => {
// Merge duplicate album rows directly (synchronous — it's just SQL, no
// external API calls). Returns the number of albums merged away.
@@ -92,17 +227,132 @@ export default async function adminRoutes(fastify: FastifyInstance, options: { j
return { status: 'Albums deduplicated', merged: res.rows[0]?.count ?? 0 };
});
/**
* Return coverage, active toggles, queued work, and recent worker failures.
* This is deliberately separate from enqueueing so an operator can diagnose
* a disabled provider or a failing worker before launching another batch.
*/
fastify.get('/reenrichment/status', async () => {
return getReenrichmentStatus(dbService, jobService);
});
/**
* Safe re-enrichment control plane. A call is a preview unless confirm=true;
* the default is a 250-entity, missing-only batch. The three job types remain
* independent so artwork work is not hidden behind track metadata work.
*/
fastify.post<{ Body: ReenrichmentRequest }>('/reenrichment', async (request, reply) => {
const options = getReenrichmentOptions(request.body);
const status = await getReenrichmentStatus(dbService, jobService);
const blockedBySettings: Partial<Record<keyof ReenrichmentScope, string>> = {};
const metadataEnabled = status.settings.enrich_metadata
|| status.settings.enrich_genres
|| status.settings.enrich_lyrics;
if (options.scope.metadata && !metadataEnabled) {
blockedBySettings.metadata = 'All track enrichment toggles are disabled.';
}
if (options.scope.artistImages && !status.settings.enrich_artist_images) {
blockedBySettings.artistImages = 'enrich_artist_images is disabled.';
}
if (options.scope.albumCovers && !status.settings.enrich_cover_art) {
blockedBySettings.albumCovers = 'enrich_cover_art is disabled.';
}
const db = dbService.pgClient;
const [tracks, artists, albums] = await Promise.all([
options.scope.metadata && !blockedBySettings.metadata
? db.query<{ id: string }>(
`SELECT t.id
FROM tracks t
LEFT JOIN albums al ON al.id = t.album_id
LEFT JOIN track_artists ta ON ta.track_id = t.id AND ta.role = 'main'
LEFT JOIN artists ar ON ar.id = ta.artist_id
WHERE t.state = 'LIBRARY'
AND ($1::boolean = false OR t.release_date IS NULL OR al.mbid IS NULL OR ar.mbid IS NULL)
ORDER BY t.id
LIMIT $2`,
[options.missingOnly, options.limit],
)
: Promise.resolve({ rows: [] as Array<{ id: string }> }),
options.scope.artistImages && !blockedBySettings.artistImages
? db.query<{ id: string }>(
`SELECT id FROM artists
WHERE $1::boolean = false OR image_path IS NULL OR image_path = ''
ORDER BY id
LIMIT $2`,
[options.missingOnly, options.limit],
)
: Promise.resolve({ rows: [] as Array<{ id: string }> }),
options.scope.albumCovers && !blockedBySettings.albumCovers
? db.query<{ id: string }>(
`SELECT id FROM albums
WHERE $1::boolean = false OR artwork_id IS NULL OR artwork_id = ''
ORDER BY id
LIMIT $2`,
[options.missingOnly, options.limit],
)
: Promise.resolve({ rows: [] as Array<{ id: string }> }),
]);
const plan = {
metadataTrackIds: tracks.rows.map((row) => row.id),
artistIds: artists.rows.map((row) => row.id),
albumIds: albums.rows.map((row) => row.id),
};
const wouldQueue = {
metadata: plan.metadataTrackIds.length,
artistImages: plan.artistIds.length,
albumCovers: plan.albumIds.length,
};
if (!options.confirm) {
return {
status: 'preview',
message: 'No jobs were queued. Repeat with confirm=true to enqueue this bounded plan.',
options,
blockedBySettings,
wouldQueue,
diagnostics: status,
};
}
if (Object.keys(blockedBySettings).length === 3 ||
(wouldQueue.metadata + wouldQueue.artistImages + wouldQueue.albumCovers === 0)) {
return reply.code(409).send({
status: 'not_queued',
message: Object.keys(blockedBySettings).length === 3
? 'Every selected scope is disabled by enrichment settings.'
: 'No eligible entities matched this re-enrichment batch.',
options,
blockedBySettings,
wouldQueue,
diagnostics: status,
});
}
const queued = await jobService.enqueueReenrichment(plan);
const enqueueFailures = queued.metadata.failed.length
+ queued.artistImages.failed.length
+ queued.albumCovers.failed.length;
return {
status: enqueueFailures === 0 ? 'queued' : 'partially_queued',
message: enqueueFailures === 0
? 'Re-enrichment jobs were queued. Check /reenrichment/status and job history for outcomes.'
: 'Some jobs could not be queued; see queued.*.failed for exact errors.',
options,
blockedBySettings,
queued,
diagnostics: await getReenrichmentStatus(dbService, jobService),
};
});
// The prior endpoint queued every track and implied cover/image work that it
// never scheduled. Keep the failure explicit instead of silently claiming a
// successful library-wide refresh.
fastify.post('/reenrich-tracks', async (_request: FastifyRequest, reply: FastifyReply) => {
// Re-enqueue metadata_refresh for every LIBRARY track without re-reading
// files from disk. This re-runs the MusicBrainz canonicalisation (artist
// names, album titles, MBIDs) and re-triggers album_cover jobs — much
// faster than a full scan when only metadata needs refreshing.
const res = await dbService.pgClient.query<{ id: string }>(
`SELECT id FROM tracks WHERE state = 'LIBRARY' ORDER BY id`
);
const trackIds = res.rows.map((r) => r.id);
const enqueued = await jobService.enqueueMetadataRefreshBatch(trackIds);
return { status: 'Re-enrich enqueued', trackCount: enqueued };
return reply.code(410).send({
error: 'Deprecated endpoint. Use POST /admin/reenrichment (preview first; confirm=true to queue).',
});
});
fastify.get('/queue-stats', async () => {
+14 -1
View File
@@ -2,9 +2,11 @@ import { FastifyInstance } from 'fastify';
import { DbService } from '../services/db.service.js';
import { DiscoveryService } from '../services/discovery.service.js';
import { ImageEnrichmentService } from '../services/image-enrichment.service.js';
import { JobService } from '../services/job.service.js';
export default async function discoveryRoutes(fastify: FastifyInstance, options: { dbService: DbService }) {
export default async function discoveryRoutes(fastify: FastifyInstance, options: { dbService: DbService; jobService: JobService }) {
const { dbService } = options;
const { jobService } = options;
const discovery = new DiscoveryService(dbService);
const images = new ImageEnrichmentService(dbService);
@@ -39,6 +41,17 @@ export default async function discoveryRoutes(fastify: FastifyInstance, options:
fastify.post('/discovery/eval', async (request, reply) => {
const userId = (request.headers['x-user-id'] as string) || '00000000-0000-0000-0000-000000000000';
const results = await discovery.evalCandidates(userId);
for (const result of results) {
if (!result.shouldAcquire) continue;
try {
await jobService.enqueueDiscoveryAcquisition(result.candidateId);
} catch (err) {
const reason = err instanceof Error ? err.message : 'failed to enqueue acquisition';
await discovery.markEnqueueFailed(result.candidateId, reason);
result.shouldAcquire = false;
result.reason = `queue unavailable: ${reason}`;
}
}
return reply.send({ evaluated: results.length, results });
});
+1
View File
@@ -3,6 +3,7 @@ import { DbService } from '../services/db.service.js';
const SETTING_KEYS = [
'enrich_metadata',
'enrich_artist_images',
'enrich_cover_art',
'enrich_genres',
'enrich_lyrics',
+114 -12
View File
@@ -9,14 +9,42 @@ interface ActivePlan {
sessionId: string;
plan: Candidate[];
seedTrackId: string | null;
/** Tracks handed to the player during this Redis-backed session. */
servedTrackIds?: string[];
/** Explicit feedback targets (skip, dislike, completion, promotion). */
excludedTrackIds?: string[];
/** Main artists served or explicitly rejected in this session. */
excludedArtistIds?: string[];
}
const PLAN_TTL_SEC = 2 * 3600;
// Retain enough history for long listening sessions without allowing an
// unbounded Redis value if a client leaves a session running for days.
const MAX_SESSION_EXCLUSIONS = 1000;
function planKey(userId: string): string {
return `v2:plan:${userId}`;
}
function appendUniqueTrackId(ids: string[] | undefined, trackId: string): string[] {
const next = ids ? [...ids] : [];
if (!next.includes(trackId)) next.push(trackId);
return next.length > MAX_SESSION_EXCLUSIONS
? next.slice(next.length - MAX_SESSION_EXCLUSIONS)
: next;
}
function sessionExclusions(active: ActivePlan): string[] {
return [...new Set([
...(active.servedTrackIds ?? []),
...(active.excludedTrackIds ?? []),
])];
}
function sessionArtistExclusions(active: ActivePlan): string[] {
return [...new Set(active.excludedArtistIds ?? [])];
}
export default async function v2Routes(fastify: FastifyInstance, options: { dbService: DbService; sessionDirector: SessionDirector }) {
const { dbService, sessionDirector: director } = options;
@@ -47,6 +75,7 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
// span multiple keys/services atomically, swap this for a WATCH/MULTI transaction
// or move the plan into a single Lua script instead of an app-level lock.
const RELEASE_LOCK_LUA = `if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end`;
const RENEW_LOCK_LUA = `if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end`;
async function withPlanLock<T>(userId: string, fn: () => Promise<T>): Promise<T> {
const lockKey = `v2:planlock:${userId}`;
@@ -54,16 +83,20 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
const deadline = Date.now() + 5000;
let acquired = false;
while (Date.now() < deadline) {
const res = await redisClient.set(lockKey, token, { NX: true, PX: 3000 });
const res = await redisClient.set(lockKey, token, { NX: true, PX: 5000 });
if (res) { acquired = true; break; }
await new Promise((r) => setTimeout(r, 20 + Math.random() * 30));
}
if (!acquired) {
throw new Error('Timed out waiting for active-plan lock');
}
const renewal = setInterval(() => {
void redisClient.eval(RENEW_LOCK_LUA, { keys: [lockKey], arguments: [token, '5000'] });
}, 1500);
try {
return await fn();
} finally {
clearInterval(renewal);
await redisClient.eval(RELEASE_LOCK_LUA, { keys: [lockKey], arguments: [token] });
}
}
@@ -78,9 +111,18 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
const { seedTrackId } = request.body as { seedTrackId?: string };
const sessionId = await dbService.createSessionState(userId, undefined, { energy: 0.5, novelty_hunger: 0.3 });
const plan = await director.buildPlan(userId, sessionId, seedTrackId);
const initialExclusions = seedTrackId ? [seedTrackId] : [];
const plan = await director.buildPlan(userId, sessionId, seedTrackId, {
excludedTrackIds: initialExclusions,
});
await setActivePlan(userId, { sessionId, plan, seedTrackId: seedTrackId ?? null });
await setActivePlan(userId, {
sessionId,
plan,
seedTrackId: seedTrackId ?? null,
servedTrackIds: [],
excludedTrackIds: initialExclusions,
});
return reply.send({ sessionId, plan: plan.slice(0, 10) });
});
@@ -90,34 +132,71 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
*/
fastify.get('/v2/vibe/next', async (request, reply) => {
const userId = (request.headers['x-user-id'] as string) || '00000000-0000-0000-0000-000000000000';
const { sessionId } = request.query as { sessionId?: string };
const result = await withPlanLock(userId, async () => {
const active = await getActivePlan(userId);
if (!active || active.plan.length === 0) {
return null;
if (!active) {
return { kind: 'missing' as const };
}
if (!sessionId || active.sessionId !== sessionId) {
return { kind: 'replaced' as const };
}
if (active.plan.length === 0) {
return { kind: 'exhausted' as const };
}
const next = active.plan.shift()!;
active.servedTrackIds = appendUniqueTrackId(active.servedTrackIds, next.trackId);
const artistResult = await dbService.pgClient.query<{ artist_id: string }>(
`SELECT artist_id FROM track_artists_v2
WHERE track_id = $1 AND role = 'main'
ORDER BY confidence DESC NULLS LAST LIMIT 1`,
[next.trackId]
);
if (artistResult.rows[0]?.artist_id) {
active.excludedArtistIds = appendUniqueTrackId(active.excludedArtistIds, artistResult.rows[0].artist_id);
}
// Enrich with track details
const track = await dbService.getTrackById(next.trackId);
// Replan if running low
if (active.plan.length < 5) {
const refill = await director.replan(userId, active.sessionId, active.plan, [next.trackId], active.seedTrackId ?? undefined);
const refill = await director.replan(
userId,
active.sessionId,
active.plan,
[next.trackId],
active.seedTrackId ?? undefined,
{ excludedTrackIds: sessionExclusions(active), excludedArtistIds: sessionArtistExclusions(active) }
);
active.plan = refill;
}
await setActivePlan(userId, active);
return { track, explanation: next.explanation, planRemaining: active.plan.length };
return { kind: 'track' as const, track, explanation: next.explanation, planRemaining: active.plan.length };
});
if (!result) {
if (result.kind === 'missing') {
return reply.code(404).send({ error: 'No active plan. POST /api/v2/vibe/start first.' });
}
if (result.kind === 'replaced') {
return reply.code(409).send({ error: 'This Vibe session was replaced by a newer session.', code: 'VIBE_SESSION_REPLACED' });
}
if (result.kind === 'exhausted') {
return reply.code(409).send({
error: 'Vibe plan exhausted: no eligible unserved tracks remain for this session.',
code: 'VIBE_PLAN_EXHAUSTED',
});
}
return reply.send(result);
return reply.send({
track: result.track,
explanation: result.explanation,
planRemaining: result.planRemaining,
});
});
/**
@@ -126,11 +205,14 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
*/
fastify.post('/v2/vibe/feedback', async (request, reply) => {
const userId = (request.headers['x-user-id'] as string) || '00000000-0000-0000-0000-000000000000';
const { trackId, action } = request.body as { trackId: string; action: string };
const { trackId, action, sessionId } = request.body as { trackId: string; action: string; sessionId?: string };
if (!trackId || !action) {
return reply.code(400).send({ error: 'trackId and action are required' });
}
if (!['completed', 'skipped', 'promoted', 'disliked'].includes(action)) {
return reply.code(400).send({ error: 'Unsupported Vibe feedback action' });
}
// Route to existing handlers for evidence wiring
if (action === 'completed') {
@@ -148,8 +230,28 @@ export default async function v2Routes(fastify: FastifyInstance, options: { dbSe
const planRemaining = await withPlanLock(userId, async () => {
const active = await getActivePlan(userId);
if (!active) return 0;
const playedTrackIds = [trackId];
const refill = await director.replan(userId, active.sessionId, active.plan, playedTrackIds, active.seedTrackId ?? undefined);
if (!sessionId || active.sessionId !== sessionId) return 0;
// Feedback may race /next or arrive after a client-side prefetch. In all
// cases its track becomes ineligible for the rest of this session.
active.excludedTrackIds = appendUniqueTrackId(active.excludedTrackIds, trackId);
const artistResult = await dbService.pgClient.query<{ artist_id: string }>(
`SELECT artist_id FROM track_artists_v2
WHERE track_id = $1 AND role = 'main'
ORDER BY confidence DESC NULLS LAST LIMIT 1`,
[trackId]
);
if (artistResult.rows[0]?.artist_id) {
active.excludedArtistIds = appendUniqueTrackId(active.excludedArtistIds, artistResult.rows[0].artist_id);
}
const sessionTrackIds = sessionExclusions(active);
const refill = await director.replan(
userId,
active.sessionId,
active.plan,
[trackId],
active.seedTrackId ?? undefined,
{ excludedTrackIds: sessionTrackIds, excludedArtistIds: sessionArtistExclusions(active) }
);
active.plan = refill;
await setActivePlan(userId, active);
return active.plan.length;