initial state: muzick music player + recommendation engine
This commit is contained in:
kami
@@ -0,0 +1,4 @@
|
||||
node_modules
|
||||
dist
|
||||
.git
|
||||
.env
|
||||
@@ -0,0 +1,16 @@
|
||||
# --- Core worker ---
|
||||
DATABASE_URL=postgresql://user:password@db:5432/muzick # Postgres connection string
|
||||
REDIS_URL=redis://redis:6379 # Redis (BullMQ) connection string
|
||||
MUSIC_DIR=/music # Root music library path scanned by the worker
|
||||
INTEGRITY_SWEEP_CRON=0 3 * * * # Cron for the periodic integrity sweep
|
||||
|
||||
# --- External enrichment integrations ---
|
||||
MUSICBRAINZ_CONTACT=you@example.com # Contact (email/URL) for the required MusicBrainz User-Agent
|
||||
LASTFM_API_KEY=your-lastfm-api-key # Last.fm API key (enrichment is disabled if empty)
|
||||
LASTFM_SHARED_SECRET=your-lastfm-shared-secret # Last.fm shared secret (optional; only for authenticated calls)
|
||||
DISCOGS_TOKEN=your-discogs-token # Discogs personal access token (enrichment is disabled if empty)
|
||||
|
||||
# --- Cleanup sweep (dislike lifecycle) ---
|
||||
CLEANUP_SWEEP_CRON=0 */6 * * * # Cron for the dislike grace-period cleanup sweep
|
||||
NTFY_URL=https://ntfy.sh # ntfy server base URL (leave empty to disable notifications)
|
||||
NTFY_TOPIC=muzick # ntfy topic to publish deletion warnings to
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM node:20-slim
|
||||
RUN apt-get update && apt-get install -y ffmpeg && rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY package*.json ./
|
||||
RUN npm install --legacy-peer-deps
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
EXPOSE 3000
|
||||
CMD ["npm", "run", "start"]
|
||||
Generated
+1832
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "muzick-worker",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"start": "node dist/index.js",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"repair:metadata": "node dist/scripts/repair-corrupted-metadata.js",
|
||||
"split:artists": "tsx src/scripts/split-collab-artists.ts",
|
||||
"prebuild": "tsc --noEmit",
|
||||
"build": "tsc"
|
||||
},
|
||||
"dependencies": {
|
||||
"bullmq": "^5.1.0",
|
||||
"essentia.js": "^0.1.3",
|
||||
"music-metadata": "^7.11.0",
|
||||
"pg": "^8.21.0",
|
||||
"redis": "^5.0.0",
|
||||
"socks-proxy-agent": "^10.0.0",
|
||||
"typesense": "^3.0.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.10.0",
|
||||
"@types/pg": "^8.20.0",
|
||||
"tsx": "^4.6.2",
|
||||
"typescript": "^5.3.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
/**
|
||||
* AudioFeaturesService — two-pass audio feature extraction.
|
||||
*
|
||||
* Pass 1 (fast): read BPM and key from embedded file tags via music-metadata.
|
||||
* Many files tagged by beets / MusicBrainz Picard / Mixed In Key already
|
||||
* carry these, so we get the answer for free.
|
||||
*
|
||||
* Pass 2 (compute): run essentia.js (WebAssembly) on the decoded PCM signal
|
||||
* for anything still missing. Audio decoding is done by a ffmpeg subprocess
|
||||
* so every format ffmpeg understands (MP3, FLAC, AAC, OGG, …) is supported.
|
||||
* Computed features: BPM, key, energy, danceability, dynamic complexity.
|
||||
*
|
||||
* The two passes are combined via COALESCE so that embedded-tag values are
|
||||
* never overwritten once they exist.
|
||||
*/
|
||||
import { spawn } from 'child_process';
|
||||
import mm from 'music-metadata';
|
||||
import { Client as PgClient } from 'pg';
|
||||
|
||||
// Lazy WASM singleton — heavy to load (~2.4 MB), so we initialise once and
|
||||
// reuse across all enrichment jobs within the same worker process.
|
||||
let essentiaReady: Promise<{ essentia: any }> | null = null;
|
||||
|
||||
async function getEssentia() {
|
||||
if (!essentiaReady) {
|
||||
essentiaReady = (async () => {
|
||||
const mod = await import('essentia.js');
|
||||
// essentia.js v0.1.3: default export contains { Essentia, EssentiaWASM, ... }
|
||||
// EssentiaWASM is already the instantiated WASM module (not a function)
|
||||
const EssentiaWASM = mod.default?.EssentiaWASM ?? mod.EssentiaWASM;
|
||||
const Essentia = mod.default?.Essentia ?? mod.Essentia;
|
||||
if (!EssentiaWASM || !Essentia) {
|
||||
throw new Error('essentia.js exports missing EssentiaWASM or Essentia');
|
||||
}
|
||||
const essentia = new Essentia(EssentiaWASM);
|
||||
return { essentia };
|
||||
})();
|
||||
}
|
||||
return essentiaReady;
|
||||
}
|
||||
|
||||
/** Decode any audio file to a mono Float32Array at 44100 Hz via ffmpeg. */
|
||||
function decodeAudioToFloat32(filePath: string): Promise<Float32Array> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
const ff = spawn('ffmpeg', [
|
||||
'-i', filePath,
|
||||
'-vn',
|
||||
'-acodec', 'pcm_f32le',
|
||||
'-ar', '44100',
|
||||
'-ac', '1',
|
||||
'-f', 'f32le',
|
||||
'pipe:1',
|
||||
], { stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
|
||||
ff.stdout.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
ff.stdout.on('end', () => {
|
||||
const buf = Buffer.concat(chunks);
|
||||
resolve(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
|
||||
});
|
||||
ff.on('error', reject);
|
||||
ff.on('close', (code) => {
|
||||
if (code !== 0 && chunks.length === 0) {
|
||||
reject(new Error(`ffmpeg exited with code ${code} for ${filePath}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export interface AudioFeatures {
|
||||
bpm: number | null;
|
||||
key: string | null;
|
||||
energy: number | null;
|
||||
danceability: number | null;
|
||||
dynamicComplexity: number | null;
|
||||
}
|
||||
|
||||
export class AudioFeaturesService {
|
||||
constructor(private pgClient: PgClient) {}
|
||||
|
||||
async ensureSchema(): Promise<void> {
|
||||
await this.pgClient.query(
|
||||
`CREATE TABLE IF NOT EXISTS track_audio_features (
|
||||
track_id UUID PRIMARY KEY REFERENCES tracks(id) ON DELETE CASCADE,
|
||||
bpm REAL,
|
||||
key TEXT,
|
||||
energy REAL,
|
||||
danceability REAL,
|
||||
valence REAL,
|
||||
acousticness REAL,
|
||||
instrumentalness REAL,
|
||||
liveness REAL,
|
||||
valence_score REAL,
|
||||
tempo REAL
|
||||
)`
|
||||
);
|
||||
}
|
||||
|
||||
// ── Pass 1: embedded tags ──────────────────────────────────────────────────
|
||||
|
||||
private async readEmbeddedTags(filePath: string): Promise<{ bpm: number | null; key: string | null; replayGainDb: number | null }> {
|
||||
try {
|
||||
const { common } = await mm.parseFile(filePath, { duration: false });
|
||||
const bpm = common.bpm && Number.isFinite(common.bpm) && common.bpm > 0 ? common.bpm : null;
|
||||
const key = common.key?.trim() || null;
|
||||
const rgRaw = (common as any).replaygain_track_gain;
|
||||
let replayGainDb: number | null = null;
|
||||
if (rgRaw != null) {
|
||||
const val = typeof rgRaw === 'object' ? rgRaw.dB : Number(String(rgRaw).replace(/[^\d.-]/g, ''));
|
||||
if (Number.isFinite(val)) replayGainDb = val;
|
||||
}
|
||||
return { bpm, key, replayGainDb };
|
||||
} catch {
|
||||
return { bpm: null, key: null, replayGainDb: null };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Pass 2: essentia.js DSP ────────────────────────────────────────────────
|
||||
|
||||
private async analyseWithEssentia(filePath: string, needBpm: boolean, needKey: boolean): Promise<AudioFeatures> {
|
||||
const result: AudioFeatures = { bpm: null, key: null, energy: null, danceability: null, dynamicComplexity: null };
|
||||
|
||||
const { essentia } = await getEssentia();
|
||||
const signal = await decodeAudioToFloat32(filePath);
|
||||
if (signal.length === 0) return result;
|
||||
|
||||
const vec = essentia.arrayToVector(signal);
|
||||
|
||||
if (needBpm) {
|
||||
try {
|
||||
const rhythm = essentia.RhythmExtractor2013(vec);
|
||||
if (rhythm.bpm > 0) result.bpm = Math.round(rhythm.bpm * 10) / 10;
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] RhythmExtractor2013 failed:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
if (needKey) {
|
||||
try {
|
||||
const keyResult = essentia.KeyExtractor(vec);
|
||||
if (keyResult.key) result.key = `${keyResult.key} ${keyResult.scale}`;
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] KeyExtractor failed:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// Energy and danceability are only available from signal analysis — always compute.
|
||||
try {
|
||||
const energy = essentia.Energy(vec);
|
||||
// Essentia Energy returns the sum of squared samples, normalised to [0,1]
|
||||
// by dividing by signal length. Map to a sensible 0-1 display value.
|
||||
const rawEnergy = energy.energy / signal.length;
|
||||
result.energy = Math.min(1, rawEnergy * 1000); // typical values << 0.001
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] Energy failed:', (err as Error).message);
|
||||
}
|
||||
|
||||
try {
|
||||
const dance = essentia.Danceability(vec);
|
||||
// Danceability output range is 0-3; normalise to 0-1.
|
||||
result.danceability = Math.min(1, dance.danceability / 3);
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] Danceability failed:', (err as Error).message);
|
||||
}
|
||||
|
||||
try {
|
||||
const dynComp = essentia.DynamicComplexity(vec);
|
||||
// DynamicComplexity is a measure of loudness variation (0 = flat/compressed,
|
||||
// higher = dynamic). Scale to 0-1 (typical max ~10 dB).
|
||||
result.dynamicComplexity = Math.min(1, dynComp.dynamicComplexity / 10);
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] DynamicComplexity failed:', (err as Error).message);
|
||||
}
|
||||
|
||||
vec.delete(); // free WASM memory
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Public entry point ─────────────────────────────────────────────────────
|
||||
|
||||
async extractAndPersist(trackId: string): Promise<void> {
|
||||
const pathRes = await this.pgClient.query<{ path: string }>(
|
||||
'SELECT path FROM tracks WHERE id = $1',
|
||||
[trackId]
|
||||
);
|
||||
const row = pathRes.rows[0];
|
||||
if (!row) {
|
||||
console.warn(`[AudioFeatures] track not found: ${trackId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const tags = await this.readEmbeddedTags(row.path);
|
||||
|
||||
// Rough energy proxy from ReplayGain when available (free, no DSP).
|
||||
let tagEnergy: number | null = null;
|
||||
if (tags.replayGainDb !== null) {
|
||||
tagEnergy = Math.min(1, Math.max(0, (tags.replayGainDb + 18) / 21));
|
||||
}
|
||||
|
||||
// Only spin up Essentia when embedded tags are missing something useful.
|
||||
const needBpm = tags.bpm === null;
|
||||
const needKey = tags.key === null;
|
||||
// Always run Essentia for energy/danceability since no tag source covers those.
|
||||
const dsq = await this.analyseWithEssentia(row.path, needBpm, needKey);
|
||||
|
||||
const bpm = tags.bpm ?? dsq.bpm;
|
||||
const key = tags.key ?? dsq.key;
|
||||
const energy = tagEnergy ?? dsq.energy;
|
||||
const danceability = dsq.danceability;
|
||||
const tempo = bpm;
|
||||
|
||||
await this.pgClient.query(
|
||||
`INSERT INTO track_audio_features (track_id, bpm, key, energy, danceability, tempo)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (track_id) DO UPDATE SET
|
||||
bpm = COALESCE(EXCLUDED.bpm, track_audio_features.bpm),
|
||||
key = COALESCE(EXCLUDED.key, track_audio_features.key),
|
||||
energy = COALESCE(EXCLUDED.energy, track_audio_features.energy),
|
||||
danceability = COALESCE(EXCLUDED.danceability, track_audio_features.danceability),
|
||||
tempo = COALESCE(EXCLUDED.tempo, track_audio_features.tempo)`,
|
||||
[trackId, bpm, key, energy, danceability, tempo]
|
||||
);
|
||||
|
||||
const parts: string[] = [];
|
||||
if (bpm) parts.push(`bpm:${bpm.toFixed(1)}${tags.bpm ? '(tag)' : '(dsp)'}`);
|
||||
if (key) parts.push(`key:${key}${tags.key ? '(tag)' : '(dsp)'}`);
|
||||
if (energy !== null) parts.push(`energy:${energy.toFixed(2)}`);
|
||||
if (danceability !== null) parts.push(`dance:${danceability.toFixed(2)}`);
|
||||
console.log(
|
||||
`[AudioFeatures] ${trackId}: ${parts.length ? parts.join(', ') : 'no features extracted'}`
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import { Client as PgClient } from 'pg';
|
||||
import { unlink } from 'fs/promises';
|
||||
|
||||
const NTFY_URL = process.env.NTFY_URL || '';
|
||||
const NTFY_TOPIC = process.env.NTFY_TOPIC || 'muzick';
|
||||
const SYSTEM_USER_ID = '00000000-0000-0000-0000-000000000000';
|
||||
|
||||
async function sendNtfy(title: string, message: string): Promise<void> {
|
||||
if (!NTFY_URL) return;
|
||||
try {
|
||||
await fetch(`${NTFY_URL}/${NTFY_TOPIC}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ title, message, priority: 3 }),
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn('[Cleanup] ntfy send failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
export class CleanupSweepService {
|
||||
constructor(private pgClient: PgClient) {}
|
||||
|
||||
async runSweep(): Promise<{ warned: number; deleted: number }> {
|
||||
const warned = await this.advanceToWarned();
|
||||
const deleted = await this.finalizeDeleted();
|
||||
return { warned, deleted };
|
||||
}
|
||||
|
||||
/**
|
||||
* Find HIDDEN dislikes past their grace period → send ntfy warning, mark WARNED.
|
||||
*/
|
||||
private async advanceToWarned(): Promise<number> {
|
||||
const res = await this.pgClient.query<{ track_id: string; track_title: string; track_artist: string }>(
|
||||
`SELECT d.track_id, t.title AS track_title, t.artist AS track_artist
|
||||
FROM dislikes d
|
||||
JOIN tracks t ON t.id = d.track_id
|
||||
WHERE d.state = 'HIDDEN'
|
||||
AND NOW() > d.disliked_at + (d.grace_hours || ' hours')::interval`
|
||||
);
|
||||
|
||||
for (const row of res.rows) {
|
||||
await this.pgClient.query(
|
||||
`UPDATE dislikes SET warned_at = NOW(), state = 'WARNED' WHERE track_id = $1`,
|
||||
[row.track_id]
|
||||
);
|
||||
await sendNtfy(
|
||||
'Track scheduled for deletion',
|
||||
`"${row.track_title}" by ${row.track_artist} will be permanently deleted in 24 hours. Restore it from Quarantine to keep it.`
|
||||
);
|
||||
console.log(`[Cleanup] Warned: ${row.track_title} (${row.track_id})`);
|
||||
}
|
||||
|
||||
return res.rows.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find WARNED dislikes where 24h has passed since warned_at → delete file + DB record.
|
||||
*/
|
||||
private async finalizeDeleted(): Promise<number> {
|
||||
const res = await this.pgClient.query<{ track_id: string; track_path: string; track_title: string }>(
|
||||
`SELECT d.track_id, t.path AS track_path, t.title AS track_title
|
||||
FROM dislikes d
|
||||
JOIN tracks t ON t.id = d.track_id
|
||||
WHERE d.state = 'WARNED'
|
||||
AND NOW() > d.warned_at + INTERVAL '24 hours'`
|
||||
);
|
||||
|
||||
let deleted = 0;
|
||||
for (const row of res.rows) {
|
||||
try {
|
||||
await unlink(row.track_path);
|
||||
} catch (err: any) {
|
||||
if (err.code !== 'ENOENT') {
|
||||
console.error(`[Cleanup] Failed to delete file ${row.track_path}:`, err);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await this.pgClient.query('BEGIN');
|
||||
await this.pgClient.query(
|
||||
`INSERT INTO feedback (user_id, track_id, action) VALUES ($1, $2, 'deleted_permanent')`,
|
||||
[SYSTEM_USER_ID, row.track_id]
|
||||
);
|
||||
await this.pgClient.query('DELETE FROM tracks WHERE id = $1', [row.track_id]);
|
||||
await this.pgClient.query('COMMIT');
|
||||
deleted++;
|
||||
console.log(`[Cleanup] Permanently deleted: ${row.track_title} (${row.track_id})`);
|
||||
} catch (err) {
|
||||
await this.pgClient.query('ROLLBACK');
|
||||
console.error(`[Cleanup] DB deletion failed for ${row.track_id}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Vendored
+13
@@ -0,0 +1,13 @@
|
||||
declare module 'essentia.js' {
|
||||
const EssentiaWASM: () => Promise<any>;
|
||||
class Essentia {
|
||||
constructor(wasmModule: any, isDebug?: boolean);
|
||||
arrayToVector(arr: Float32Array): any;
|
||||
RhythmExtractor2013(signal: any, maxTempo?: number, method?: string, minTempo?: number): { bpm: number; ticks: number[]; confidence: number };
|
||||
KeyExtractor(audio: any, averageDetuningCorrection?: boolean, frameSize?: number, hopSize?: number): { key: string; scale: string; strength: number };
|
||||
Energy(signal: any): { energy: number };
|
||||
Danceability(signal: any, maxTau?: number, minTau?: number, sampleRate?: number): { danceability: number; dfa: number[] };
|
||||
DynamicComplexity(signal: any, frameSize?: number, sampleRate?: number): { dynamicComplexity: number; loudness: number };
|
||||
}
|
||||
export { EssentiaWASM, Essentia };
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
import { Worker, Job } from 'bullmq';
|
||||
import { connection, QUEUE_NAME, queue } from './queue.js';
|
||||
import { MetadataRefreshJob, AudioAnalysisJob, LibraryScanJob, ArtistSimilarityJob, ArtistImageJob, AlbumCoverJob, ReprocessArtistsJob } from './types.js';
|
||||
import { Client as PgClient } from 'pg';
|
||||
import { ScannerService } from './scanner.service.js';
|
||||
import { IntegrityService } from './integrity.service.js';
|
||||
import { EnrichmentService } from './enrichment.service.js';
|
||||
import { AudioFeaturesService } from './audio-features.service.js';
|
||||
import { CleanupSweepService } from './cleanup.service.js';
|
||||
|
||||
// Cron for the periodic integrity sweep (default: daily at 03:00). Configurable
|
||||
// via INTEGRITY_SWEEP_CRON. MUSIC_DIR (consumed by IntegrityService) controls
|
||||
// the rescan root.
|
||||
const INTEGRITY_SWEEP_CRON = process.env.INTEGRITY_SWEEP_CRON || '0 3 * * *';
|
||||
// Cron for the dislike cleanup sweep (default: every 6 hours). Configurable via CLEANUP_SWEEP_CRON.
|
||||
const CLEANUP_SWEEP_CRON = process.env.CLEANUP_SWEEP_CRON || '0 */6 * * *';
|
||||
// Cron for the stale vibe-session reaper (default: hourly). Configurable via
|
||||
// VIBE_REAP_CRON. Spec §4 / Invariant B: ACTIVE batches with no interaction for
|
||||
// 24h must transition to RESOLVED so returning users start fresh sessions.
|
||||
const VIBE_REAP_CRON = process.env.VIBE_REAP_CRON || '0 * * * *';
|
||||
// Worker concurrency - how many jobs to process in parallel
|
||||
const WORKER_CONCURRENCY = parseInt(process.env.WORKER_CONCURRENCY || '10', 10);
|
||||
|
||||
const pgClient = new PgClient({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
});
|
||||
|
||||
async function initWorker() {
|
||||
await pgClient.connect();
|
||||
console.log('Worker connected to PostgreSQL');
|
||||
|
||||
const scannerService = new ScannerService(pgClient, queue);
|
||||
const enrichmentService = new EnrichmentService(pgClient);
|
||||
const audioFeaturesService = new AudioFeaturesService(pgClient);
|
||||
await audioFeaturesService.ensureSchema();
|
||||
|
||||
// Self-provision the enrichment schema additions (artist_similar + reused
|
||||
// columns) at startup, alongside the integrity table.
|
||||
await enrichmentService.ensureSchema();
|
||||
await new IntegrityService(pgClient, queue).ensureSchema();
|
||||
|
||||
const worker = new Worker(QUEUE_NAME, async (job: Job<any>) => {
|
||||
// console.log(`Processing job: ${job.name} (ID: ${job.id})`);
|
||||
console.log(`Processing job: ${job.name} (ID: ${job.id})`);
|
||||
|
||||
switch (job.name) {
|
||||
case 'scan_library': {
|
||||
const payload = job.data as LibraryScanJob;
|
||||
console.log(`[Scanner] Starting library scan in: ${payload.directory}`);
|
||||
await scannerService.scanDirectory(payload.directory);
|
||||
// Re-sync Typesense after every scan so search reflects new/changed tracks.
|
||||
await queue.add('reindex_tracks', {}, { removeOnComplete: { age: 86400, count: 100 }, removeOnFail: { age: 86400, count: 100 } });
|
||||
console.log(`[Scanner] Enqueued Typesense reindex after scan`);
|
||||
break;
|
||||
}
|
||||
case 'metadata_refresh': {
|
||||
const payload = job.data as MetadataRefreshJob;
|
||||
console.log(`[Metadata] Refreshing track: ${payload.trackId} (Mode: ${payload.refreshType})`);
|
||||
|
||||
// Real best-effort enrichment via the external-integration clients.
|
||||
// Each provider is isolated inside the service so one failing source
|
||||
// never aborts the others or fails the job.
|
||||
await enrichmentService.enrichTrack(payload.trackId);
|
||||
console.log(`[Metadata] Successfully refreshed track: ${payload.trackId}`);
|
||||
break;
|
||||
}
|
||||
case 'artist_similarity': {
|
||||
const payload = job.data as ArtistSimilarityJob;
|
||||
console.log(`[Similarity] Refreshing similar artists for: ${payload.artistId}`);
|
||||
await enrichmentService.refreshArtistSimilarity(payload.artistId);
|
||||
break;
|
||||
}
|
||||
case 'artist_image': {
|
||||
const payload = job.data as ArtistImageJob;
|
||||
console.log(`[ArtistImage] Refreshing image for artist: ${payload.artistId}`);
|
||||
await enrichmentService.refreshArtistImage(payload.artistId);
|
||||
break;
|
||||
}
|
||||
case 'album_cover': {
|
||||
const payload = job.data as AlbumCoverJob;
|
||||
console.log(`[AlbumCover] Refreshing cover for album: ${payload.albumId}`);
|
||||
await enrichmentService.refreshAlbumCover(payload.albumId);
|
||||
break;
|
||||
}
|
||||
case 'audio_analysis': {
|
||||
const payload = job.data as AudioAnalysisJob;
|
||||
console.log(`[Audio] Analyzing track: ${payload.trackId}`);
|
||||
await audioFeaturesService.extractAndPersist(payload.trackId);
|
||||
console.log(`[Audio] Successfully analyzed track: ${payload.trackId}`);
|
||||
break;
|
||||
}
|
||||
case 'integrity_sweep': {
|
||||
// Periodic self-healing pass: detect corrupt/missing track metadata,
|
||||
// auto-fix via rescan + SQL strip, flag the rest for manual review.
|
||||
console.log('[Integrity] Starting integrity sweep');
|
||||
const integrityService = new IntegrityService(pgClient, queue);
|
||||
const summary = await integrityService.runSweep();
|
||||
console.log(
|
||||
`[Integrity] Sweep complete: detected=${summary.detected} fixed=${summary.fixed} needsReview=${summary.needsReview}`
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'cleanup_sweep': {
|
||||
console.log('[Cleanup] Starting dislike cleanup sweep');
|
||||
const cleanupService = new CleanupSweepService(pgClient);
|
||||
const result = await cleanupService.runSweep();
|
||||
console.log(`[Cleanup] Sweep complete: warned=${result.warned} deleted=${result.deleted}`);
|
||||
break;
|
||||
}
|
||||
case 'vibe_reap': {
|
||||
// Stale-session reaper (Invariant B). Transitions ACTIVE batches with no
|
||||
// interaction for 24h to RESOLVED. Runs hourly; the SQL is idempotent.
|
||||
const reaped = await pgClient.query(
|
||||
`UPDATE recommendation_batch
|
||||
SET status = 'RESOLVED'
|
||||
WHERE status = 'ACTIVE'
|
||||
AND last_interaction_at < NOW() - INTERVAL '24 hours'`
|
||||
);
|
||||
const n = reaped.rowCount ?? 0;
|
||||
if (n > 0) console.log(`[VibeReap] Reaped ${n} stale ACTIVE session(s)`);
|
||||
break;
|
||||
}
|
||||
case 'reindex_tracks': {
|
||||
// Sync all LIBRARY tracks from Postgres into the Typesense 'tracks'
|
||||
// collection. Inlined to avoid duplicating a service module.
|
||||
const { Client } = await import('typesense');
|
||||
const searchHost = process.env.TYPESENSE_HOST || 'localhost';
|
||||
const searchPort = parseInt(process.env.TYPESENSE_PORT || '8108', 10);
|
||||
const searchApiKey = process.env.TYPESENSE_API_KEY || '';
|
||||
|
||||
const tsClient = new Client({
|
||||
nodes: [{ host: searchHost, port: searchPort, protocol: 'http' }],
|
||||
apiKey: searchApiKey,
|
||||
});
|
||||
|
||||
// Ensure collection schema exists.
|
||||
try {
|
||||
await tsClient.collections().create({
|
||||
name: 'tracks',
|
||||
fields: [
|
||||
{ name: 'id', type: 'string' },
|
||||
{ name: 'title', type: 'string' },
|
||||
{ name: 'artist', type: 'string' },
|
||||
{ name: 'album', type: 'string' },
|
||||
{ name: 'duration', type: 'int32' },
|
||||
{ name: 'play_count', type: 'int32' },
|
||||
{ name: 'genre', type: 'string[]', facet: true },
|
||||
{ name: 'source_type', type: 'string' },
|
||||
],
|
||||
});
|
||||
} catch (err: any) {
|
||||
if (!err?.message?.includes('already exists')) throw err;
|
||||
}
|
||||
|
||||
const tracksRes = await pgClient.query(
|
||||
`SELECT t.id, t.title, t.artist, al.title AS album, t.duration, t.play_count, t.source_type
|
||||
FROM tracks t
|
||||
LEFT JOIN albums al ON al.id = t.album_id
|
||||
WHERE t.state = 'LIBRARY'`
|
||||
);
|
||||
const tracks = tracksRes.rows;
|
||||
|
||||
const genreMap = new Map<string, string[]>();
|
||||
const genreRes = await pgClient.query(
|
||||
`SELECT tg.track_id, g.name FROM track_genre tg JOIN genre g ON g.id = tg.genre_id`
|
||||
);
|
||||
for (const row of genreRes.rows) {
|
||||
const arr = genreMap.get(row.track_id) || [];
|
||||
arr.push(row.name);
|
||||
genreMap.set(row.track_id, arr);
|
||||
}
|
||||
|
||||
const collection = tsClient.collections('tracks').documents();
|
||||
const batchSize = 100;
|
||||
let indexed = 0;
|
||||
|
||||
for (let i = 0; i < tracks.length; i += batchSize) {
|
||||
const batch = tracks.slice(i, i + batchSize);
|
||||
const documents = batch.map((t: any) => ({
|
||||
id: String(t.id),
|
||||
title: t.title,
|
||||
artist: t.artist,
|
||||
album: t.album ?? '',
|
||||
duration: Math.round(Number(t.duration)),
|
||||
play_count: Number(t.play_count),
|
||||
genre: genreMap.get(String(t.id)) || [],
|
||||
source_type: t.source_type,
|
||||
}));
|
||||
await collection.import(documents, { action: 'upsert' });
|
||||
indexed += documents.length;
|
||||
}
|
||||
|
||||
console.log(`[Reindex] Indexed ${indexed} tracks into Typesense`);
|
||||
break;
|
||||
}
|
||||
case 'reprocess_artists': {
|
||||
const payload = job.data as ReprocessArtistsJob;
|
||||
const batchSize = payload.batchSize ?? 100;
|
||||
const offset = payload.offset ?? 0;
|
||||
console.log(`[ReprocessArtists] Starting artist reprocessing (batch=${batchSize}, offset=${offset})`);
|
||||
|
||||
const artistsRes = await pgClient.query(
|
||||
`SELECT id, name, canonical_name, mbid FROM artists ORDER BY id LIMIT $1 OFFSET $2`,
|
||||
[batchSize, offset]
|
||||
);
|
||||
|
||||
let processed = 0;
|
||||
let updated = 0;
|
||||
let merged = 0;
|
||||
|
||||
for (const artist of artistsRes.rows) {
|
||||
processed++;
|
||||
try {
|
||||
const result = await enrichmentService.resolveArtistIdentity(artist.name);
|
||||
// Merge if resolved to a different artist (duplicate detected)
|
||||
if (result.artistId !== artist.id) {
|
||||
merged++;
|
||||
// Move track links to the keeper, skipping any (track, role) the
|
||||
// keeper already has, then drop the loser's — a plain UPDATE would
|
||||
// violate track_artists_pkey when both are on the same track.
|
||||
await pgClient.query(
|
||||
`INSERT INTO track_artists (track_id, artist_id, role)
|
||||
SELECT track_id, $1, role FROM track_artists WHERE artist_id = $2
|
||||
ON CONFLICT (track_id, artist_id, role) DO NOTHING`,
|
||||
[result.artistId, artist.id]
|
||||
);
|
||||
await pgClient.query(
|
||||
`DELETE FROM track_artists WHERE artist_id = $1`,
|
||||
[artist.id]
|
||||
);
|
||||
// Fold albums into the keeper. Move tracks of any same-title album
|
||||
// to the keeper's matching album first (UNIQUE(artist_id,title) and
|
||||
// ON DELETE CASCADE mean a blind UPDATE could collide or, worse,
|
||||
// cascade-delete tracks when the loser artist is removed).
|
||||
const dupAlbums = await pgClient.query(
|
||||
`SELECT l.id AS loser_id, k.id AS keeper_id
|
||||
FROM albums l JOIN albums k
|
||||
ON k.artist_id = $1 AND lower(k.title) = lower(l.title)
|
||||
WHERE l.artist_id = $2`,
|
||||
[result.artistId, artist.id]
|
||||
);
|
||||
for (const { loser_id, keeper_id } of dupAlbums.rows) {
|
||||
await pgClient.query(`UPDATE tracks SET album_id = $1 WHERE album_id = $2`, [keeper_id, loser_id]);
|
||||
await pgClient.query(`DELETE FROM albums WHERE id = $1`, [loser_id]);
|
||||
}
|
||||
// Remaining (non-colliding) albums move over cleanly.
|
||||
await pgClient.query(
|
||||
`UPDATE albums SET artist_id = $1 WHERE artist_id = $2`,
|
||||
[result.artistId, artist.id]
|
||||
);
|
||||
await pgClient.query(
|
||||
`DELETE FROM artists WHERE id = $1`,
|
||||
[artist.id]
|
||||
);
|
||||
console.log(`[ReprocessArtists] Merged "${artist.name}" (${artist.id}) -> "${result.canonicalName}" (${result.artistId})`);
|
||||
} else {
|
||||
// Update the existing artist record with new canonical_name and/or mbid
|
||||
const updates: string[] = [];
|
||||
const params: any[] = [];
|
||||
let paramIdx = 1;
|
||||
|
||||
if (result.canonicalName && artist.canonical_name !== result.canonicalName) {
|
||||
updates.push(`canonical_name = $${paramIdx++}`);
|
||||
params.push(result.canonicalName);
|
||||
}
|
||||
if (result.mbid && artist.mbid !== result.mbid) {
|
||||
updates.push(`mbid = $${paramIdx++}`);
|
||||
params.push(result.mbid);
|
||||
}
|
||||
if (result.sortName && artist.sort_name !== result.sortName) {
|
||||
updates.push(`sort_name = $${paramIdx++}`);
|
||||
params.push(result.sortName);
|
||||
}
|
||||
if (updates.length > 0) {
|
||||
updates.push(`updated_at = CURRENT_TIMESTAMP`);
|
||||
params.push(artist.id);
|
||||
await pgClient.query(
|
||||
`UPDATE artists SET ${updates.join(', ')} WHERE id = $${paramIdx}`,
|
||||
params
|
||||
);
|
||||
updated++;
|
||||
console.log(`[ReprocessArtists] Updated "${artist.name}" (${artist.id}): ${updates.join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh the artist image via the dedicated job rather than inline,
|
||||
// so the reprocess batches aren't blocked on image HTTP. Deduped by
|
||||
// jobId across the run.
|
||||
await queue.add(
|
||||
'artist_image',
|
||||
{ artistId: result.artistId } satisfies ArtistImageJob,
|
||||
{
|
||||
jobId: `artist-image-${result.artistId}`,
|
||||
removeOnComplete: { age: 86400, count: 5000 },
|
||||
removeOnFail: { age: 86400, count: 5000 },
|
||||
}
|
||||
);
|
||||
} catch (err) {
|
||||
console.error(`[ReprocessArtists] Failed for artist ${artist.id} (${artist.name}):`, err);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[ReprocessArtists] Batch complete: processed=${processed}, updated=${updated}, merged=${merged}`);
|
||||
|
||||
// If we processed a full batch, enqueue the next one
|
||||
if (artistsRes.rows.length === batchSize) {
|
||||
await queue.add('reprocess_artists', { batchSize, offset: offset + batchSize }, {
|
||||
removeOnComplete: { age: 86400, count: 100 },
|
||||
removeOnFail: { age: 86400, count: 100 },
|
||||
});
|
||||
console.log(`[ReprocessArtists] Enqueued next batch at offset ${offset + batchSize}`);
|
||||
} else {
|
||||
// Final batch - run deduplication pass to merge artists with same normalized_name
|
||||
console.log(`[ReprocessArtists] All batches complete, running deduplication pass...`);
|
||||
|
||||
const dupRes = await pgClient.query(
|
||||
`SELECT normalized_name, array_agg(id ORDER BY
|
||||
CASE WHEN mbid IS NOT NULL THEN 0 ELSE 1 END,
|
||||
CASE WHEN canonical_name IS NOT NULL THEN 0 ELSE 1 END,
|
||||
id
|
||||
) as ids
|
||||
FROM artists
|
||||
GROUP BY normalized_name
|
||||
HAVING COUNT(*) > 1`
|
||||
);
|
||||
|
||||
let dedupMerged = 0;
|
||||
for (const row of dupRes.rows) {
|
||||
const ids = row.ids;
|
||||
const keepId = ids[0]; // First one (prefers MBID, then canonical_name, then lowest id)
|
||||
const mergeIds = ids.slice(1);
|
||||
|
||||
for (const mergeId of mergeIds) {
|
||||
if (mergeId === keepId) continue;
|
||||
try {
|
||||
// First, handle track_artists conflicts: if both artists are on same track,
|
||||
// keep the 'main' role, or merge roles. Use ON CONFLICT DO NOTHING to skip duplicates.
|
||||
await pgClient.query(
|
||||
`INSERT INTO track_artists (track_id, artist_id, role)
|
||||
SELECT track_id, $1, role FROM track_artists WHERE artist_id = $2
|
||||
ON CONFLICT (track_id, artist_id, role) DO NOTHING`,
|
||||
[keepId, mergeId]
|
||||
);
|
||||
// Then delete the old track_artists entries
|
||||
await pgClient.query(
|
||||
`DELETE FROM track_artists WHERE artist_id = $1`,
|
||||
[mergeId]
|
||||
);
|
||||
|
||||
// Fold same-title albums (move tracks) before reassigning the rest,
|
||||
// to avoid UNIQUE(artist_id,title) collisions / cascade deletes.
|
||||
const dupAlbums = await pgClient.query(
|
||||
`SELECT l.id AS loser_id, k.id AS keeper_id
|
||||
FROM albums l JOIN albums k
|
||||
ON k.artist_id = $1 AND lower(k.title) = lower(l.title)
|
||||
WHERE l.artist_id = $2`,
|
||||
[keepId, mergeId]
|
||||
);
|
||||
for (const { loser_id, keeper_id } of dupAlbums.rows) {
|
||||
await pgClient.query(`UPDATE tracks SET album_id = $1 WHERE album_id = $2`, [keeper_id, loser_id]);
|
||||
await pgClient.query(`DELETE FROM albums WHERE id = $1`, [loser_id]);
|
||||
}
|
||||
await pgClient.query(
|
||||
`UPDATE albums SET artist_id = $1 WHERE artist_id = $2`,
|
||||
[keepId, mergeId]
|
||||
);
|
||||
await pgClient.query(
|
||||
`DELETE FROM artists WHERE id = $1`,
|
||||
[mergeId]
|
||||
);
|
||||
dedupMerged++;
|
||||
console.log(`[ReprocessArtists] Dedup merged ${mergeId} -> ${keepId} (normalized: ${row.normalized_name})`);
|
||||
} catch (err) {
|
||||
console.error(`[ReprocessArtists] Dedup failed for ${mergeId}:`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[ReprocessArtists] Deduplication complete: merged=${dedupMerged}`);
|
||||
|
||||
// Album dedup pass: merge duplicate album rows (same title or same
|
||||
// MBID) that accumulated before the albumartist scanner fix. Run
|
||||
// after artist dedup so album artist_id refs are already resolved.
|
||||
try {
|
||||
const albumMerged = await enrichmentService.dedupAlbums();
|
||||
console.log(`[ReprocessArtists] Album dedup: merged=${albumMerged}`);
|
||||
} catch (err) {
|
||||
console.error('[ReprocessArtists] Album dedup failed:', err);
|
||||
}
|
||||
|
||||
console.log(`[ReprocessArtists] All artists processed!`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
console.log(`Received job of type: ${job.name} with data:`, job.data);
|
||||
break;
|
||||
}
|
||||
}, { connection, concurrency: WORKER_CONCURRENCY });
|
||||
|
||||
worker.on('completed', (job) => {
|
||||
console.log(`Job ${job.id} has completed!`);
|
||||
});
|
||||
|
||||
worker.on('failed', (job, err) => {
|
||||
console.log(`Job ${job?.id} has failed with error: ${err.message}`);
|
||||
});
|
||||
|
||||
// Register the repeatable integrity sweep. bullmq's upsertJobScheduler is
|
||||
// idempotent: keying on a fixed scheduler id ('integrity-sweep') means worker
|
||||
// restarts update the existing schedule in place rather than stacking
|
||||
// duplicate repeatable jobs.
|
||||
await queue.upsertJobScheduler(
|
||||
'integrity-sweep',
|
||||
{ pattern: INTEGRITY_SWEEP_CRON },
|
||||
{ name: 'integrity_sweep', data: { reason: 'scheduled' } }
|
||||
);
|
||||
console.log(`[Integrity] Sweep scheduled with cron: ${INTEGRITY_SWEEP_CRON}`);
|
||||
|
||||
await queue.upsertJobScheduler(
|
||||
'cleanup-sweep',
|
||||
{ pattern: CLEANUP_SWEEP_CRON },
|
||||
{ name: 'cleanup_sweep', data: { reason: 'scheduled' } }
|
||||
);
|
||||
console.log(`[Cleanup] Sweep scheduled with cron: ${CLEANUP_SWEEP_CRON}`);
|
||||
|
||||
// Stale vibe-session reaper (Invariant B): hourly sweep that transitions
|
||||
// ACTIVE batches idle for 24h to RESOLVED.
|
||||
await queue.upsertJobScheduler(
|
||||
'vibe-reap',
|
||||
{ pattern: VIBE_REAP_CRON },
|
||||
{ name: 'vibe_reap', data: { reason: 'scheduled' } }
|
||||
);
|
||||
console.log(`[VibeReap] Reaper scheduled with cron: ${VIBE_REAP_CRON}`);
|
||||
|
||||
// Enqueue a startup scan only when the database is empty (fresh volume after
|
||||
// deleting data/postgres, or first deploy). On subsequent restarts the library
|
||||
// is already populated, so an unconditional scan would be wasteful.
|
||||
const startupDir = process.env.MUSIC_DIR || '/music';
|
||||
const trackCount = await pgClient.query('SELECT COUNT(*)::int AS n FROM tracks');
|
||||
if (trackCount.rows[0].n === 0) {
|
||||
await queue.add('scan_library', { directory: startupDir }, {
|
||||
removeOnComplete: { age: 86400, count: 100 },
|
||||
removeOnFail: { age: 86400, count: 100 },
|
||||
});
|
||||
console.log(`[Startup] Enqueued library scan (fresh DB): ${startupDir}`);
|
||||
} else {
|
||||
console.log(`[Startup] Library has ${trackCount.rows[0].n} tracks — skipping startup scan`);
|
||||
}
|
||||
|
||||
console.log('Worker is running and listening for jobs...');
|
||||
|
||||
// Graceful shutdown: close the worker, the scheduler queue and the pg client
|
||||
// on termination signals. Guarded so a second signal is a no-op.
|
||||
let shuttingDown = false;
|
||||
const shutdown = async (signal: string) => {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
console.log(`[Shutdown] Received ${signal}, shutting down gracefully...`);
|
||||
try {
|
||||
await worker.close();
|
||||
await queue.close();
|
||||
await pgClient.end();
|
||||
console.log('[Shutdown] Clean shutdown complete.');
|
||||
} catch (err) {
|
||||
console.error('[Shutdown] Error during shutdown:', err);
|
||||
}
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on('SIGTERM', () => void shutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => void shutdown('SIGINT'));
|
||||
}
|
||||
|
||||
initWorker().catch(err => {
|
||||
console.error('Failed to initialize worker:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,112 @@
|
||||
// Centralised, typed configuration for the external-integration clients.
|
||||
//
|
||||
// Enrichment is best-effort: a missing required key must NOT crash the worker.
|
||||
// Instead the owning client logs a one-time warning at construction and its
|
||||
// methods short-circuit to null/[] (see musicbrainz.client.ts / lastfm.client.ts).
|
||||
|
||||
export interface MusicBrainzConfig {
|
||||
/** Base URL for the MusicBrainz web service v2. No trailing slash. */
|
||||
baseUrl: string;
|
||||
/**
|
||||
* Contact (email or URL) embedded in the required User-Agent. MusicBrainz
|
||||
* rejects/throttles requests without a meaningful UA. Empty if unset.
|
||||
*/
|
||||
contact: string;
|
||||
/** Minimum interval between requests to the MB host (MB requires <=1 req/sec). */
|
||||
minIntervalMs: number;
|
||||
}
|
||||
|
||||
export interface LastFmConfig {
|
||||
/** Base URL for the Last.fm 2.0 API. No trailing slash. */
|
||||
baseUrl: string;
|
||||
/** API key. Empty string when unconfigured -> client degrades to no-ops. */
|
||||
apiKey: string;
|
||||
/** Shared secret (only needed for authenticated/write calls; optional here). */
|
||||
sharedSecret: string;
|
||||
/** Minimum interval between requests to the Last.fm host. */
|
||||
minIntervalMs: number;
|
||||
}
|
||||
|
||||
export interface DiscogsConfig {
|
||||
/** Base URL for the Discogs API. No trailing slash. */
|
||||
baseUrl: string;
|
||||
/** Personal access token. Empty string when unconfigured -> client degrades to no-ops. */
|
||||
token: string;
|
||||
/** Minimum interval between requests (Discogs allows ~60 req/min authenticated). */
|
||||
minIntervalMs: number;
|
||||
}
|
||||
|
||||
export interface LrcLibConfig {
|
||||
/** Base URL for the LRCLib API. No trailing slash. */
|
||||
baseUrl: string;
|
||||
/** Minimum interval between requests (polite default; no key required). */
|
||||
minIntervalMs: number;
|
||||
}
|
||||
|
||||
export interface CoverArtConfig {
|
||||
/** Base URL for the Cover Art Archive. No trailing slash. */
|
||||
baseUrl: string;
|
||||
/** Minimum interval between requests (polite default; no key required). */
|
||||
minIntervalMs: number;
|
||||
}
|
||||
|
||||
export interface WikimediaConfig {
|
||||
/** Base URL for the Wikimedia API. No trailing slash. */
|
||||
baseUrl: string;
|
||||
/** Minimum interval between requests (polite default; no key required). */
|
||||
minIntervalMs: number;
|
||||
}
|
||||
|
||||
export interface IntegrationsConfig {
|
||||
musicbrainz: MusicBrainzConfig;
|
||||
lastfm: LastFmConfig;
|
||||
discogs: DiscogsConfig;
|
||||
lrclib: LrcLibConfig;
|
||||
coverart: CoverArtConfig;
|
||||
wikimedia: WikimediaConfig;
|
||||
}
|
||||
|
||||
function stripTrailingSlash(url: string): string {
|
||||
return url.replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
function env(name: string, fallback = ''): string {
|
||||
const v = process.env[name];
|
||||
return v && v.trim() !== '' ? v.trim() : fallback;
|
||||
}
|
||||
|
||||
export const integrationsConfig: IntegrationsConfig = {
|
||||
musicbrainz: {
|
||||
baseUrl: stripTrailingSlash(
|
||||
env('MUSICBRAINZ_BASE_URL', 'https://musicbrainz.org/ws/2')
|
||||
),
|
||||
contact: env('MUSICBRAINZ_CONTACT'),
|
||||
minIntervalMs: 1000,
|
||||
},
|
||||
lastfm: {
|
||||
baseUrl: stripTrailingSlash(
|
||||
env('LASTFM_BASE_URL', 'http://ws.audioscrobbler.com/2.0')
|
||||
),
|
||||
apiKey: env('LASTFM_API_KEY'),
|
||||
sharedSecret: env('LASTFM_SHARED_SECRET'),
|
||||
minIntervalMs: 250,
|
||||
},
|
||||
discogs: {
|
||||
baseUrl: stripTrailingSlash(env('DISCOGS_BASE_URL', 'https://api.discogs.com')),
|
||||
token: env('DISCOGS_TOKEN'),
|
||||
// ~60 req/min authenticated -> ~1.1s between requests.
|
||||
minIntervalMs: 1100,
|
||||
},
|
||||
lrclib: {
|
||||
baseUrl: stripTrailingSlash(env('LRCLIB_BASE_URL', 'https://lrclib.net/api')),
|
||||
minIntervalMs: 250,
|
||||
},
|
||||
coverart: {
|
||||
baseUrl: stripTrailingSlash(env('COVERART_BASE_URL', 'https://coverartarchive.org')),
|
||||
minIntervalMs: 250,
|
||||
},
|
||||
wikimedia: {
|
||||
baseUrl: stripTrailingSlash(env('WIKIMEDIA_BASE_URL', 'https://commons.wikimedia.org')),
|
||||
minIntervalMs: 250,
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,102 @@
|
||||
// Cover Art Archive client.
|
||||
//
|
||||
// No API key is required, so the client is always enabled. A polite per-host
|
||||
// rate limit (~250ms) is enforced via the http base.
|
||||
//
|
||||
// /release/{mbid} returns JSON listing images[]; we pick the front image. CAA
|
||||
// then redirects the image URL to archive.org, but we return the URL string
|
||||
// directly (callers can fetch it; the http base follows redirects by default).
|
||||
// A 404 (no cover art for that release) is treated as a clean miss (null).
|
||||
//
|
||||
// /artist/{mbid} returns artist images (same structure).
|
||||
|
||||
import { integrationsConfig, CoverArtConfig } from './config.js';
|
||||
import { buildUserAgent, requestJson, HttpError } from './http.js';
|
||||
|
||||
// --- Raw CAA response shape (only the fields we read) ----------------------
|
||||
|
||||
interface CaaImage {
|
||||
image?: string;
|
||||
front?: boolean;
|
||||
thumbnails?: { '500'?: string; large?: string; small?: string };
|
||||
}
|
||||
|
||||
interface CaaReleaseResponse {
|
||||
images?: CaaImage[];
|
||||
}
|
||||
|
||||
export class CoverArtClient {
|
||||
private readonly cfg: CoverArtConfig;
|
||||
private readonly userAgent: string;
|
||||
|
||||
constructor(cfg: CoverArtConfig = integrationsConfig.coverart) {
|
||||
this.cfg = cfg;
|
||||
this.userAgent = buildUserAgent(integrationsConfig.musicbrainz.contact);
|
||||
}
|
||||
|
||||
private async get<T>(path: string): Promise<T> {
|
||||
return requestJson<T>(`${this.cfg.baseUrl}${path}`, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: this.cfg.minIntervalMs,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the front cover URL for a release MBID (preferring the 500px
|
||||
* thumbnail, falling back to the full image), or null on no art (404) / error.
|
||||
*/
|
||||
async getReleaseCoverUrl(releaseMbid: string): Promise<string | null> {
|
||||
if (releaseMbid.trim() === '') return null;
|
||||
|
||||
const path = `/release/${encodeURIComponent(releaseMbid)}`;
|
||||
|
||||
try {
|
||||
const data = await this.get<CaaReleaseResponse>(path);
|
||||
const images = data.images ?? [];
|
||||
const front = images.find((img) => img.front) ?? images[0];
|
||||
if (!front) return null;
|
||||
return front.thumbnails?.['500'] ?? front.image ?? null;
|
||||
} catch (err) {
|
||||
// 404 means no cover art for this release; treat as a clean miss.
|
||||
if (err instanceof HttpError && err.status === 404) return null;
|
||||
console.warn('[CoverArt] getReleaseCoverUrl failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the front cover URL for a release-group MBID (preferring the 500px
|
||||
* thumbnail, falling back to the full image), or null on no art (404) / error.
|
||||
*
|
||||
* This is the preferred CAA endpoint when a release-group MBID is available:
|
||||
* it aggregates cover art across all releases in the group, so it hits more
|
||||
* often than the per-release endpoint. Now that `albums.mbid` stores
|
||||
* release-group MBIDs (from the MusicBrainz release-group search), this is a
|
||||
* direct, exact lookup — no search step required.
|
||||
*/
|
||||
async getReleaseGroupCoverUrl(releaseGroupMbid: string): Promise<string | null> {
|
||||
if (releaseGroupMbid.trim() === '') return null;
|
||||
|
||||
const path = `/release-group/${encodeURIComponent(releaseGroupMbid)}`;
|
||||
|
||||
try {
|
||||
const data = await this.get<CaaReleaseResponse>(path);
|
||||
const images = data.images ?? [];
|
||||
const front = images.find((img) => img.front) ?? images[0];
|
||||
if (!front) return null;
|
||||
return front.thumbnails?.['500'] ?? front.image ?? null;
|
||||
} catch (err) {
|
||||
if (err instanceof HttpError && err.status === 404) return null;
|
||||
console.warn('[CoverArt] getReleaseGroupCoverUrl failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cover Art Archive does NOT provide artist images, only release images.
|
||||
* This method always returns null — kept for interface compatibility.
|
||||
*/
|
||||
async getArtistImageUrl(_artistMbid: string): Promise<string | null> {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
// Deezer API client (read-only).
|
||||
//
|
||||
// Deezer provides a free, no-auth search API with high-quality cover art
|
||||
// (up to 1000×1000 via the `cover_xl` field). Coverage is very good for
|
||||
// European and mainstream releases.
|
||||
//
|
||||
// Rate limit: Deezer doesn't document a hard limit for the public search API,
|
||||
// but ~50 req/min is a safe polite default, enforced via the http base.
|
||||
//
|
||||
// Best-effort: on any error, methods log a warning and return null rather than
|
||||
// throwing, so enrichment never breaks the worker.
|
||||
|
||||
import { buildUserAgent, requestJson } from './http.js';
|
||||
|
||||
/** A normalised Deezer album search result (only the fields we read). */
|
||||
export interface DeezerAlbum {
|
||||
artistName: string;
|
||||
title: string;
|
||||
/** 1000×1000 cover URL. */
|
||||
coverXl: string;
|
||||
/** 500×500 cover URL (fallback). */
|
||||
coverBig: string;
|
||||
}
|
||||
|
||||
interface DeezerSearchResult {
|
||||
artist?: { name?: string };
|
||||
title?: string;
|
||||
cover_xl?: string;
|
||||
cover_big?: string;
|
||||
}
|
||||
|
||||
interface DeezerSearchResponse {
|
||||
data?: DeezerSearchResult[];
|
||||
}
|
||||
|
||||
export class DeezerClient {
|
||||
private readonly baseUrl = 'https://api.deezer.com';
|
||||
private readonly userAgent: string;
|
||||
private readonly minIntervalMs: number;
|
||||
|
||||
constructor(contact = '', minIntervalMs = 1200) {
|
||||
this.userAgent = buildUserAgent(contact);
|
||||
this.minIntervalMs = minIntervalMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the Deezer catalog for an album by artist + title and return the
|
||||
* best match, or null if no match / error.
|
||||
*
|
||||
* Deezer's search `q` supports the fielded syntax `artist:"…" album:"…"`
|
||||
* which gives precise matches.
|
||||
*/
|
||||
async searchAlbum(artist: string, album: string): Promise<DeezerAlbum | null> {
|
||||
if (artist.trim() === '' || album.trim() === '') return null;
|
||||
|
||||
const q = `artist:"${artist}" album:"${album}"`;
|
||||
const qs = new URLSearchParams({ q });
|
||||
const url = `${this.baseUrl}/search/album?${qs.toString()}`;
|
||||
|
||||
try {
|
||||
const data = await requestJson<DeezerSearchResponse>(url, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: this.minIntervalMs,
|
||||
});
|
||||
const results = data.data ?? [];
|
||||
if (results.length === 0) return null;
|
||||
|
||||
const best = results[0];
|
||||
const coverXl = best.cover_xl ?? best.cover_big ?? null;
|
||||
const coverBig = best.cover_big ?? best.cover_xl ?? null;
|
||||
if (!coverXl) return null;
|
||||
|
||||
return {
|
||||
artistName: best.artist?.name ?? artist,
|
||||
title: best.title ?? album,
|
||||
coverXl,
|
||||
coverBig: coverBig ?? coverXl,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[Deezer] searchAlbum failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// Discogs API client (read-only search).
|
||||
//
|
||||
// Auth is via a personal access token sent as the
|
||||
// `Authorization: Discogs token=<TOKEN>` header (set through the http base).
|
||||
// Discogs allows ~60 requests/minute for authenticated clients, enforced via the
|
||||
// http base's per-host rate limit (minIntervalMs ~1100ms).
|
||||
//
|
||||
// Best-effort: with no token configured, or on any error, methods log a warning
|
||||
// and return null rather than throwing, so enrichment never breaks the worker.
|
||||
|
||||
import { integrationsConfig, DiscogsConfig } from './config.js';
|
||||
import { buildUserAgent, requestJson } from './http.js';
|
||||
|
||||
/** A normalised Discogs release match. */
|
||||
export interface DiscogsRelease {
|
||||
discogsId: number;
|
||||
year: number | null;
|
||||
genres: string[];
|
||||
styles: string[];
|
||||
/** Cover image (full-size) URL, if any. */
|
||||
coverImage: string | null;
|
||||
/** Thumbnail URL, if any. */
|
||||
thumb: string | null;
|
||||
}
|
||||
|
||||
// --- Raw Discogs response shapes (only the fields we read) -----------------
|
||||
|
||||
interface DiscogsSearchResult {
|
||||
id?: number;
|
||||
year?: string | number;
|
||||
genre?: string[];
|
||||
style?: string[];
|
||||
cover_image?: string;
|
||||
thumb?: string;
|
||||
}
|
||||
|
||||
interface DiscogsSearchResponse {
|
||||
results?: DiscogsSearchResult[];
|
||||
}
|
||||
|
||||
interface DiscogsArtistResponse {
|
||||
images?: { uri?: string; uri150?: string; type?: string }[];
|
||||
}
|
||||
|
||||
export class DiscogsClient {
|
||||
private readonly cfg: DiscogsConfig;
|
||||
private readonly userAgent: string;
|
||||
private readonly enabled: boolean;
|
||||
|
||||
constructor(cfg: DiscogsConfig = integrationsConfig.discogs) {
|
||||
this.cfg = cfg;
|
||||
this.userAgent = buildUserAgent(integrationsConfig.musicbrainz.contact);
|
||||
this.enabled = cfg.token.trim() !== '';
|
||||
if (!this.enabled) {
|
||||
console.warn(
|
||||
'[Discogs] DISCOGS_TOKEN not set; client disabled (methods return null).'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async get<T>(path: string): Promise<T> {
|
||||
return requestJson<T>(`${this.cfg.baseUrl}${path}`, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: this.cfg.minIntervalMs,
|
||||
headers: { Authorization: `Discogs token=${this.cfg.token}` },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the database for the best release match for artist/album and return
|
||||
* its id, year, genres/styles and cover image URLs, or null if no match (or
|
||||
* the client is disabled / errors).
|
||||
*/
|
||||
async searchRelease(artist: string, album: string): Promise<DiscogsRelease | null> {
|
||||
if (!this.enabled) return null;
|
||||
if (artist.trim() === '' || album.trim() === '') return null;
|
||||
|
||||
const qs = new URLSearchParams({
|
||||
type: 'release',
|
||||
artist,
|
||||
release_title: album,
|
||||
});
|
||||
const path = `/database/search?${qs.toString()}`;
|
||||
|
||||
try {
|
||||
const data = await this.get<DiscogsSearchResponse>(path);
|
||||
const best = (data.results ?? [])[0];
|
||||
if (!best || typeof best.id !== 'number') return null;
|
||||
|
||||
const year =
|
||||
best.year !== undefined && Number.isFinite(Number(best.year))
|
||||
? Number(best.year)
|
||||
: null;
|
||||
|
||||
return {
|
||||
discogsId: best.id,
|
||||
year,
|
||||
genres: best.genre ?? [],
|
||||
styles: best.style ?? [],
|
||||
coverImage: best.cover_image ?? null,
|
||||
thumb: best.thumb ?? null,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[Discogs] searchRelease failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch artist images from Discogs using MusicBrainz ID for exact match.
|
||||
* Does NOT fall back to name-based search (too error-prone for common names).
|
||||
* Returns null if no MBID provided or no match found.
|
||||
*/
|
||||
async getArtistImageUrl(_artistName: string, artistMbid?: string): Promise<string | null> {
|
||||
if (!this.enabled || !artistMbid?.trim()) return null;
|
||||
|
||||
try {
|
||||
const mbPath = `/artists/${encodeURIComponent(artistMbid)}`;
|
||||
const data = await this.get<DiscogsArtistResponse>(mbPath);
|
||||
const images = data.images ?? [];
|
||||
|
||||
// Prefer primary image, then first available
|
||||
const primary = images.find((img) => img.type === 'primary') ?? images[0];
|
||||
return primary?.uri150 ?? primary?.uri ?? null;
|
||||
} catch (err) {
|
||||
console.warn('[Discogs] getArtistImageUrl failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// Fanart.tv client for artist images.
|
||||
// API: https://fanart.tv/2015/03/api/
|
||||
// Requires API key (free tier available).
|
||||
|
||||
import { requestJson } from './http.js';
|
||||
|
||||
interface FanartArtistResponse {
|
||||
artistbackground?: Array<{ url: string }>;
|
||||
artistthumb?: Array<{ url: string }>;
|
||||
hdartistclearart?: Array<{ url: string }>;
|
||||
artistlogo?: Array<{ url: string }>;
|
||||
musicbanner?: Array<{ url: string }>;
|
||||
}
|
||||
|
||||
export class FanartClient {
|
||||
private readonly baseUrl = 'https://webservice.fanart.tv/v3/music';
|
||||
private readonly apiKey: string;
|
||||
private readonly userAgent = 'muzick/0.1';
|
||||
|
||||
constructor(apiKey?: string) {
|
||||
this.apiKey = apiKey || process.env.FANART_API_KEY || '';
|
||||
}
|
||||
|
||||
private get enabled(): boolean {
|
||||
return this.apiKey.trim() !== '';
|
||||
}
|
||||
|
||||
async getArtistImages(mbid: string): Promise<{
|
||||
background?: string;
|
||||
thumb?: string;
|
||||
clearart?: string;
|
||||
logo?: string;
|
||||
banner?: string;
|
||||
} | null> {
|
||||
if (!this.enabled) return null;
|
||||
if (!mbid) return null;
|
||||
|
||||
const url = `${this.baseUrl}/${encodeURIComponent(mbid)}?api_key=${encodeURIComponent(this.apiKey)}`;
|
||||
|
||||
try {
|
||||
const data = await requestJson<FanartArtistResponse>(url, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: 1000, // Be respectful
|
||||
});
|
||||
|
||||
return {
|
||||
background: data.artistbackground?.[0]?.url,
|
||||
thumb: data.artistthumb?.[0]?.url,
|
||||
clearart: data.hdartistclearart?.[0]?.url,
|
||||
logo: data.artistlogo?.[0]?.url,
|
||||
banner: data.musicbanner?.[0]?.url,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[Fanart.tv] getArtistImages failed:', (err as Error).message);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the best available artist image.
|
||||
* Priority: thumb > clearart > logo > background > banner
|
||||
*/
|
||||
async getBestArtistImage(mbid: string): Promise<string | null> {
|
||||
const images = await this.getArtistImages(mbid);
|
||||
if (!images) return null;
|
||||
|
||||
return images.thumb ?? images.clearart ?? images.logo ?? images.background ?? images.banner ?? null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
// HTTP base shared by every external-integration client.
|
||||
//
|
||||
// Provides:
|
||||
// - configurable User-Agent (REQUIRED by MusicBrainz, format:
|
||||
// `muzick/0.1 ( <contact> )`)
|
||||
// - SOCKS5 proxy support via SOCKS_PROXY_URL env var
|
||||
// (e.g. `socks5://127.0.0.1:10808`; no proxy when unset/empty)
|
||||
// - exponential backoff with jitter on 429 / 5xx / network errors,
|
||||
// honouring a `Retry-After` header when present
|
||||
// - a simple per-host rate limit (min interval between requests to the same
|
||||
// host) so e.g. MusicBrainz's <=1 req/sec rule is respected
|
||||
// - typed JSON parsing via generics and a typed error on persistent failure.
|
||||
//
|
||||
// Why not global fetch? Node's built-in fetch doesn't support proxy agents.
|
||||
// When SOCKS_PROXY_URL is set we build a proxy-aware fetcher via
|
||||
// socks-proxy-agent + native https.request. Otherwise we use the global fetch
|
||||
// (faster path, no overhead).
|
||||
|
||||
import * as http from 'http';
|
||||
import * as https from 'https';
|
||||
import { SocksProxyAgent } from 'socks-proxy-agent';
|
||||
|
||||
/** Error thrown when a request ultimately fails (non-2xx after all retries). */
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly status: number,
|
||||
readonly url: string,
|
||||
readonly body?: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'HttpError';
|
||||
}
|
||||
}
|
||||
|
||||
// ── Proxy-aware fetch factory ────────────────────────────────────────────────
|
||||
|
||||
const SOCKS_PROXY_URL = process.env.SOCKS_PROXY_URL || '';
|
||||
|
||||
/**
|
||||
* Return a `fetch`-compatible function that routes through the configured
|
||||
* SOCKS5 proxy (if any), or the bare global fetch when no proxy is needed.
|
||||
*/
|
||||
function createFetch(): typeof fetch {
|
||||
if (!SOCKS_PROXY_URL) return globalThis.fetch.bind(globalThis);
|
||||
|
||||
const agent = new SocksProxyAgent(SOCKS_PROXY_URL);
|
||||
|
||||
// Internal helper: issue a single request (no redirect following).
|
||||
function requestOnce(
|
||||
url: string,
|
||||
method: string,
|
||||
headers: Record<string, string> | undefined,
|
||||
body: BodyInit | undefined,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<Response> {
|
||||
const urlObj = new URL(url);
|
||||
const mod = urlObj.protocol === 'https:' ? https : http;
|
||||
|
||||
return new Promise<Response>((resolve, reject) => {
|
||||
const req = mod.request(url, { agent, method, headers, signal }, (res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
res.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
res.on('end', () => {
|
||||
resolve(
|
||||
new Response(Buffer.concat(chunks), {
|
||||
status: res.statusCode,
|
||||
statusText: res.statusMessage || '',
|
||||
headers: new Headers(
|
||||
Object.entries(res.headers).reduce(
|
||||
(acc, [k, v]) => {
|
||||
acc[k] = Array.isArray(v) ? v.join(', ') : String(v ?? '');
|
||||
return acc;
|
||||
},
|
||||
{} as Record<string, string>,
|
||||
)
|
||||
),
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
req.on('error', reject);
|
||||
if (body) req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
const MAX_REDIRECTS = 5;
|
||||
|
||||
return async (input, init): Promise<Response> => {
|
||||
let url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||
const method = init?.method || 'GET';
|
||||
const headers = init?.headers as Record<string, string> | undefined;
|
||||
const body = init?.body ?? undefined;
|
||||
const signal = init?.signal ?? undefined;
|
||||
|
||||
for (let i = 0; i <= MAX_REDIRECTS; i++) {
|
||||
const res = await requestOnce(url, method, headers, body, signal);
|
||||
const code = res.status;
|
||||
|
||||
// Follow redirect (301, 302, 307, 308) — max 5 hops.
|
||||
if (code === 301 || code === 302 || code === 307 || code === 308) {
|
||||
const location = res.headers.get('location');
|
||||
if (!location) return res;
|
||||
url = new URL(location, url).href;
|
||||
continue;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// Too many redirects.
|
||||
return new Response(null, { status: 508, statusText: 'Loop Detected' });
|
||||
};
|
||||
}
|
||||
|
||||
// Singleton fetcher — created once at module load.
|
||||
const fetchFn = createFetch();
|
||||
|
||||
export interface RequestJsonOptions {
|
||||
/** Full User-Agent header value. Required for MusicBrainz; recommended always. */
|
||||
userAgent: string;
|
||||
/** Minimum interval (ms) between requests to the same host. Default 1000. */
|
||||
minIntervalMs?: number;
|
||||
/** Max retry attempts on transient failures. Default 5. */
|
||||
maxRetries?: number;
|
||||
/** Base backoff delay (ms). Default 500. */
|
||||
baseDelayMs?: number;
|
||||
/** Cap on a single backoff delay (ms). Default 20000. */
|
||||
maxDelayMs?: number;
|
||||
/** Extra request headers (e.g. Accept). */
|
||||
headers?: Record<string, string>;
|
||||
/** Per-request timeout (ms). Default 15000. */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
// Per-host timestamp of the last request start, used to enforce the rate limit.
|
||||
// Module-scoped so all clients sharing a host coordinate automatically.
|
||||
const lastRequestAt = new Map<string, number>();
|
||||
|
||||
const delay = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function hostOf(url: string): string {
|
||||
try {
|
||||
return new URL(url).host;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
/** Block until at least `minIntervalMs` has elapsed since this host's last request. */
|
||||
async function throttle(host: string, minIntervalMs: number): Promise<void> {
|
||||
const now = Date.now();
|
||||
const last = lastRequestAt.get(host);
|
||||
if (last !== undefined) {
|
||||
const wait = minIntervalMs - (now - last);
|
||||
if (wait > 0) await delay(wait);
|
||||
}
|
||||
lastRequestAt.set(host, Date.now());
|
||||
}
|
||||
|
||||
/** Parse a Retry-After header (delta-seconds or HTTP date) into ms, or null. */
|
||||
function parseRetryAfter(value: string | null): number | null {
|
||||
if (!value) return null;
|
||||
const secs = Number(value);
|
||||
if (Number.isFinite(secs)) return Math.max(0, secs * 1000);
|
||||
const when = Date.parse(value);
|
||||
if (!Number.isNaN(when)) return Math.max(0, when - Date.now());
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Exponential backoff with full jitter, capped at maxDelayMs. */
|
||||
function backoffDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number {
|
||||
const exp = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
|
||||
return Math.floor(Math.random() * exp);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a URL and parse the JSON body as `T`. Retries transient failures with
|
||||
* backoff + jitter; enforces the per-host rate limit. Throws `HttpError` on a
|
||||
* persistent non-2xx response and rethrows the last network error if all
|
||||
* attempts fail.
|
||||
*/
|
||||
export async function requestJson<T>(
|
||||
url: string,
|
||||
opts: RequestJsonOptions
|
||||
): Promise<T> {
|
||||
const {
|
||||
userAgent,
|
||||
minIntervalMs = 1000,
|
||||
maxRetries = 5,
|
||||
baseDelayMs = 500,
|
||||
maxDelayMs = 20000,
|
||||
headers = {},
|
||||
timeoutMs = 15000,
|
||||
} = opts;
|
||||
|
||||
const host = hostOf(url);
|
||||
let lastError: unknown;
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
await throttle(host, minIntervalMs);
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetchFn(url, {
|
||||
headers: {
|
||||
'User-Agent': userAgent,
|
||||
Accept: 'application/json',
|
||||
...headers,
|
||||
},
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
// Retry on 429 (rate limited) and 5xx (transient server errors).
|
||||
if (res.status === 429 || res.status >= 500) {
|
||||
if (attempt < maxRetries) {
|
||||
const retryAfter = parseRetryAfter(res.headers.get('retry-after'));
|
||||
await delay(retryAfter ?? backoffDelay(attempt, baseDelayMs, maxDelayMs));
|
||||
continue;
|
||||
}
|
||||
const body = await res.text().catch(() => undefined);
|
||||
throw new HttpError(
|
||||
`Request failed after ${maxRetries} retries: ${res.status} ${res.statusText}`,
|
||||
res.status,
|
||||
url,
|
||||
body
|
||||
);
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
const body = await res.text().catch(() => undefined);
|
||||
throw new HttpError(
|
||||
`Request failed: ${res.status} ${res.statusText}`,
|
||||
res.status,
|
||||
url,
|
||||
body
|
||||
);
|
||||
}
|
||||
|
||||
return (await res.json()) as T;
|
||||
} catch (err) {
|
||||
// Non-retryable application errors propagate immediately.
|
||||
if (err instanceof HttpError) throw err;
|
||||
|
||||
// Network / abort errors: retry with backoff, otherwise rethrow.
|
||||
lastError = err;
|
||||
if (attempt < maxRetries) {
|
||||
await delay(backoffDelay(attempt, baseDelayMs, maxDelayMs));
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
// Unreachable in practice; satisfies the type checker.
|
||||
throw lastError instanceof Error
|
||||
? lastError
|
||||
: new Error('requestJson: exhausted retries');
|
||||
}
|
||||
|
||||
/** Build the MusicBrainz-compliant User-Agent string. */
|
||||
export function buildUserAgent(contact: string): string {
|
||||
const c = contact && contact.trim() !== '' ? contact.trim() : 'no-contact-configured';
|
||||
return `muzick/0.1 ( ${c} )`;
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// Public surface of the external-integration client layer.
|
||||
export { integrationsConfig } from './config.js';
|
||||
export type {
|
||||
IntegrationsConfig,
|
||||
MusicBrainzConfig,
|
||||
LastFmConfig,
|
||||
DiscogsConfig,
|
||||
LrcLibConfig,
|
||||
CoverArtConfig,
|
||||
WikimediaConfig,
|
||||
} from './config.js';
|
||||
|
||||
export { requestJson, buildUserAgent, HttpError } from './http.js';
|
||||
export type { RequestJsonOptions } from './http.js';
|
||||
|
||||
export { MusicBrainzClient } from './musicbrainz.client.js';
|
||||
export type { RecordingMatch, RecordingDetail, RecordingCredit, ArtistTag as MusicBrainzArtistTag } from './musicbrainz.client.js';
|
||||
|
||||
export { LastFmClient } from './lastfm.client.js';
|
||||
export type {
|
||||
SimilarArtist,
|
||||
SimilarTrack,
|
||||
ArtistTag as LastFmArtistTag,
|
||||
} from './lastfm.client.js';
|
||||
|
||||
export { DiscogsClient } from './discogs.client.js';
|
||||
export type { DiscogsRelease } from './discogs.client.js';
|
||||
|
||||
export { LrcLibClient } from './lrclib.client.js';
|
||||
export type { LyricsResult } from './lrclib.client.js';
|
||||
|
||||
export { CoverArtClient } from './coverart.client.js';
|
||||
|
||||
export { ITunesClient, upscaleITunesArtwork } from './itunes.client.js';
|
||||
export type { ITunesAlbum } from './itunes.client.js';
|
||||
|
||||
export { DeezerClient } from './deezer.client.js';
|
||||
export type { DeezerAlbum } from './deezer.client.js';
|
||||
|
||||
export { WikimediaClient } from './wikimedia.client.js';
|
||||
|
||||
export { WikidataClient } from './wikidata.client.js';
|
||||
|
||||
export { FanartClient } from './fanart.client.js';
|
||||
|
||||
export { TheAudioDbClient } from './theaudiodb.client.js';
|
||||
@@ -0,0 +1,98 @@
|
||||
// iTunes Search API client (read-only).
|
||||
//
|
||||
// The iTunes Search API is free, requires no API key, and has excellent
|
||||
// coverage — nearly every released album is present. We use the album search
|
||||
// to resolve cover art URLs. The default artwork URL returned is 100×100, but
|
||||
// can be upgraded to 600×600 by swapping the size suffix in the URL string.
|
||||
//
|
||||
// Rate limit: Apple asks for "moderate" use (~20 req/min). We enforce a polite
|
||||
// per-host interval via the http base.
|
||||
//
|
||||
// Best-effort: on any error, methods log a warning and return null rather than
|
||||
// throwing, so enrichment never breaks the worker.
|
||||
|
||||
import { buildUserAgent, requestJson } from './http.js';
|
||||
|
||||
/** A normalised iTunes album search result (only the fields we read). */
|
||||
export interface ITunesAlbum {
|
||||
/** Artist canonical name. */
|
||||
artistName: string;
|
||||
/** Album / collection title. */
|
||||
collectionName: string;
|
||||
/** Cover art URL (100×100 — caller upscales). */
|
||||
artworkUrl100: string;
|
||||
}
|
||||
|
||||
interface ITunesSearchResult {
|
||||
artistName?: string;
|
||||
collectionName?: string;
|
||||
artworkUrl100?: string;
|
||||
}
|
||||
|
||||
interface ITunesSearchResponse {
|
||||
results?: ITunesSearchResult[];
|
||||
}
|
||||
|
||||
export class ITunesClient {
|
||||
private readonly baseUrl = 'https://itunes.apple.com';
|
||||
private readonly userAgent: string;
|
||||
private readonly minIntervalMs: number;
|
||||
|
||||
constructor(contact = '', minIntervalMs = 3000) {
|
||||
this.userAgent = buildUserAgent(contact);
|
||||
this.minIntervalMs = minIntervalMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the iTunes catalog for an album by artist + title and return the
|
||||
* best match, or null if no match / error. The returned `artworkUrl100` is
|
||||
* the 100×100 thumbnail; callers upscale it to 600×600 via `upscaleArtwork`.
|
||||
*/
|
||||
async searchAlbum(artist: string, album: string): Promise<ITunesAlbum | null> {
|
||||
if (artist.trim() === '' || album.trim() === '') return null;
|
||||
|
||||
// The iTunes search `term` is a free-text query; we quote both values to
|
||||
// narrow the match. entity=album restricts to album collections.
|
||||
const term = `${artist} ${album}`;
|
||||
const qs = new URLSearchParams({ term, entity: 'album', limit: '5' });
|
||||
const url = `${this.baseUrl}/search?${qs.toString()}`;
|
||||
|
||||
try {
|
||||
const data = await requestJson<ITunesSearchResponse>(url, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: this.minIntervalMs,
|
||||
});
|
||||
const results = data.results ?? [];
|
||||
if (results.length === 0) return null;
|
||||
|
||||
// Prefer an exact title match (case-insensitive) to avoid grabbing a
|
||||
// "Greatest Hits" when searching for a studio album, then fall back to
|
||||
// the first result.
|
||||
const lowerAlbum = album.toLowerCase();
|
||||
const exact = results.find(
|
||||
(r) => r.collectionName?.toLowerCase() === lowerAlbum
|
||||
);
|
||||
const best = exact ?? results[0];
|
||||
|
||||
if (!best.artworkUrl100) return null;
|
||||
return {
|
||||
artistName: best.artistName ?? artist,
|
||||
collectionName: best.collectionName ?? album,
|
||||
artworkUrl100: best.artworkUrl100,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[iTunes] searchAlbum failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upscale an iTunes 100×100 artwork URL to a larger size. iTunes serves
|
||||
* artwork from a CDN that supports arbitrary sizes via the `NNNxNNN` path
|
||||
* segment (e.g. 600x600). Returns the input unchanged if it doesn't match the
|
||||
* expected shape.
|
||||
*/
|
||||
export function upscaleITunesArtwork(url: string, size = 600): string {
|
||||
return url.replace(/\/\d+x\d+(bb)?\.(jpg|png)$/, `/${size}x${size}bb.$2`);
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
// Last.fm 2.0 API client (read-only methods).
|
||||
//
|
||||
// Every call appends api_key + format=json and goes through the http base, which
|
||||
// enforces a small per-host rate limit. Best-effort: with no API key configured,
|
||||
// or on any error, methods return [] rather than throwing.
|
||||
|
||||
import { integrationsConfig, LastFmConfig } from './config.js';
|
||||
import { buildUserAgent, requestJson } from './http.js';
|
||||
|
||||
export interface SimilarArtist {
|
||||
name: string;
|
||||
/** Similarity 0..1 as reported by Last.fm. */
|
||||
match: number;
|
||||
}
|
||||
|
||||
export interface ArtistTag {
|
||||
name: string;
|
||||
/** Normalised 0..1 weight (relative to the strongest tag). */
|
||||
weight: number;
|
||||
}
|
||||
|
||||
export interface SimilarTrack {
|
||||
name: string;
|
||||
artist: string;
|
||||
/** Similarity 0..1 as reported by Last.fm. */
|
||||
match: number;
|
||||
}
|
||||
|
||||
// --- Raw Last.fm response shapes (only the fields we read) -----------------
|
||||
|
||||
interface LfmSimilarArtist {
|
||||
name?: string;
|
||||
match?: string | number;
|
||||
}
|
||||
interface LfmSimilarArtistsResponse {
|
||||
similarartists?: { artist?: LfmSimilarArtist[] };
|
||||
}
|
||||
|
||||
interface LfmTag {
|
||||
name?: string;
|
||||
count?: string | number;
|
||||
}
|
||||
interface LfmTopTagsResponse {
|
||||
toptags?: { tag?: LfmTag[] };
|
||||
}
|
||||
|
||||
interface LfmSimilarTrack {
|
||||
name?: string;
|
||||
match?: string | number;
|
||||
artist?: { name?: string };
|
||||
}
|
||||
interface LfmSimilarTracksResponse {
|
||||
similartracks?: { track?: LfmSimilarTrack[] };
|
||||
}
|
||||
|
||||
interface LfmImage {
|
||||
'#text'?: string;
|
||||
size?: string;
|
||||
}
|
||||
interface LfmArtistInfoResponse {
|
||||
artist?: { image?: LfmImage[] };
|
||||
}
|
||||
|
||||
const toNum = (v: string | number | undefined): number => {
|
||||
const n = typeof v === 'number' ? v : Number(v);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
};
|
||||
|
||||
export class LastFmClient {
|
||||
private readonly cfg: LastFmConfig;
|
||||
private readonly userAgent: string;
|
||||
private readonly enabled: boolean;
|
||||
|
||||
constructor(cfg: LastFmConfig = integrationsConfig.lastfm) {
|
||||
this.cfg = cfg;
|
||||
this.userAgent = buildUserAgent(integrationsConfig.musicbrainz.contact);
|
||||
this.enabled = cfg.apiKey.trim() !== '';
|
||||
if (!this.enabled) {
|
||||
console.warn(
|
||||
'[Last.fm] LASTFM_API_KEY not set; client disabled (methods return []).'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a method URL with the supplied params plus api_key + format=json. */
|
||||
private url(method: string, params: Record<string, string | number>): string {
|
||||
const qs = new URLSearchParams({ method, api_key: this.cfg.apiKey, format: 'json' });
|
||||
for (const [k, v] of Object.entries(params)) qs.set(k, String(v));
|
||||
return `${this.cfg.baseUrl}/?${qs.toString()}`;
|
||||
}
|
||||
|
||||
private async get<T>(method: string, params: Record<string, string | number>): Promise<T> {
|
||||
return requestJson<T>(this.url(method, params), {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: this.cfg.minIntervalMs,
|
||||
});
|
||||
}
|
||||
|
||||
/** artist.getsimilar -> up to `limit` similar artists with 0..1 match. */
|
||||
async getSimilarArtists(artist: string, limit = 20): Promise<SimilarArtist[]> {
|
||||
if (!this.enabled || artist.trim() === '') return [];
|
||||
try {
|
||||
const data = await this.get<LfmSimilarArtistsResponse>('artist.getsimilar', {
|
||||
artist,
|
||||
limit,
|
||||
autocorrect: 1,
|
||||
});
|
||||
const list = data.similarartists?.artist ?? [];
|
||||
return list
|
||||
.filter((a): a is LfmSimilarArtist & { name: string } => !!a.name)
|
||||
.map((a) => ({ name: a.name, match: toNum(a.match) }));
|
||||
} catch (err) {
|
||||
console.warn('[Last.fm] getSimilarArtists failed:', (err as Error).message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** artist.gettoptags -> tags with count normalised to 0..1. */
|
||||
async getArtistTopTags(artist: string): Promise<ArtistTag[]> {
|
||||
if (!this.enabled || artist.trim() === '') return [];
|
||||
try {
|
||||
const data = await this.get<LfmTopTagsResponse>('artist.gettoptags', {
|
||||
artist,
|
||||
autocorrect: 1,
|
||||
});
|
||||
const tags = (data.toptags?.tag ?? []).filter(
|
||||
(t): t is LfmTag & { name: string } => !!t.name
|
||||
);
|
||||
if (tags.length === 0) return [];
|
||||
const maxCount = Math.max(...tags.map((t) => toNum(t.count)), 1);
|
||||
return tags
|
||||
.map((t) => ({ name: t.name, weight: toNum(t.count) / maxCount }))
|
||||
.sort((a, b) => b.weight - a.weight);
|
||||
} catch (err) {
|
||||
console.warn('[Last.fm] getArtistTopTags failed:', (err as Error).message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** artist.getinfo -> best available image URL, or null. */
|
||||
async getArtistImageUrl(artist: string): Promise<string | null> {
|
||||
if (!this.enabled || artist.trim() === '') return null;
|
||||
try {
|
||||
const data = await this.get<LfmArtistInfoResponse>('artist.getinfo', {
|
||||
artist,
|
||||
autocorrect: 1,
|
||||
});
|
||||
const images = data.artist?.image ?? [];
|
||||
// Prefer 'extralarge', fall back through sizes in descending order.
|
||||
for (const size of ['extralarge', 'large', 'medium', 'small']) {
|
||||
const img = images.find((i) => i.size === size);
|
||||
const url = img?.['#text']?.trim();
|
||||
if (url && !url.includes('2a96cbd8b46e442fc41c2b86b821562f')) return url;
|
||||
}
|
||||
return null;
|
||||
} catch (err) {
|
||||
console.warn('[Last.fm] getArtistImageUrl failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** track.getsimilar -> up to `limit` similar tracks with 0..1 match. */
|
||||
async getSimilarTracks(
|
||||
artist: string,
|
||||
track: string,
|
||||
limit = 20
|
||||
): Promise<SimilarTrack[]> {
|
||||
if (!this.enabled || artist.trim() === '' || track.trim() === '') return [];
|
||||
try {
|
||||
const data = await this.get<LfmSimilarTracksResponse>('track.getsimilar', {
|
||||
artist,
|
||||
track,
|
||||
limit,
|
||||
autocorrect: 1,
|
||||
});
|
||||
const list = data.similartracks?.track ?? [];
|
||||
return list
|
||||
.filter((t): t is LfmSimilarTrack & { name: string } => !!t.name)
|
||||
.map((t) => ({
|
||||
name: t.name,
|
||||
artist: t.artist?.name ?? '',
|
||||
match: toNum(t.match),
|
||||
}));
|
||||
} catch (err) {
|
||||
console.warn('[Last.fm] getSimilarTracks failed:', (err as Error).message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// LRCLib client (lyrics lookup).
|
||||
//
|
||||
// No API key is required, so the client is always enabled. A polite per-host
|
||||
// rate limit (~250ms) is enforced via the http base.
|
||||
//
|
||||
// LRCLib's /get endpoint matches on track/artist/album/duration and returns 404
|
||||
// when nothing matches; that 404 is treated as "no lyrics" (null), not an error.
|
||||
// Any other failure also degrades to null so enrichment never breaks the worker.
|
||||
|
||||
import { integrationsConfig, LrcLibConfig } from './config.js';
|
||||
import { buildUserAgent, requestJson, HttpError } from './http.js';
|
||||
|
||||
export interface LyricsResult {
|
||||
/** Plain-text lyrics, if available. */
|
||||
plainLyrics: string | null;
|
||||
/** Time-synced lyrics as an LRC string, if available. */
|
||||
syncedLyrics: string | null;
|
||||
provider: 'lrclib';
|
||||
}
|
||||
|
||||
// --- Raw LRCLib response shape (only the fields we read) -------------------
|
||||
|
||||
interface LrcLibGetResponse {
|
||||
plainLyrics?: string | null;
|
||||
syncedLyrics?: string | null;
|
||||
}
|
||||
|
||||
export class LrcLibClient {
|
||||
private readonly cfg: LrcLibConfig;
|
||||
private readonly userAgent: string;
|
||||
|
||||
constructor(cfg: LrcLibConfig = integrationsConfig.lrclib) {
|
||||
this.cfg = cfg;
|
||||
this.userAgent = buildUserAgent(integrationsConfig.musicbrainz.contact);
|
||||
}
|
||||
|
||||
private async get<T>(path: string): Promise<T> {
|
||||
return requestJson<T>(`${this.cfg.baseUrl}${path}`, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: this.cfg.minIntervalMs,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up lyrics for a track. Returns plain + synced lyrics, or null when no
|
||||
* match (404) or on any error.
|
||||
*/
|
||||
async getLyrics(
|
||||
artist: string,
|
||||
title: string,
|
||||
album?: string,
|
||||
durationSec?: number
|
||||
): Promise<LyricsResult | null> {
|
||||
if (artist.trim() === '' || title.trim() === '') return null;
|
||||
|
||||
const qs = new URLSearchParams({ track_name: title, artist_name: artist });
|
||||
if (album && album.trim() !== '') qs.set('album_name', album);
|
||||
if (durationSec !== undefined && Number.isFinite(durationSec)) {
|
||||
qs.set('duration', String(Math.round(durationSec)));
|
||||
}
|
||||
const path = `/get?${qs.toString()}`;
|
||||
|
||||
try {
|
||||
const data = await this.get<LrcLibGetResponse>(path);
|
||||
const plainLyrics = data.plainLyrics ?? null;
|
||||
const syncedLyrics = data.syncedLyrics ?? null;
|
||||
if (plainLyrics === null && syncedLyrics === null) return null;
|
||||
return { plainLyrics, syncedLyrics, provider: 'lrclib' };
|
||||
} catch (err) {
|
||||
// 404 simply means no lyrics matched; treat as a clean miss.
|
||||
if (err instanceof HttpError && err.status === 404) return null;
|
||||
console.warn('[LRCLib] getLyrics failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
// MusicBrainz web-service v2 client.
|
||||
//
|
||||
// IMPORTANT MB requirements (https://musicbrainz.org/doc/MusicBrainz_API):
|
||||
// - A meaningful User-Agent is REQUIRED (set via http base, see buildUserAgent).
|
||||
// - Anonymous clients must not exceed 1 request/second. We enforce this through
|
||||
// the http base's per-host rate limit (minIntervalMs = 1000 by default).
|
||||
//
|
||||
// All methods are best-effort: on missing contact config or any error they log a
|
||||
// warning and return null / [] rather than throwing, so enrichment never breaks
|
||||
// the worker.
|
||||
|
||||
import { integrationsConfig, MusicBrainzConfig } from './config.js';
|
||||
import { buildUserAgent, requestJson } from './http.js';
|
||||
|
||||
/** Result of a recording lookup: the chosen recording plus linked entities. */
|
||||
export interface RecordingMatch {
|
||||
recordingMbid: string;
|
||||
title: string;
|
||||
/** Primary credited artist, if any. */
|
||||
artistMbid: string | null;
|
||||
artistName: string | null;
|
||||
/** First associated release (album), if any. */
|
||||
releaseMbid: string | null;
|
||||
releaseTitle: string | null;
|
||||
/** MB search score (0..100), surfaced for callers that want confidence. */
|
||||
score: number;
|
||||
}
|
||||
|
||||
/** Result of an artist search: the canonical identity for a name query. */
|
||||
export interface ArtistMatch {
|
||||
artistMbid: string;
|
||||
/** MB canonical display name (e.g. "P!nk" not "Pink"). */
|
||||
name: string;
|
||||
/** MB sort-name (e.g. "Pink, P!" / "Beatles, The"). */
|
||||
sortName: string | null;
|
||||
/** MB search score (0..100). */
|
||||
score: number;
|
||||
/** Disambiguation comment when ambiguous (e.g. "US rock band"). */
|
||||
disambiguation: string | null;
|
||||
}
|
||||
|
||||
/** Result of a release-group search: the canonical album identity. */
|
||||
export interface ReleaseGroupMatch {
|
||||
releaseGroupMbid: string;
|
||||
title: string;
|
||||
/** Primary artist MBID on the release-group, if any. */
|
||||
artistMbid: string | null;
|
||||
artistName: string | null;
|
||||
/** First release date as a 4-digit year, if available. */
|
||||
year: number | null;
|
||||
/** First release date as an ISO string (YYYY-MM-DD or YYYY-MM or YYYY), if available. */
|
||||
firstReleaseDate: string | null;
|
||||
/** MB search score (0..100). */
|
||||
score: number;
|
||||
}
|
||||
|
||||
/** A normalised genre/tag with a 0..1 weight. */
|
||||
export interface ArtistTag {
|
||||
name: string;
|
||||
/** Normalised 0..1 weight (relative to the strongest tag for this artist). */
|
||||
weight: number;
|
||||
}
|
||||
|
||||
/** One entry in a MusicBrainz artist-credit array. */
|
||||
export interface RecordingCredit {
|
||||
/** The MB artist ID (UUID). */
|
||||
artistMbid: string;
|
||||
/** Canonical display name from the artist object. */
|
||||
artistName: string;
|
||||
/** Name as credited (may differ, e.g. "P!nk" vs "Pink"). */
|
||||
creditName: string;
|
||||
/** Joinphrase that follows this credit (" feat. ", " & ", etc.). */
|
||||
joinphrase?: string;
|
||||
}
|
||||
|
||||
/** Full recording fetched by MBID with inc=artist-credits. */
|
||||
export interface RecordingDetail {
|
||||
recordingMbid: string;
|
||||
title: string;
|
||||
artistCredit: RecordingCredit[];
|
||||
}
|
||||
|
||||
/** Full release-group fetched by MBID with inc=artists. */
|
||||
export interface ReleaseGroupDetail {
|
||||
releaseGroupMbid: string;
|
||||
title: string;
|
||||
artistCredit: RecordingCredit[];
|
||||
}
|
||||
|
||||
/** One artist-relation entry on an artist-rels lookup. */
|
||||
export interface ArtistRelation {
|
||||
/** MB relation type label, e.g. "member of", "is performance name of". */
|
||||
type: string;
|
||||
/** "forward" or "backward". */
|
||||
direction: string;
|
||||
/** The related artist's MBID. */
|
||||
targetMbid: string;
|
||||
/** The related artist's canonical display name. */
|
||||
targetName: string;
|
||||
}
|
||||
|
||||
// --- Raw MB response shapes (only the fields we read) ----------------------
|
||||
|
||||
interface MbArtistCredit {
|
||||
artist?: { id?: string; name?: string };
|
||||
name?: string;
|
||||
}
|
||||
|
||||
interface MbReleaseRef {
|
||||
id?: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
interface MbRecording {
|
||||
id: string;
|
||||
title: string;
|
||||
score?: number;
|
||||
'artist-credit'?: MbArtistCredit[];
|
||||
releases?: MbReleaseRef[];
|
||||
}
|
||||
|
||||
interface MbRecordingDetail {
|
||||
id: string;
|
||||
title: string;
|
||||
'artist-credit'?: Array<{
|
||||
artist?: { id?: string; name?: string };
|
||||
name?: string;
|
||||
joinphrase?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface MbRecordingSearchResponse {
|
||||
recordings?: MbRecording[];
|
||||
}
|
||||
|
||||
interface MbReleaseGroupDetail {
|
||||
id: string;
|
||||
title?: string;
|
||||
'artist-credit'?: Array<{
|
||||
artist?: { id?: string; name?: string };
|
||||
name?: string;
|
||||
joinphrase?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface MbRelation {
|
||||
type?: string;
|
||||
direction?: string;
|
||||
// MusicBrainz returns the related entity under a typed key (`artist` for
|
||||
// artist-rels); recent MB versions may also surface a generic `target`
|
||||
// string. We read `artist.id` / `artist.name` defensively.
|
||||
artist?: { id?: string; name?: string };
|
||||
target?: string;
|
||||
}
|
||||
|
||||
interface MbRelationList {
|
||||
'target-type'?: string;
|
||||
relations?: MbRelation[];
|
||||
}
|
||||
|
||||
interface MbArtistRelationsResponse {
|
||||
id?: string;
|
||||
name?: string;
|
||||
// Modern MB: top-level `relations` array.
|
||||
relations?: MbRelation[];
|
||||
// Older MB: `relation-list` array, each grouping relations by target-type.
|
||||
'relation-list'?: MbRelationList[];
|
||||
}
|
||||
|
||||
interface MbTag {
|
||||
name?: string;
|
||||
count?: number;
|
||||
}
|
||||
|
||||
interface MbGenre {
|
||||
name?: string;
|
||||
count?: number;
|
||||
}
|
||||
|
||||
interface MbArtistResponse {
|
||||
tags?: MbTag[];
|
||||
genres?: MbGenre[];
|
||||
}
|
||||
|
||||
interface MbArtistSearch {
|
||||
id: string;
|
||||
name?: string;
|
||||
'sort-name'?: string;
|
||||
score?: number;
|
||||
disambiguation?: string;
|
||||
}
|
||||
|
||||
interface MbArtistSearchResponse {
|
||||
artists?: MbArtistSearch[];
|
||||
}
|
||||
|
||||
interface MbReleaseGroupSearch {
|
||||
id: string;
|
||||
title?: string;
|
||||
score?: number;
|
||||
'first-release-date'?: string;
|
||||
'artist-credit'?: MbArtistCredit[];
|
||||
}
|
||||
|
||||
interface MbReleaseGroupSearchResponse {
|
||||
'release-groups'?: MbReleaseGroupSearch[];
|
||||
}
|
||||
|
||||
export class MusicBrainzClient {
|
||||
private readonly cfg: MusicBrainzConfig;
|
||||
private readonly userAgent: string;
|
||||
private readonly enabled: boolean;
|
||||
|
||||
constructor(cfg: MusicBrainzConfig = integrationsConfig.musicbrainz) {
|
||||
this.cfg = cfg;
|
||||
this.userAgent = buildUserAgent(cfg.contact);
|
||||
this.enabled = cfg.contact.trim() !== '';
|
||||
if (!this.enabled) {
|
||||
console.warn(
|
||||
'[MusicBrainz] MUSICBRAINZ_CONTACT not set; client disabled (methods return null/[]).'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async get<T>(path: string): Promise<T> {
|
||||
return requestJson<T>(`${this.cfg.baseUrl}${path}`, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: this.cfg.minIntervalMs,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the recording endpoint for the best match for artist/title(/album)
|
||||
* and return its MBID plus the linked artist + first release, or null if no
|
||||
* match (or the client is disabled / errors).
|
||||
*
|
||||
* `minScore` (default 85): MB returns results pre-sorted by a 0..100 search
|
||||
* score. Below this threshold the top hit is more often wrong than right, so
|
||||
* we return null rather than poisoning identity with a bad MBID. Pass a lower
|
||||
* value only for deliberately fuzzy matching.
|
||||
*/
|
||||
async lookupRecording(
|
||||
artist: string,
|
||||
title: string,
|
||||
album?: string,
|
||||
minScore = 85
|
||||
): Promise<RecordingMatch | null> {
|
||||
if (!this.enabled) return null;
|
||||
|
||||
// Build a fielded Lucene query; each value is quoted + escaped.
|
||||
const esc = (v: string): string => v.replace(/(["\\])/g, '\\$1');
|
||||
const terms = [`artist:"${esc(artist)}"`, `recording:"${esc(title)}"`];
|
||||
if (album && album.trim() !== '') terms.push(`release:"${esc(album)}"`);
|
||||
const query = terms.join(' AND ');
|
||||
|
||||
const path = `/recording?query=${encodeURIComponent(query)}&fmt=json&limit=5`;
|
||||
|
||||
try {
|
||||
const data = await this.get<MbRecordingSearchResponse>(path);
|
||||
const recordings = data.recordings ?? [];
|
||||
if (recordings.length === 0) return null;
|
||||
|
||||
// MB returns results pre-sorted by score; take the top entry.
|
||||
const best = recordings[0];
|
||||
const score = typeof best.score === 'number' ? best.score : 0;
|
||||
if (score < minScore) return null;
|
||||
|
||||
const credit = best['artist-credit']?.[0];
|
||||
const release = best.releases?.[0];
|
||||
|
||||
return {
|
||||
recordingMbid: best.id,
|
||||
title: best.title,
|
||||
artistMbid: credit?.artist?.id ?? null,
|
||||
artistName: credit?.artist?.name ?? credit?.name ?? null,
|
||||
releaseMbid: release?.id ?? null,
|
||||
releaseTitle: release?.title ?? null,
|
||||
score,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[MusicBrainz] lookupRecording failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the artist endpoint for the canonical identity matching a name.
|
||||
* Returns the best match above `minScore` (default 85), or null. This is the
|
||||
* correct endpoint for artist canonicalisation — unlike recording search it
|
||||
* returns the authoritative `name` + `sort-name` directly.
|
||||
*/
|
||||
async searchArtist(name: string, minScore = 85): Promise<ArtistMatch | null> {
|
||||
if (!this.enabled) return null;
|
||||
if (name.trim() === '') return null;
|
||||
|
||||
const esc = (v: string): string => v.replace(/(["\\])/g, '\\$1');
|
||||
const path = `/artist?query=${encodeURIComponent(`artist:"${esc(name)}"`)}&fmt=json&limit=5`;
|
||||
|
||||
try {
|
||||
const data = await this.get<MbArtistSearchResponse>(path);
|
||||
const artists = data.artists ?? [];
|
||||
if (artists.length === 0) return null;
|
||||
|
||||
const best = artists[0];
|
||||
const score = typeof best.score === 'number' ? best.score : 0;
|
||||
if (score < minScore) return null;
|
||||
|
||||
return {
|
||||
artistMbid: best.id,
|
||||
name: best.name ?? name,
|
||||
sortName: best['sort-name'] ?? null,
|
||||
score,
|
||||
disambiguation: best.disambiguation ?? null,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[MusicBrainz] searchArtist failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search the release-group endpoint for the canonical album identity matching
|
||||
* an artist + album title. Returns the best match above `minScore` (default
|
||||
* 80 — album titles vary more than artist names, so a slightly looser
|
||||
* threshold avoids missing legitimate matches), or null.
|
||||
*
|
||||
* A release-group is MB's canonical "album" entity: it groups all
|
||||
* editions/releases of the same album under one stable MBID, making it the
|
||||
* right value for `albums.mbid`.
|
||||
*/
|
||||
async searchReleaseGroup(
|
||||
artist: string,
|
||||
album: string,
|
||||
minScore = 80
|
||||
): Promise<ReleaseGroupMatch | null> {
|
||||
if (!this.enabled) return null;
|
||||
if (artist.trim() === '' || album.trim() === '') return null;
|
||||
|
||||
const esc = (v: string): string => v.replace(/(["\\])/g, '\\$1');
|
||||
const query = `artist:"${esc(artist)}" AND release:"${esc(album)}"`;
|
||||
const path = `/release-group?query=${encodeURIComponent(query)}&fmt=json&limit=5`;
|
||||
|
||||
try {
|
||||
const data = await this.get<MbReleaseGroupSearchResponse>(path);
|
||||
const groups = data['release-groups'] ?? [];
|
||||
if (groups.length === 0) return null;
|
||||
|
||||
const best = groups[0];
|
||||
const score = typeof best.score === 'number' ? best.score : 0;
|
||||
if (score < minScore) return null;
|
||||
|
||||
const credit = best['artist-credit']?.[0];
|
||||
const firstReleaseDate = best['first-release-date'] ?? null;
|
||||
const year = firstReleaseDate
|
||||
? parseInt(firstReleaseDate.slice(0, 4), 10)
|
||||
: null;
|
||||
|
||||
return {
|
||||
releaseGroupMbid: best.id,
|
||||
title: best.title ?? album,
|
||||
artistMbid: credit?.artist?.id ?? null,
|
||||
artistName: credit?.artist?.name ?? credit?.name ?? null,
|
||||
year: Number.isFinite(year) ? year : null,
|
||||
firstReleaseDate,
|
||||
score,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[MusicBrainz] searchReleaseGroup failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch an artist's tags + genres and return them normalised to 0..1 weights
|
||||
* (relative to the strongest tag). Returns [] if none / disabled / error.
|
||||
*/
|
||||
async getArtistTags(artistMbid: string): Promise<ArtistTag[]> {
|
||||
if (!this.enabled) return [];
|
||||
if (!artistMbid) return [];
|
||||
|
||||
const path = `/artist/${encodeURIComponent(artistMbid)}?inc=tags+genres&fmt=json`;
|
||||
|
||||
try {
|
||||
const data = await this.get<MbArtistResponse>(path);
|
||||
const raw = [...(data.genres ?? []), ...(data.tags ?? [])];
|
||||
|
||||
// Merge by name (genre + tag lists can overlap), keeping the max count.
|
||||
const byName = new Map<string, number>();
|
||||
for (const t of raw) {
|
||||
const name = t.name?.trim();
|
||||
if (!name) continue;
|
||||
const count = typeof t.count === 'number' ? t.count : 0;
|
||||
byName.set(name, Math.max(byName.get(name) ?? 0, count));
|
||||
}
|
||||
if (byName.size === 0) return [];
|
||||
|
||||
const maxCount = Math.max(...byName.values(), 1);
|
||||
return [...byName.entries()]
|
||||
.map(([name, count]) => ({ name, weight: count / maxCount }))
|
||||
.sort((a, b) => b.weight - a.weight);
|
||||
} catch (err) {
|
||||
console.warn('[MusicBrainz] getArtistTags failed:', (err as Error).message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a recording by its stable MBID with the full artist-credit array.
|
||||
* Unlike lookupRecording (search), this is a direct ID lookup with no search
|
||||
* step, making it the correct endpoint for spine-claim generation.
|
||||
*/
|
||||
async getRecording(recordingMbid: string): Promise<RecordingDetail | null> {
|
||||
if (!this.enabled) return null;
|
||||
if (!recordingMbid) return null;
|
||||
|
||||
const path = `/recording/${encodeURIComponent(recordingMbid)}?inc=artist-credits&fmt=json`;
|
||||
|
||||
try {
|
||||
const data = await this.get<MbRecordingDetail>(path);
|
||||
if (!data || !data.id) return null;
|
||||
|
||||
const credit = (data['artist-credit'] ?? [])
|
||||
.map(c => ({
|
||||
artistMbid: c.artist?.id ?? '',
|
||||
artistName: c.artist?.name ?? c.name ?? '',
|
||||
creditName: c.name ?? c.artist?.name ?? '',
|
||||
joinphrase: c.joinphrase,
|
||||
}))
|
||||
.filter(c => c.artistMbid !== '');
|
||||
|
||||
return {
|
||||
recordingMbid: data.id,
|
||||
title: data.title,
|
||||
artistCredit: credit,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[MusicBrainz] getRecording failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a release-group by its stable MBID with the full artist-credit
|
||||
* array (`inc=artists`). Direct ID lookup — the analogous endpoint to
|
||||
* getRecording() for album spine-claim generation.
|
||||
*/
|
||||
async getReleaseGroup(releaseGroupMbid: string): Promise<ReleaseGroupDetail | null> {
|
||||
if (!this.enabled) return null;
|
||||
if (!releaseGroupMbid) return null;
|
||||
|
||||
const path = `/release-group/${encodeURIComponent(releaseGroupMbid)}?inc=artists&fmt=json`;
|
||||
|
||||
try {
|
||||
const data = await this.get<MbReleaseGroupDetail>(path);
|
||||
if (!data || !data.id) return null;
|
||||
|
||||
const credit = (data['artist-credit'] ?? [])
|
||||
.map(c => ({
|
||||
artistMbid: c.artist?.id ?? '',
|
||||
artistName: c.artist?.name ?? c.name ?? '',
|
||||
creditName: c.name ?? c.artist?.name ?? '',
|
||||
joinphrase: c.joinphrase,
|
||||
}))
|
||||
.filter(c => c.artistMbid !== '');
|
||||
|
||||
return {
|
||||
releaseGroupMbid: data.id,
|
||||
title: data.title ?? '',
|
||||
artistCredit: credit,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[MusicBrainz] getReleaseGroup failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch an artist's artist-relations (`inc=artist-rels`) and return the
|
||||
* relation list normalised. Direct ID lookup — the source for `member_of`
|
||||
* and `alias_of` claim generation.
|
||||
*
|
||||
* The MB web service exposes relations both as a modern top-level `relations`
|
||||
* array and the legacy `relation-list` groupings keyed by `target-type`; we
|
||||
* read both defensively. Only entries with a target artist MBID are kept.
|
||||
*/
|
||||
async getArtistRelations(artistMbid: string): Promise<ArtistRelation[]> {
|
||||
if (!this.enabled) return [];
|
||||
if (!artistMbid) return [];
|
||||
|
||||
const path = `/artist/${encodeURIComponent(artistMbid)}?inc=artist-rels&fmt=json`;
|
||||
|
||||
try {
|
||||
const data = await this.get<MbArtistRelationsResponse>(path);
|
||||
if (!data) return [];
|
||||
|
||||
const raw: MbRelation[] = [];
|
||||
if (Array.isArray(data.relations)) {
|
||||
raw.push(...data.relations);
|
||||
}
|
||||
for (const list of data['relation-list'] ?? []) {
|
||||
if (list['target-type'] === 'artist' && Array.isArray(list.relations)) {
|
||||
raw.push(...list.relations);
|
||||
}
|
||||
}
|
||||
|
||||
return raw
|
||||
.map(r => ({
|
||||
type: r.type ?? '',
|
||||
direction: r.direction ?? 'forward',
|
||||
targetMbid: r.artist?.id ?? r.target ?? '',
|
||||
targetName: r.artist?.name ?? '',
|
||||
}))
|
||||
.filter(r => r.targetMbid !== '');
|
||||
} catch (err) {
|
||||
console.warn('[MusicBrainz] getArtistRelations failed:', (err as Error).message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
// TheAudioDB client for artist images.
|
||||
// API: https://www.theaudiodb.com/api_guide.php
|
||||
// Free tier: 2 requests/second, no key required for basic use.
|
||||
// With API key: higher limits.
|
||||
|
||||
import { requestJson } from './http.js';
|
||||
|
||||
interface AudioDbArtist {
|
||||
idArtist: string;
|
||||
strArtist: string;
|
||||
strArtistThumb?: string;
|
||||
strArtistBanner?: string;
|
||||
strArtistFanart?: string;
|
||||
strArtistLogo?: string;
|
||||
strArtistClearart?: string;
|
||||
strArtistWideThumb?: string;
|
||||
}
|
||||
|
||||
interface AudioDbResponse {
|
||||
artists?: AudioDbArtist[];
|
||||
}
|
||||
|
||||
export class TheAudioDbClient {
|
||||
private readonly baseUrl = 'https://www.theaudiodb.com/api/v1/json';
|
||||
private readonly apiKey: string;
|
||||
private readonly userAgent = 'muzick/0.1';
|
||||
|
||||
constructor(apiKey?: string) {
|
||||
// Free tier uses '1' as key, paid tiers get custom keys
|
||||
this.apiKey = apiKey || process.env.THEAUDIO_DB_API_KEY || '1';
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for artist by name.
|
||||
*/
|
||||
async searchArtist(name: string): Promise<AudioDbArtist | null> {
|
||||
const url = `${this.baseUrl}/${this.apiKey}/search.php?s=${encodeURIComponent(name)}`;
|
||||
|
||||
try {
|
||||
const data = await requestJson<AudioDbResponse>(url, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: 500, // 2 req/sec limit
|
||||
});
|
||||
|
||||
return data.artists?.[0] ?? null;
|
||||
} catch (err) {
|
||||
console.warn('[TheAudioDB] searchArtist failed:', (err as Error).message);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get artist by MusicBrainz ID.
|
||||
*/
|
||||
async getArtistByMbid(mbid: string): Promise<AudioDbArtist | null> {
|
||||
const url = `${this.baseUrl}/${this.apiKey}/artist-mb.php?i=${encodeURIComponent(mbid)}`;
|
||||
|
||||
try {
|
||||
const data = await requestJson<AudioDbResponse>(url, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: 500,
|
||||
});
|
||||
|
||||
return data.artists?.[0] ?? null;
|
||||
} catch (err) {
|
||||
console.warn('[TheAudioDB] getArtistByMbid failed:', (err as Error).message);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the best available artist image.
|
||||
* Priority: thumb > fanart > banner > logo > clearart > wide thumb
|
||||
*/
|
||||
async getBestArtistImage(mbid: string): Promise<string | null> {
|
||||
// Try by MBID first (most accurate)
|
||||
let artist = await this.getArtistByMbid(mbid);
|
||||
|
||||
// Fallback: search by name would need the name, which we don't have here
|
||||
// Caller should handle name-based fallback
|
||||
|
||||
if (!artist) return null;
|
||||
|
||||
return (
|
||||
artist.strArtistThumb ??
|
||||
artist.strArtistFanart ??
|
||||
artist.strArtistBanner ??
|
||||
artist.strArtistLogo ??
|
||||
artist.strArtistClearart ??
|
||||
artist.strArtistWideThumb ??
|
||||
null
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
// Wikidata client for artist images.
|
||||
// Flow: MusicBrainz artist MBID → Wikidata ID (via MB) → Image (P18 property).
|
||||
//
|
||||
// Wikidata API: https://www.wikidata.org/w/api.php
|
||||
// Property P18 = "image" (the main image for an entity).
|
||||
|
||||
import { requestJson } from './http.js';
|
||||
|
||||
interface WikidataEntity {
|
||||
id: string;
|
||||
labels?: Record<string, { value: string }>;
|
||||
claims?: Record<string, Array<{
|
||||
mainsnak?: {
|
||||
datavalue?: { value: string };
|
||||
};
|
||||
}>>;
|
||||
}
|
||||
|
||||
interface WikidataSearchResponse {
|
||||
search?: Array<{
|
||||
id: string;
|
||||
label: string;
|
||||
description: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface WikidataEntitiesResponse {
|
||||
entities?: Record<string, WikidataEntity>;
|
||||
}
|
||||
|
||||
export class WikidataClient {
|
||||
private readonly baseUrl = 'https://www.wikidata.org/w/api.php';
|
||||
private readonly userAgent = 'muzick/0.1 (https://github.com/user/muzick)';
|
||||
|
||||
/**
|
||||
* Get Wikidata ID for a MusicBrainz artist MBID.
|
||||
* Queries Wikidata for entities with the MusicBrainz artist ID (P434).
|
||||
*/
|
||||
async getWikidataIdFromMbid(mbid: string): Promise<string | null> {
|
||||
// Search Wikidata for entities with this MBID (P434 = MusicBrainz artist ID)
|
||||
const url = `${this.baseUrl}?action=wbsearchentities&search=${encodeURIComponent(mbid)}&language=en&format=json&type=item&props=claims&sitefilter=musicbrainz`;
|
||||
|
||||
try {
|
||||
const data = await requestJson<WikidataSearchResponse>(url, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: 500, // Wikidata allows higher rate
|
||||
});
|
||||
|
||||
const results = data.search ?? [];
|
||||
if (results.length > 0) {
|
||||
return results[0].id;
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[Wikidata] getWikidataIdFromMbid failed:', (err as Error).message);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search Wikidata for an artist by name.
|
||||
* Less reliable than MBID → Wikidata, but works as fallback.
|
||||
*/
|
||||
async searchArtistByName(name: string): Promise<string | null> {
|
||||
const url = `${this.baseUrl}?action=wbsearchentities&search=${encodeURIComponent(name)}&language=en&format=json&type=item&limit=5`;
|
||||
|
||||
try {
|
||||
const data = await requestJson<WikidataSearchResponse>(url, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: 500,
|
||||
});
|
||||
|
||||
const results = data.search ?? [];
|
||||
// Filter for music artists (instance of: human, musical artist, band, etc.)
|
||||
for (const result of results) {
|
||||
if (this.looksLikeMusicArtist(result)) {
|
||||
return result.id;
|
||||
}
|
||||
}
|
||||
return results[0]?.id ?? null;
|
||||
} catch (err) {
|
||||
console.warn('[Wikidata] searchArtistByName failed:', (err as Error).message);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private looksLikeMusicArtist(result: { label: string; description: string }): boolean {
|
||||
const desc = result.description.toLowerCase();
|
||||
const musicKeywords = ['singer', 'musician', 'band', 'artist', 'rapper', 'producer', 'composer', 'dj', 'group'];
|
||||
return musicKeywords.some(k => desc.includes(k));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the main image (P18) for a Wikidata entity.
|
||||
* Returns the image filename (e.g., "Artist_Name.jpg") which can be used
|
||||
* with Wikimedia Commons URL: https://commons.wikimedia.org/wiki/File:...
|
||||
*/
|
||||
async getImage(wikidataId: string): Promise<string | null> {
|
||||
const url = `${this.baseUrl}?action=wbgetentities&ids=${encodeURIComponent(wikidataId)}&props=claims&format=json`;
|
||||
|
||||
try {
|
||||
const data = await requestJson<WikidataEntitiesResponse>(url, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: 500,
|
||||
});
|
||||
|
||||
const entity = data.entities?.[wikidataId];
|
||||
if (!entity?.claims?.P18) return null;
|
||||
|
||||
const imageClaim = entity.claims.P18[0];
|
||||
const imageName = imageClaim?.mainsnak?.datavalue?.value;
|
||||
if (!imageName) return null;
|
||||
|
||||
// Return Wikimedia Commons URL
|
||||
// Format: https://commons.wikimedia.org/wiki/Special:FilePath/filename
|
||||
const encoded = encodeURIComponent(imageName.replace(/ /g, '_'));
|
||||
return `https://commons.wikimedia.org/wiki/Special:FilePath/${encoded}`;
|
||||
} catch (err) {
|
||||
console.warn('[Wikidata] getImage failed:', (err as Error).message);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full pipeline: MBID → Wikidata ID → Image URL.
|
||||
*/
|
||||
async getArtistImageFromMbid(mbid: string): Promise<string | null> {
|
||||
const wikidataId = await this.getWikidataIdFromMbid(mbid);
|
||||
if (!wikidataId) return null;
|
||||
return this.getImage(wikidataId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Full pipeline fallback: name → Wikidata ID → Image URL.
|
||||
*/
|
||||
async getArtistImageFromName(name: string): Promise<string | null> {
|
||||
const wikidataId = await this.searchArtistByName(name);
|
||||
if (!wikidataId) return null;
|
||||
return this.getImage(wikidataId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// Wikimedia Commons client for artist images.
|
||||
// Uses the MediaWiki API to search for and fetch artist images.
|
||||
// No API key required; polite rate limit (~250ms) enforced via http base.
|
||||
|
||||
import { integrationsConfig, WikimediaConfig } from './config.js';
|
||||
import { buildUserAgent, requestJson, HttpError } from './http.js';
|
||||
|
||||
interface WikimediaSearchResult {
|
||||
title?: string;
|
||||
pageid?: number;
|
||||
thumbnail?: { source?: string; width?: number; height?: number };
|
||||
}
|
||||
|
||||
interface WikimediaSearchResponse {
|
||||
query?: {
|
||||
search?: WikimediaSearchResult[];
|
||||
};
|
||||
}
|
||||
|
||||
interface WikimediaImageInfoResponse {
|
||||
query?: {
|
||||
pages?: {
|
||||
[pageId: string]: {
|
||||
imageinfo?: { url?: string; thumburl?: string; width?: number; height?: number }[];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export class WikimediaClient {
|
||||
private readonly cfg: WikimediaConfig;
|
||||
private readonly userAgent: string;
|
||||
|
||||
constructor(cfg: WikimediaConfig = integrationsConfig.wikimedia) {
|
||||
this.cfg = cfg;
|
||||
this.userAgent = buildUserAgent(integrationsConfig.musicbrainz.contact);
|
||||
}
|
||||
|
||||
private async get<T>(path: string): Promise<T> {
|
||||
return requestJson<T>(`${this.cfg.baseUrl}${path}`, {
|
||||
userAgent: this.userAgent,
|
||||
minIntervalMs: this.cfg.minIntervalMs,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Search Wikimedia Commons for an artist image and return the best available
|
||||
* thumbnail URL (preferring larger sizes), or null on miss/error.
|
||||
*/
|
||||
async getArtistImageUrl(artistName: string): Promise<string | null> {
|
||||
if (artistName.trim() === '') return null;
|
||||
|
||||
try {
|
||||
// Search for the artist on Wikimedia Commons
|
||||
const searchQs = new URLSearchParams({
|
||||
action: 'query',
|
||||
list: 'search',
|
||||
srsearch: `${artistName} artist`,
|
||||
srnamespace: '6', // File namespace
|
||||
srlimit: '5',
|
||||
format: 'json',
|
||||
});
|
||||
const searchPath = `/w/api.php?${searchQs.toString()}`;
|
||||
const searchData = await this.get<WikimediaSearchResponse>(searchPath);
|
||||
|
||||
const results = searchData.query?.search ?? [];
|
||||
if (results.length === 0) return null;
|
||||
|
||||
// Get image info for the first few results to find the best thumbnail
|
||||
const pageIds = results.slice(0, 3).map((r) => r.pageid).filter((id): id is number => id !== undefined);
|
||||
if (pageIds.length === 0) return null;
|
||||
|
||||
const imageQs = new URLSearchParams({
|
||||
action: 'query',
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|thumburl|width|height',
|
||||
iiurlwidth: '300',
|
||||
iiurlheight: '300',
|
||||
pageids: pageIds.join('|'),
|
||||
format: 'json',
|
||||
});
|
||||
const imagePath = `/w/api.php?${imageQs.toString()}`;
|
||||
const imageData = await this.get<WikimediaImageInfoResponse>(imagePath);
|
||||
|
||||
const pages = imageData.query?.pages ?? {};
|
||||
for (const pageId of pageIds) {
|
||||
const page = pages[pageId.toString()];
|
||||
if (page?.imageinfo?.[0]?.thumburl) {
|
||||
return page.imageinfo[0].thumburl;
|
||||
}
|
||||
if (page?.imageinfo?.[0]?.url) {
|
||||
return page.imageinfo[0].url;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
} catch (err) {
|
||||
if (err instanceof HttpError && err.status === 404) return null;
|
||||
console.warn('[Wikimedia] getArtistImageUrl failed:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
import fs from 'fs/promises';
|
||||
import { Client as PgClient } from 'pg';
|
||||
import { Queue } from 'bullmq';
|
||||
import { ScannerService } from './scanner.service.js';
|
||||
|
||||
const MUSIC_DIR = process.env.MUSIC_DIR || '/mnt/hdd1/media/Music';
|
||||
|
||||
// Detection regexes for the known (now-fixed) corruption bug. These are the
|
||||
// exact patterns used by the original one-off repair script:
|
||||
// title : trailing ' (Enriched)' (possibly stacked)
|
||||
// artist : trailing 'Unknown Artist' (concatenated, no separator, possibly stacked)
|
||||
// Anchored to end-of-string; '+' handles stacked corruption.
|
||||
const TITLE_CORRUPT_RE = `( \\(Enriched\\))+\\s*$`;
|
||||
const ARTIST_CORRUPT_RE = `\\s*(Unknown Artist)+\\s*$`;
|
||||
|
||||
interface CorruptRow {
|
||||
id: string;
|
||||
path: string;
|
||||
title: string;
|
||||
artist: string;
|
||||
}
|
||||
|
||||
interface MissingRow {
|
||||
id: string;
|
||||
path: string;
|
||||
title: string;
|
||||
artist: string;
|
||||
}
|
||||
|
||||
export interface SweepSummary {
|
||||
detected: number;
|
||||
fixed: number;
|
||||
needsReview: number;
|
||||
}
|
||||
|
||||
export class IntegrityService {
|
||||
private scanner: ScannerService;
|
||||
|
||||
constructor(private pgClient: PgClient, private queue: Queue) {
|
||||
this.scanner = new ScannerService(pgClient, queue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Self-provision the issues table. Idempotent (CREATE TABLE IF NOT EXISTS),
|
||||
* mirrors backend/src/db/schema.sql so the worker runs on older DBs.
|
||||
*/
|
||||
async ensureSchema(): Promise<void> {
|
||||
await this.pgClient.query(
|
||||
`CREATE TABLE IF NOT EXISTS track_integrity_issues (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
|
||||
issue_type TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'OPEN',
|
||||
details TEXT,
|
||||
detected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
resolved_at TIMESTAMP,
|
||||
UNIQUE(track_id, issue_type)
|
||||
)`
|
||||
);
|
||||
}
|
||||
|
||||
/** Tracks whose title or artist still carry the corruption markers. */
|
||||
async detectCorruptedMetadata(): Promise<CorruptRow[]> {
|
||||
const res = await this.pgClient.query<CorruptRow>(
|
||||
`SELECT id, path, title, artist
|
||||
FROM tracks
|
||||
WHERE title ~ $1 OR artist ~ $2`,
|
||||
[TITLE_CORRUPT_RE, ARTIST_CORRUPT_RE]
|
||||
);
|
||||
return res.rows;
|
||||
}
|
||||
|
||||
/** Tracks (excluding ones already flagged DELETED) whose file is gone from disk. */
|
||||
async detectMissingFiles(limit = 5000): Promise<MissingRow[]> {
|
||||
const res = await this.pgClient.query<MissingRow>(
|
||||
`SELECT id, path, title, artist
|
||||
FROM tracks
|
||||
WHERE state <> 'DELETED'
|
||||
LIMIT $1`,
|
||||
[limit]
|
||||
);
|
||||
|
||||
const missing: MissingRow[] = [];
|
||||
for (const row of res.rows) {
|
||||
try {
|
||||
await fs.access(row.path);
|
||||
} catch {
|
||||
missing.push(row);
|
||||
}
|
||||
}
|
||||
return missing;
|
||||
}
|
||||
|
||||
/** Upsert an issue row keyed on (track_id, issue_type), refreshing on re-detection. */
|
||||
private async upsertIssue(
|
||||
trackId: string,
|
||||
issueType: 'CORRUPT_METADATA' | 'MISSING_FILE',
|
||||
status: 'OPEN' | 'FIXED' | 'NEEDS_REVIEW',
|
||||
details: string | null
|
||||
): Promise<void> {
|
||||
await this.pgClient.query(
|
||||
`INSERT INTO track_integrity_issues (track_id, issue_type, status, details, detected_at, resolved_at)
|
||||
VALUES ($1, $2, $3, $4, NOW(), CASE WHEN $3 = 'FIXED' THEN NOW() ELSE NULL END)
|
||||
ON CONFLICT (track_id, issue_type) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
details = EXCLUDED.details,
|
||||
detected_at = NOW(),
|
||||
resolved_at = CASE WHEN EXCLUDED.status = 'FIXED' THEN NOW() ELSE NULL END`,
|
||||
[trackId, issueType, status, details]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The defensive SQL strip (PASS 2 from the original repair script). Anchored,
|
||||
* idempotent, only touches rows that actually match — a no-op on clean data.
|
||||
*/
|
||||
private async stripCorruptionMarkers(): Promise<void> {
|
||||
await this.pgClient.query(
|
||||
`UPDATE tracks
|
||||
SET title = btrim(regexp_replace(title, '( \\(Enriched\\))+\\s*$', ''))
|
||||
WHERE title ~ '( \\(Enriched\\))+\\s*$' OR title <> btrim(title)`
|
||||
);
|
||||
await this.pgClient.query(
|
||||
`UPDATE tracks
|
||||
SET artist = btrim(regexp_replace(artist, '\\s*(Unknown Artist)+\\s*$', ''))
|
||||
WHERE artist ~ '\\s*(Unknown Artist)+\\s*$' OR artist <> btrim(artist)`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrate a full integrity sweep: detect, attempt auto-repair, then
|
||||
* reconcile issue rows. Safe to run repeatedly; a clean DB is a no-op and
|
||||
* produces no spurious issue rows.
|
||||
*/
|
||||
async runSweep(): Promise<SweepSummary> {
|
||||
await this.ensureSchema();
|
||||
|
||||
const summary: SweepSummary = { detected: 0, fixed: 0, needsReview: 0 };
|
||||
|
||||
// --- Corrupted metadata --------------------------------------------------
|
||||
const corrupt = await this.detectCorruptedMetadata();
|
||||
summary.detected = corrupt.length;
|
||||
|
||||
// Record each corrupt row as an OPEN issue (only real matches get rows).
|
||||
for (const row of corrupt) {
|
||||
const details = `corrupt title="${row.title}" artist="${row.artist}"`;
|
||||
await this.upsertIssue(row.id, 'CORRUPT_METADATA', 'OPEN', details);
|
||||
}
|
||||
|
||||
if (corrupt.length > 0) {
|
||||
// PASS 1: authoritative re-scan from disk overwrites title/artist by path.
|
||||
console.log(`[Integrity] PASS 1: re-scanning library at ${MUSIC_DIR}`);
|
||||
try {
|
||||
await this.scanner.scanDirectory(MUSIC_DIR);
|
||||
} catch (err) {
|
||||
console.error('[Integrity] PASS 1 rescan failed (continuing to PASS 2):', err);
|
||||
}
|
||||
// PASS 2: defensive SQL strip for rows the scanner couldn't fix.
|
||||
console.log('[Integrity] PASS 2: defensive SQL strip of corruption markers');
|
||||
await this.stripCorruptionMarkers();
|
||||
|
||||
// Reconcile: re-check each previously-corrupt row individually.
|
||||
const stillCorrupt = await this.pgClient.query<{ id: string }>(
|
||||
`SELECT id FROM tracks
|
||||
WHERE id = ANY($1::uuid[]) AND (title ~ $2 OR artist ~ $3)`,
|
||||
[corrupt.map((r) => r.id), TITLE_CORRUPT_RE, ARTIST_CORRUPT_RE]
|
||||
);
|
||||
const stillCorruptIds = new Set(stillCorrupt.rows.map((r) => r.id));
|
||||
|
||||
for (const row of corrupt) {
|
||||
if (stillCorruptIds.has(row.id)) {
|
||||
// Could not auto-fix (e.g. file missing) -> flag for manual review.
|
||||
await this.upsertIssue(
|
||||
row.id,
|
||||
'CORRUPT_METADATA',
|
||||
'NEEDS_REVIEW',
|
||||
`could not auto-repair; corrupt title="${row.title}" artist="${row.artist}"`
|
||||
);
|
||||
summary.needsReview++;
|
||||
} else {
|
||||
await this.upsertIssue(row.id, 'CORRUPT_METADATA', 'FIXED', null);
|
||||
summary.fixed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Missing files -------------------------------------------------------
|
||||
const missing = await this.detectMissingFiles();
|
||||
for (const row of missing) {
|
||||
summary.detected++;
|
||||
summary.needsReview++;
|
||||
await this.upsertIssue(
|
||||
row.id,
|
||||
'MISSING_FILE',
|
||||
'NEEDS_REVIEW',
|
||||
`file not found on disk: ${row.path}`
|
||||
);
|
||||
// No-ghost rule: mark the track MISSING rather than deleting/keeping it as
|
||||
// playable. Guarded UPDATE keeps this idempotent (only flips non-MISSING rows).
|
||||
await this.pgClient.query(
|
||||
`UPDATE tracks SET state = 'MISSING' WHERE id = $1 AND state <> 'MISSING'`,
|
||||
[row.id]
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[Integrity] Sweep summary: detected=${summary.detected} fixed=${summary.fixed} needsReview=${summary.needsReview}`
|
||||
);
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
import { Client as PgClient } from 'pg';
|
||||
import { MusicBrainzClient } from './integrations/musicbrainz.client.js';
|
||||
import { normalizeForMatching } from './utils/fuzzy-match.js';
|
||||
|
||||
function generateSortName(name: string): string {
|
||||
const match = name.match(/^(The|A|An)\s+(.+)$/i);
|
||||
if (match) return `${match[2]}, ${match[1]}`;
|
||||
return name;
|
||||
}
|
||||
|
||||
export class MbSpineWriter {
|
||||
constructor(private pgClient: PgClient) {}
|
||||
|
||||
/**
|
||||
* Fetch full artist-credit for a recording MBID and write claims.
|
||||
* - First credit → "credited_main_on"
|
||||
* - Additional credits → "featured_on"
|
||||
* - Updates tracks.recording_mbid
|
||||
* - Resolves or stubs credited artists by MBID
|
||||
*/
|
||||
async writeRecordingClaims(
|
||||
recordingMbid: string,
|
||||
trackId: string,
|
||||
mbClient: MusicBrainzClient
|
||||
): Promise<number> {
|
||||
const recording = await mbClient.getRecording(recordingMbid);
|
||||
if (!recording || recording.artistCredit.length === 0) return 0;
|
||||
|
||||
await this.pgClient.query(
|
||||
`UPDATE tracks SET recording_mbid = $1 WHERE id = $2 AND recording_mbid IS DISTINCT FROM $1`,
|
||||
[recordingMbid, trackId]
|
||||
);
|
||||
|
||||
let claimsWritten = 0;
|
||||
|
||||
for (let i = 0; i < recording.artistCredit.length; i++) {
|
||||
const credit = recording.artistCredit[i];
|
||||
|
||||
const artistId = await this.resolveArtist(
|
||||
credit.artistMbid,
|
||||
credit.artistName,
|
||||
credit.creditName,
|
||||
);
|
||||
if (!artistId) {
|
||||
console.warn(
|
||||
`[MbSpineWriter] Could not resolve artist for credit ${i} on recording ${recordingMbid}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const predicate = i === 0 ? 'credited_main_on' : 'featured_on';
|
||||
const raw = credit.joinphrase ? JSON.stringify({ joinphrase: credit.joinphrase }) : null;
|
||||
|
||||
await this.pgClient.query(
|
||||
`INSERT INTO claims
|
||||
(subject_type, subject_id, predicate, object_type, object_id, source, confidence, raw)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (subject_type, subject_id, predicate, object_type, object_id, source, user_id)
|
||||
DO NOTHING`,
|
||||
['track', trackId, predicate, 'artist', artistId, 'mb', 1.0, raw]
|
||||
);
|
||||
claimsWritten++;
|
||||
}
|
||||
|
||||
return claimsWritten;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch full artist-credit for a release-group MBID and write claims.
|
||||
* - First credit → "credited_main_on_album"
|
||||
* - Additional credits → "featured_on_album"
|
||||
* - Resolves or stubs credited artists by MBID
|
||||
*/
|
||||
async writeAlbumClaims(
|
||||
releaseGroupMbid: string,
|
||||
albumId: string,
|
||||
mbClient: MusicBrainzClient
|
||||
): Promise<number> {
|
||||
const releaseGroup = await mbClient.getReleaseGroup(releaseGroupMbid);
|
||||
if (!releaseGroup || releaseGroup.artistCredit.length === 0) return 0;
|
||||
|
||||
let claimsWritten = 0;
|
||||
|
||||
for (let i = 0; i < releaseGroup.artistCredit.length; i++) {
|
||||
const credit = releaseGroup.artistCredit[i];
|
||||
|
||||
const artistId = await this.resolveArtist(
|
||||
credit.artistMbid,
|
||||
credit.artistName,
|
||||
credit.creditName,
|
||||
);
|
||||
if (!artistId) {
|
||||
console.warn(
|
||||
`[MbSpineWriter] Could not resolve artist for credit ${i} on release-group ${releaseGroupMbid}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const predicate = i === 0 ? 'credited_main_on_album' : 'featured_on_album';
|
||||
const raw = credit.joinphrase ? JSON.stringify({ joinphrase: credit.joinphrase }) : null;
|
||||
|
||||
await this.pgClient.query(
|
||||
`INSERT INTO claims
|
||||
(subject_type, subject_id, predicate, object_type, object_id, source, confidence, raw)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (subject_type, subject_id, predicate, object_type, object_id, source, user_id)
|
||||
DO NOTHING`,
|
||||
['artist', artistId, predicate, 'album', albumId, 'mb', 1.0, raw]
|
||||
);
|
||||
claimsWritten++;
|
||||
}
|
||||
|
||||
return claimsWritten;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch an artist's MB artist-relations and write structural claims:
|
||||
* - "member of" → "member_of" (subject = this artist, object = target group)
|
||||
* - "is alias of" / "is performance name of" / "is legal name of" → "alias_of"
|
||||
*
|
||||
* Direction handling: MB marks the relationship as forward when it points from
|
||||
* the queried artist to the target. For "is performance name of" (forward)
|
||||
* the queried artist is the alias of the target, so subject = queried artist,
|
||||
* object = target. For backward relations we flip subject/object.
|
||||
*
|
||||
* Collab/vocal/instrument ARs are artist-artist contributions rather than
|
||||
* identity/structure, and the spec maps them to track-level featured_on; we
|
||||
* skip them here to avoid over-claiming artist-artist edges.
|
||||
*/
|
||||
async writeArtistRelationClaims(
|
||||
artistMbid: string,
|
||||
artistId: string,
|
||||
mbClient: MusicBrainzClient
|
||||
): Promise<number> {
|
||||
const relations = await mbClient.getArtistRelations(artistMbid);
|
||||
if (relations.length === 0) return 0;
|
||||
|
||||
const memberOfTypes = new Set(['member of', 'founder of', 'founder']);
|
||||
const aliasOfTypes = new Set(['is alias of', 'is performance name of', 'is legal name of']);
|
||||
|
||||
let claimsWritten = 0;
|
||||
|
||||
for (const rel of relations) {
|
||||
const isMemberOf = [...memberOfTypes].some(t => rel.type.toLowerCase() === t);
|
||||
const aliasMatch = [...aliasOfTypes].find(t => rel.type.toLowerCase() === t);
|
||||
if (!isMemberOf && !aliasMatch) continue;
|
||||
|
||||
const targetArtistId = await this.resolveArtist(
|
||||
rel.targetMbid,
|
||||
rel.targetName,
|
||||
rel.targetName,
|
||||
);
|
||||
if (!targetArtistId) {
|
||||
console.warn(
|
||||
`[MbSpineWriter] Could not resolve target artist for relation "${rel.type}" on artist ${artistMbid}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Forward direction: subject = this artist, object = target.
|
||||
// Backward direction: the relation is phrased from target → subject, so
|
||||
// we flip subject/object to preserve the predicate's intended direction.
|
||||
let subjectId = artistId;
|
||||
let objectId = targetArtistId;
|
||||
if (rel.direction === 'backward') {
|
||||
subjectId = targetArtistId;
|
||||
objectId = artistId;
|
||||
}
|
||||
|
||||
const predicate = aliasMatch ? 'alias_of' : 'member_of';
|
||||
const raw = JSON.stringify({
|
||||
mb_relation_type: rel.type,
|
||||
direction: rel.direction,
|
||||
});
|
||||
|
||||
await this.pgClient.query(
|
||||
`INSERT INTO claims
|
||||
(subject_type, subject_id, predicate, object_type, object_id, source, confidence, raw)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (subject_type, subject_id, predicate, object_type, object_id, source, user_id)
|
||||
DO NOTHING`,
|
||||
['artist', subjectId, predicate, 'artist', objectId, 'mb', 1.0, raw]
|
||||
);
|
||||
claimsWritten++;
|
||||
}
|
||||
|
||||
return claimsWritten;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve an MB artist to a local artists.id. Priority:
|
||||
* 1. Match by mbid (UUID column)
|
||||
* 2. Match by normalized credit name
|
||||
* 3. Create a stub row
|
||||
*/
|
||||
private async resolveArtist(
|
||||
mbid: string,
|
||||
artistName: string,
|
||||
creditName: string,
|
||||
): Promise<string | null> {
|
||||
const existing = await this.pgClient.query<{ id: string }>(
|
||||
`SELECT id FROM artists WHERE mbid = $1`,
|
||||
[mbid]
|
||||
);
|
||||
if (existing.rows.length > 0) return existing.rows[0].id;
|
||||
|
||||
const normalized = normalizeForMatching(creditName);
|
||||
const nameMatch = await this.pgClient.query<{ id: string }>(
|
||||
`SELECT id FROM artists WHERE normalized_name = $1`,
|
||||
[normalized]
|
||||
);
|
||||
if (nameMatch.rows.length > 0) {
|
||||
await this.pgClient.query(
|
||||
`UPDATE artists SET mbid = $1, updated_at = CURRENT_TIMESTAMP WHERE id = $2`,
|
||||
[mbid, nameMatch.rows[0].id]
|
||||
);
|
||||
return nameMatch.rows[0].id;
|
||||
}
|
||||
|
||||
const sortName = generateSortName(artistName);
|
||||
const result = await this.pgClient.query<{ id: string }>(
|
||||
`INSERT INTO artists (name, canonical_name, sort_name, mbid)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (mbid) WHERE mbid IS NOT NULL
|
||||
DO UPDATE SET name = EXCLUDED.name, updated_at = CURRENT_TIMESTAMP
|
||||
RETURNING id`,
|
||||
[creditName, artistName, sortName, mbid]
|
||||
);
|
||||
return result.rows[0]?.id ?? null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { ConnectionOptions, Queue } from 'bullmq';
|
||||
|
||||
export const connection: ConnectionOptions = {
|
||||
url: process.env.REDIS_URL || 'redis://localhost:6379',
|
||||
};
|
||||
|
||||
export const QUEUE_NAME = 'muzick-queue';
|
||||
|
||||
// Shared Queue instance used to register repeatable/scheduled jobs (e.g. the
|
||||
// periodic integrity sweep). Producers in other services can import this too.
|
||||
export const queue = new Queue(QUEUE_NAME, { connection });
|
||||
@@ -0,0 +1,293 @@
|
||||
import fs from 'fs/promises';
|
||||
import { createReadStream } from 'fs';
|
||||
import { createHash } from 'crypto';
|
||||
import path from 'path';
|
||||
import mm from 'music-metadata';
|
||||
import { Client as PgClient } from 'pg';
|
||||
import { Queue } from 'bullmq';
|
||||
import { MetadataRefreshJob, ArtistSimilarityJob, ArtistImageJob, AlbumCoverJob } from './types.js';
|
||||
import { splitArtistNames, parseArtists } from './utils/artist-names.js';
|
||||
|
||||
/**
|
||||
* Parse main + featured artists from music-metadata. Prefers the structured
|
||||
* `artists[]` array when the tag provides it (each entry already one artist),
|
||||
* otherwise falls back to splitting the single `artist` string. Feature markers
|
||||
* in the title ("Song (feat. X)") are folded in either way. First artist is the
|
||||
* main artist; the rest are featured. See ./utils/artist-names for the rules.
|
||||
*/
|
||||
function parseArtistsFromMetadata(common: any): { main: string; featured: string[] } {
|
||||
const rawTitle = common.title || '';
|
||||
|
||||
// Structured array present: still split each entry (tags sometimes put a whole
|
||||
// "A & B" collaboration in one slot) and fold in any title feature.
|
||||
if (common.artists && Array.isArray(common.artists) && common.artists.length > 0) {
|
||||
const fromArray = common.artists.flatMap((a: string) => splitArtistNames(a));
|
||||
const fromTitleParse = parseArtists('', rawTitle);
|
||||
const all: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const name of [...fromArray, ...fromTitleParse.featured]) {
|
||||
const n = name.trim();
|
||||
if (!n || seen.has(n.toLowerCase())) continue;
|
||||
seen.add(n.toLowerCase());
|
||||
all.push(n);
|
||||
}
|
||||
if (all.length > 0) {
|
||||
return { main: all[0], featured: all.slice(1) };
|
||||
}
|
||||
}
|
||||
|
||||
return parseArtists(common.artist || 'Unknown Artist', rawTitle);
|
||||
}
|
||||
|
||||
function hashFile(filePath: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash('md5');
|
||||
const stream = createReadStream(filePath);
|
||||
stream.on('data', (d) => hash.update(d));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the album artist from metadata tags. The `albumartist` tag is the
|
||||
* standard way to identify who "owns" the album as a whole — "Various Artists"
|
||||
* for compilations, "Soundtrack" for OSTs, or the primary artist for normal
|
||||
* albums. Without it, compilations where each track has a different artist
|
||||
* would create duplicate album rows (one per track artist).
|
||||
*
|
||||
* Falls back to the track's main artist when the tag is absent (single-artist
|
||||
* albums where the tag is redundant).
|
||||
*
|
||||
* Also splits the albumartist string via splitArtistNames so that "A & B" as
|
||||
* albumartist uses just the first-billed artist as the album's identity —
|
||||
* consistent with how normalize_artist treats track artists.
|
||||
*/
|
||||
function resolveAlbumArtist(common: any, fallback: string): string {
|
||||
// music-metadata exposes albumartist as either a string or an array.
|
||||
const raw = common.albumartist ?? common.albumArtist;
|
||||
if (!raw) return fallback;
|
||||
const str = Array.isArray(raw) ? raw[0] : String(raw);
|
||||
if (str.trim() === '') return fallback;
|
||||
const parts = splitArtistNames(str);
|
||||
return parts.length > 0 ? parts[0] : fallback;
|
||||
}
|
||||
|
||||
export class ScannerService {
|
||||
// Per-scan dedupe sets for per-artist / per-album jobs. Belt-and-braces
|
||||
// alongside the BullMQ jobId dedupe (jobId dedupes across overlapping scans /
|
||||
// pending jobs; these avoid even issuing the redundant add() within one scan).
|
||||
private enqueuedArtists = new Set<string>();
|
||||
private enqueuedAlbums = new Set<string>();
|
||||
|
||||
constructor(private pgClient: PgClient, private queue: Queue) {}
|
||||
|
||||
async scanDirectory(directory: string) {
|
||||
console.log(`[Scanner] Starting scan in: ${directory}`);
|
||||
this.enqueuedArtists.clear();
|
||||
this.enqueuedAlbums.clear();
|
||||
await this.walk(directory);
|
||||
console.log(`[Scanner] Scan completed.`);
|
||||
}
|
||||
|
||||
private async walk(dir: string) {
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await this.walk(fullPath);
|
||||
} else if (this.isMusicFile(entry.name)) {
|
||||
await this.processFile(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private isMusicFile(fileName: string): boolean {
|
||||
const extensions = ['.mp3', '.flac', '.m4a', '.wav', '.ogg'];
|
||||
return extensions.includes(path.extname(fileName).toLowerCase());
|
||||
}
|
||||
|
||||
private async processFile(filePath: string) {
|
||||
try {
|
||||
console.log(`[Scanner] Processing: ${filePath}`);
|
||||
const metadata = await mm.parseFile(filePath);
|
||||
const { common, format } = metadata;
|
||||
|
||||
// 1. Ensure Artist(s) exist.
|
||||
const trackTitle = common.title || path.basename(filePath);
|
||||
const { main: mainArtistRaw, featured: featuredArtistNames } = parseArtistsFromMetadata(common);
|
||||
|
||||
const { id: artistId, name: resolvedArtist } = await this.resolveOrCreateArtist(mainArtistRaw);
|
||||
|
||||
// Upsert featured artists into artists table and collect their ids.
|
||||
const featuredIds: string[] = [];
|
||||
for (const featName of featuredArtistNames) {
|
||||
const feat = await this.resolveOrCreateArtist(featName);
|
||||
featuredIds.push(feat.id);
|
||||
}
|
||||
|
||||
// 2. Ensure Album exists.
|
||||
// The album is keyed by (album_artist_id, title), NOT (track_artist_id,
|
||||
// title). This prevents duplicate album rows for compilations where
|
||||
// each track has a different artist but they all belong to one album.
|
||||
// We resolve the ALBUM artist from the `albumartist` tag when present
|
||||
// ("Various Artists", "Soundtrack", or the primary artist), falling
|
||||
// back to the track's main artist when the tag is absent.
|
||||
const albumTitle = common.album || 'Unknown Album';
|
||||
const albumArtistRaw = resolveAlbumArtist(common, mainArtistRaw);
|
||||
const albumArtistId = albumArtistRaw === mainArtistRaw
|
||||
? artistId
|
||||
: (await this.resolveOrCreateArtist(albumArtistRaw)).id;
|
||||
|
||||
const existingAlbum = await this.pgClient.query(
|
||||
'SELECT id FROM albums WHERE artist_id = $1 AND lower(title) = lower($2) LIMIT 1',
|
||||
[albumArtistId, albumTitle]
|
||||
);
|
||||
let albumId: string;
|
||||
if (existingAlbum.rows.length > 0) {
|
||||
albumId = String(existingAlbum.rows[0].id);
|
||||
} else {
|
||||
const albumRes = await this.pgClient.query(
|
||||
'INSERT INTO albums (artist_id, title) VALUES ($1, $2) ON CONFLICT (artist_id, title) DO UPDATE SET title = EXCLUDED.title RETURNING id',
|
||||
[albumArtistId, albumTitle]
|
||||
);
|
||||
albumId = String(albumRes.rows[0].id);
|
||||
}
|
||||
|
||||
// 3. Upsert Track
|
||||
const duration = format.duration || 0;
|
||||
const fileHash = await hashFile(filePath);
|
||||
|
||||
const trackRes = await this.pgClient.query(
|
||||
`INSERT INTO tracks (path, hash, title, artist, album_id, duration, state)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'LIBRARY')
|
||||
ON CONFLICT (path) DO UPDATE SET
|
||||
hash = EXCLUDED.hash,
|
||||
title = EXCLUDED.title,
|
||||
artist = EXCLUDED.artist,
|
||||
album_id = EXCLUDED.album_id,
|
||||
duration = EXCLUDED.duration,
|
||||
mtime = EXTRACT(EPOCH FROM NOW())
|
||||
RETURNING id
|
||||
`,
|
||||
[filePath, fileHash, trackTitle, resolvedArtist, albumId, duration]
|
||||
);
|
||||
const trackId = String(trackRes.rows[0].id);
|
||||
|
||||
// 4. Populate track_artists junction table.
|
||||
await this.pgClient.query(
|
||||
`INSERT INTO track_artists (track_id, artist_id, role)
|
||||
VALUES ($1, $2, 'main')
|
||||
ON CONFLICT (track_id, artist_id, role) DO NOTHING`,
|
||||
[trackId, artistId]
|
||||
);
|
||||
for (const featId of featuredIds) {
|
||||
await this.pgClient.query(
|
||||
`INSERT INTO track_artists (track_id, artist_id, role)
|
||||
VALUES ($1, $2, 'featured')
|
||||
ON CONFLICT (track_id, artist_id, role) DO NOTHING`,
|
||||
[trackId, featId]
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`[Scanner] Successfully processed: ${trackTitle}`);
|
||||
|
||||
// Trigger external-API enrichment for this track + artist + album.
|
||||
// Best-effort: an enqueue failure must never abort the scan of remaining files.
|
||||
await this.enqueueEnrichment(trackId, String(artistId), String(albumId));
|
||||
} catch (err) {
|
||||
console.error(`[Scanner] Error processing ${filePath}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve an artist by its case-insensitive normalized identity, creating the
|
||||
* row only when no existing artist matches. This is what prevents case /
|
||||
* punctuation variants ("Booker" vs "BOOKER", "Acryl madness" vs "Acryl
|
||||
* Madness") from becoming separate rows (and duplicate albums). Returns the
|
||||
* artist id and the stored display name.
|
||||
*/
|
||||
private async resolveOrCreateArtist(rawName: string): Promise<{ id: string; name: string }> {
|
||||
const norm = await this.pgClient.query('SELECT normalize_artist($1) AS name', [rawName]);
|
||||
let name: string = norm.rows[0].name;
|
||||
|
||||
// Comma heuristic: prefer the pre-comma part if it already exists.
|
||||
if (name.includes(',')) {
|
||||
const before = name.split(',')[0].trim();
|
||||
const existing = await this.pgClient.query(
|
||||
'SELECT id FROM artists WHERE lower(normalized_name) = lower(normalize_artist($1)) LIMIT 1',
|
||||
[before]
|
||||
);
|
||||
if (existing.rows.length > 0) name = before;
|
||||
}
|
||||
|
||||
// Case-insensitive lookup by normalized identity; prefer the most "complete"
|
||||
// row when several somehow match.
|
||||
const found = await this.pgClient.query(
|
||||
`SELECT id, name FROM artists
|
||||
WHERE lower(normalized_name) = lower(normalize_artist($1))
|
||||
ORDER BY CASE WHEN mbid IS NOT NULL THEN 0 ELSE 1 END,
|
||||
CASE WHEN image_path IS NOT NULL AND image_path <> '' THEN 0 ELSE 1 END,
|
||||
created_at
|
||||
LIMIT 1`,
|
||||
[name]
|
||||
);
|
||||
if (found.rows.length > 0) {
|
||||
return { id: String(found.rows[0].id), name: String(found.rows[0].name) };
|
||||
}
|
||||
|
||||
const inserted = await this.pgClient.query(
|
||||
'INSERT INTO artists (name) VALUES ($1) RETURNING id, name',
|
||||
[name]
|
||||
);
|
||||
return { id: String(inserted.rows[0].id), name: String(inserted.rows[0].name) };
|
||||
}
|
||||
|
||||
private async enqueueEnrichment(trackId: string, artistId: string, albumId: string) {
|
||||
const keep = {
|
||||
removeOnComplete: { age: 86400, count: 5000 },
|
||||
removeOnFail: { age: 86400, count: 5000 },
|
||||
} as const;
|
||||
|
||||
// metadata_refresh per track. jobId `meta-<trackId>` collapses duplicate
|
||||
// pending jobs across re-scans; the handler (enrichTrack) is idempotent so
|
||||
// re-enqueues are always safe. BullMQ 5.x rejects colons in custom ids.
|
||||
try {
|
||||
const payload: MetadataRefreshJob = { trackId, refreshType: 'full' };
|
||||
await this.queue.add('metadata_refresh', payload, { jobId: `meta-${trackId}`, ...keep });
|
||||
} catch (err) {
|
||||
console.error(`[Scanner] Failed to enqueue metadata_refresh for track ${trackId}:`, err);
|
||||
}
|
||||
|
||||
// album_cover per album, deduped per scan (Set) + jobId across scans.
|
||||
if (!this.enqueuedAlbums.has(albumId)) {
|
||||
this.enqueuedAlbums.add(albumId);
|
||||
try {
|
||||
const payload: AlbumCoverJob = { albumId };
|
||||
await this.queue.add('album_cover', payload, { jobId: `album-cover-${albumId}`, ...keep });
|
||||
} catch (err) {
|
||||
console.error(`[Scanner] Failed to enqueue album_cover for album ${albumId}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
// artist_similarity + artist_image per artist, deduped per scan (Set) AND
|
||||
// across overlapping scans / pending jobs via their jobIds.
|
||||
if (this.enqueuedArtists.has(artistId)) {
|
||||
return;
|
||||
}
|
||||
this.enqueuedArtists.add(artistId);
|
||||
try {
|
||||
const payload: ArtistSimilarityJob = { artistId };
|
||||
await this.queue.add('artist_similarity', payload, { jobId: `artist-sim-${artistId}`, ...keep });
|
||||
} catch (err) {
|
||||
console.error(`[Scanner] Failed to enqueue artist_similarity for artist ${artistId}:`, err);
|
||||
}
|
||||
try {
|
||||
const payload: ArtistImageJob = { artistId };
|
||||
await this.queue.add('artist_image', payload, { jobId: `artist-image-${artistId}`, ...keep });
|
||||
} catch (err) {
|
||||
console.error(`[Scanner] Failed to enqueue artist_image for artist ${artistId}:`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* One-off cleanup: case-insensitive dedupe of artists AND albums.
|
||||
*
|
||||
* normalize_artist() preserves case, so case/punctuation variants of the same
|
||||
* artist ("Acryl madness" vs "Acryl Madness", "Booker" vs "BOOKER") were stored
|
||||
* as distinct rows. Each variant carried its own album, producing duplicate —
|
||||
* and often empty — albums (e.g. two "Before Neon 2", two "BROKESTAR").
|
||||
*
|
||||
* This:
|
||||
* 1. Merges artists sharing lower(normalized_name) into one keeper (prefers a
|
||||
* row with an MBID, then an image, then the most track links, then oldest).
|
||||
* track_artists / albums / artist_similar / artist_aliases are moved over;
|
||||
* losing artist rows are deleted.
|
||||
* 2. Merges albums sharing (artist_id, lower(title)) into one keeper (prefers a
|
||||
* row with artwork, then the most tracks, then oldest). Tracks are moved to
|
||||
* the keeper BEFORE the empty duplicate is deleted (albums cascade-delete
|
||||
* their tracks, so order matters).
|
||||
* 3. Drops redundant 'featured' track_artists where the same artist is already
|
||||
* 'main' on that track (case variants previously looked like a feature).
|
||||
*
|
||||
* Transactional + idempotent. Run:
|
||||
* DATABASE_URL=... npx tsx src/scripts/dedup-artists-albums.ts [--dry-run]
|
||||
*/
|
||||
import { Client as PgClient } from 'pg';
|
||||
|
||||
const DRY_RUN = process.argv.includes('--dry-run');
|
||||
|
||||
async function main() {
|
||||
const pg = new PgClient({ connectionString: process.env.DATABASE_URL });
|
||||
await pg.connect();
|
||||
console.log(`[Dedup] Connected${DRY_RUN ? ' (DRY RUN)' : ''}`);
|
||||
|
||||
if (!DRY_RUN) await pg.query('BEGIN');
|
||||
|
||||
let artistsMerged = 0;
|
||||
let albumsMerged = 0;
|
||||
|
||||
// --- 1. Artist dedupe (case-insensitive) ---------------------------------
|
||||
const artistGroups = await pg.query<{ ids: string[] }>(
|
||||
`SELECT array_agg(id ORDER BY
|
||||
CASE WHEN mbid IS NOT NULL THEN 0 ELSE 1 END,
|
||||
CASE WHEN image_path IS NOT NULL AND image_path <> '' THEN 0 ELSE 1 END,
|
||||
(SELECT count(*) FROM track_artists ta WHERE ta.artist_id = artists.id) DESC,
|
||||
created_at) AS ids
|
||||
FROM artists
|
||||
GROUP BY lower(normalized_name)
|
||||
HAVING count(*) > 1`
|
||||
);
|
||||
|
||||
for (const { ids } of artistGroups.rows) {
|
||||
const keep = ids[0];
|
||||
for (const loser of ids.slice(1)) {
|
||||
if (DRY_RUN) { artistsMerged++; continue; }
|
||||
|
||||
// track_artists: copy to keeper, drop loser's.
|
||||
await pg.query(
|
||||
`INSERT INTO track_artists (track_id, artist_id, role)
|
||||
SELECT track_id, $1, role FROM track_artists WHERE artist_id = $2
|
||||
ON CONFLICT (track_id, artist_id, role) DO NOTHING`,
|
||||
[keep, loser]
|
||||
);
|
||||
await pg.query(`DELETE FROM track_artists WHERE artist_id = $1`, [loser]);
|
||||
|
||||
// artist_similar + aliases: move with conflict-skip.
|
||||
await pg.query(
|
||||
`INSERT INTO artist_similar (artist_id, similar_name, match, fetched_at)
|
||||
SELECT $1, similar_name, match, fetched_at FROM artist_similar WHERE artist_id = $2
|
||||
ON CONFLICT (artist_id, similar_name) DO NOTHING`,
|
||||
[keep, loser]
|
||||
);
|
||||
await pg.query(`DELETE FROM artist_similar WHERE artist_id = $1`, [loser]);
|
||||
await pg.query(
|
||||
`INSERT INTO artist_aliases (artist_id, alias)
|
||||
SELECT $1, alias FROM artist_aliases WHERE artist_id = $2
|
||||
ON CONFLICT (artist_id, alias) DO NOTHING`,
|
||||
[keep, loser]
|
||||
);
|
||||
await pg.query(`DELETE FROM artist_aliases WHERE artist_id = $1`, [loser]);
|
||||
|
||||
// Albums: move to keeper where no same-title album exists; otherwise fold
|
||||
// tracks into the keeper's matching album, then drop the empty duplicate.
|
||||
const loserAlbums = await pg.query<{ id: string; title: string }>(
|
||||
`SELECT id, title FROM albums WHERE artist_id = $1`,
|
||||
[loser]
|
||||
);
|
||||
for (const la of loserAlbums.rows) {
|
||||
const match = await pg.query<{ id: string }>(
|
||||
`SELECT id FROM albums WHERE artist_id = $1 AND lower(title) = lower($2) LIMIT 1`,
|
||||
[keep, la.title]
|
||||
);
|
||||
if (match.rows.length === 0) {
|
||||
await pg.query(`UPDATE albums SET artist_id = $1 WHERE id = $2`, [keep, la.id]);
|
||||
} else {
|
||||
await foldAlbum(pg, match.rows[0].id, la.id);
|
||||
}
|
||||
}
|
||||
|
||||
// Carry over an image if the keeper lacks one.
|
||||
await pg.query(
|
||||
`UPDATE artists k SET image_path = l.image_path
|
||||
FROM artists l
|
||||
WHERE k.id = $1 AND l.id = $2
|
||||
AND (k.image_path IS NULL OR k.image_path = '')
|
||||
AND l.image_path IS NOT NULL AND l.image_path <> ''`,
|
||||
[keep, loser]
|
||||
);
|
||||
|
||||
await pg.query(`DELETE FROM artists WHERE id = $1`, [loser]);
|
||||
artistsMerged++;
|
||||
}
|
||||
}
|
||||
|
||||
// --- 2. Album dedupe (case-insensitive, within an artist) -----------------
|
||||
const albumGroups = await pg.query<{ ids: string[] }>(
|
||||
`SELECT array_agg(id ORDER BY
|
||||
CASE WHEN artwork_id IS NOT NULL AND artwork_id <> '' THEN 0 ELSE 1 END,
|
||||
(SELECT count(*) FROM tracks t WHERE t.album_id = albums.id) DESC,
|
||||
id) AS ids
|
||||
FROM albums
|
||||
WHERE artist_id IS NOT NULL
|
||||
GROUP BY artist_id, lower(title)
|
||||
HAVING count(*) > 1`
|
||||
);
|
||||
for (const { ids } of albumGroups.rows) {
|
||||
const keep = ids[0];
|
||||
for (const loser of ids.slice(1)) {
|
||||
if (DRY_RUN) { albumsMerged++; continue; }
|
||||
await foldAlbum(pg, keep, loser);
|
||||
albumsMerged++;
|
||||
}
|
||||
}
|
||||
|
||||
// --- 3. Drop redundant 'featured' where the artist is also 'main' ---------
|
||||
let redundant = 0;
|
||||
if (!DRY_RUN) {
|
||||
const r = await pg.query(
|
||||
`DELETE FROM track_artists f
|
||||
WHERE f.role = 'featured'
|
||||
AND EXISTS (SELECT 1 FROM track_artists m
|
||||
WHERE m.track_id = f.track_id AND m.artist_id = f.artist_id AND m.role = 'main')`
|
||||
);
|
||||
redundant = r.rowCount ?? 0;
|
||||
}
|
||||
|
||||
if (!DRY_RUN) await pg.query('COMMIT');
|
||||
console.log(`[Dedup] Done: artistsMerged=${artistsMerged} albumsMerged=${albumsMerged} redundantFeatured=${redundant}`);
|
||||
await pg.end();
|
||||
}
|
||||
|
||||
/** Move all tracks + (missing) artwork from loserAlbum into keeperAlbum, then delete loserAlbum. */
|
||||
async function foldAlbum(pg: PgClient, keeperId: string, loserId: string): Promise<void> {
|
||||
if (keeperId === loserId) return;
|
||||
await pg.query(`UPDATE tracks SET album_id = $1 WHERE album_id = $2`, [keeperId, loserId]);
|
||||
await pg.query(
|
||||
`UPDATE albums k SET artwork_id = l.artwork_id, year = COALESCE(k.year, l.year)
|
||||
FROM albums l
|
||||
WHERE k.id = $1 AND l.id = $2
|
||||
AND (k.artwork_id IS NULL OR k.artwork_id = '')
|
||||
AND l.artwork_id IS NOT NULL AND l.artwork_id <> ''`,
|
||||
[keeperId, loserId]
|
||||
);
|
||||
await pg.query(`DELETE FROM albums WHERE id = $1`, [loserId]);
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch((err) => {
|
||||
console.error('[Dedup] Failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* One-off cleanup: repair track rows corrupted by an earlier buggy
|
||||
* `metadata_refresh` worker job.
|
||||
*
|
||||
* The bug ran on EVERY refresh and concatenated (no separator) corruption
|
||||
* markers directly onto the real values:
|
||||
* - tracks.title += ' (Enriched)'
|
||||
* - tracks.artist += 'Unknown Artist'
|
||||
* Because it ran repeatedly, corruption can be stacked, e.g.
|
||||
* title = "Song (Enriched) (Enriched)"
|
||||
* artist = "RealName Unknown ArtistUnknown Artist"
|
||||
*
|
||||
* The repair logic (PASS 1 authoritative rescan + PASS 2 defensive SQL strip,
|
||||
* plus issue tracking) now lives in IntegrityService.runSweep(). This script is
|
||||
* just a run-once-and-exit entry point that delegates to it, so the regex/rescan
|
||||
* logic is never duplicated. Safe to run multiple times (idempotent).
|
||||
*/
|
||||
import { Client as PgClient } from 'pg';
|
||||
import { IntegrityService } from '../integrity.service.js';
|
||||
import { queue } from '../queue.js';
|
||||
|
||||
async function main() {
|
||||
const pgClient = new PgClient({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
});
|
||||
|
||||
await pgClient.connect();
|
||||
console.log('[Repair] Connected to PostgreSQL');
|
||||
|
||||
const integrity = new IntegrityService(pgClient, queue);
|
||||
const summary = await integrity.runSweep();
|
||||
console.log(
|
||||
`[Repair] Done: detected=${summary.detected} fixed=${summary.fixed} needsReview=${summary.needsReview}`
|
||||
);
|
||||
|
||||
await pgClient.end();
|
||||
console.log('[Repair] Connection closed.');
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch((err) => {
|
||||
console.error('[Repair] Failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,204 @@
|
||||
/**
|
||||
* One-off cleanup: split combined collaboration artist rows into the real
|
||||
* individual artists they name, and dedupe the result.
|
||||
*
|
||||
* Background: tags routinely store several artists in one string
|
||||
* ("Booker & ЗАМАЙ", "STED.D; Alphavite", "21 Savage & Metro Boomin"). An
|
||||
* earlier normalize_artist only handled commas + feat suffixes, so these were
|
||||
* persisted as a single bogus artist row — duplicating the real artists (which
|
||||
* also exist on their own) and producing two-artists-in-one entries.
|
||||
*
|
||||
* What this does, per combined artist row C (name splits into >1 artist):
|
||||
* 1. Resolve/create a canonical artist row for each component, keyed by
|
||||
* normalize_artist() so "Booker & X" folds into the existing "Booker".
|
||||
* 2. For every track linked to C (via track_artists), link it to all the real
|
||||
* components instead: first = 'main', the rest = 'featured'.
|
||||
* 3. Reassign C's albums to its primary component.
|
||||
* 4. Delete C (cascades its track_artists / artist_similar / aliases).
|
||||
* Then a final pass merges any artists that share a normalized_name (preferring
|
||||
* the row with an MBID, then a canonical_name).
|
||||
*
|
||||
* Idempotent: re-running after a clean run is a no-op (no row will split into
|
||||
* more than one component once the data is fixed). Wrapped in a transaction —
|
||||
* either the whole cleanup commits or nothing does.
|
||||
*
|
||||
* Run: DATABASE_URL=... npx tsx src/scripts/split-collab-artists.ts
|
||||
* (or, built) node dist/scripts/split-collab-artists.js
|
||||
* Add --dry-run to print what would change without writing.
|
||||
*/
|
||||
import { Client as PgClient } from 'pg';
|
||||
import { splitArtistNames } from '../utils/artist-names.js';
|
||||
|
||||
interface ArtistRow {
|
||||
id: string;
|
||||
name: string;
|
||||
mbid: string | null;
|
||||
canonical_name: string | null;
|
||||
}
|
||||
|
||||
const DRY_RUN = process.argv.includes('--dry-run');
|
||||
|
||||
async function main() {
|
||||
const pg = new PgClient({ connectionString: process.env.DATABASE_URL });
|
||||
await pg.connect();
|
||||
console.log(`[SplitCollab] Connected${DRY_RUN ? ' (DRY RUN — no writes)' : ''}`);
|
||||
|
||||
const { rows: artists } = await pg.query<ArtistRow>(
|
||||
'SELECT id, name, mbid, canonical_name FROM artists'
|
||||
);
|
||||
|
||||
// Identify the combined rows: name names more than one artist.
|
||||
const combined = artists.filter((a) => splitArtistNames(a.name).length > 1);
|
||||
console.log(`[SplitCollab] ${artists.length} artists total, ${combined.length} combined rows to split`);
|
||||
|
||||
if (!DRY_RUN) await pg.query('BEGIN');
|
||||
|
||||
let tracksRelinked = 0;
|
||||
let createdArtists = 0;
|
||||
|
||||
/**
|
||||
* Resolve a single artist name to a canonical artist id, by normalized_name.
|
||||
* Creates the row if no match exists. Returns the id.
|
||||
*/
|
||||
async function resolveArtistId(name: string): Promise<string> {
|
||||
const existing = await pg.query(
|
||||
`SELECT id FROM artists
|
||||
WHERE normalized_name = normalize_artist($1)
|
||||
ORDER BY CASE WHEN mbid IS NOT NULL THEN 0 ELSE 1 END,
|
||||
CASE WHEN canonical_name IS NOT NULL THEN 0 ELSE 1 END,
|
||||
created_at
|
||||
LIMIT 1`,
|
||||
[name]
|
||||
);
|
||||
if (existing.rows.length > 0) return existing.rows[0].id;
|
||||
|
||||
if (DRY_RUN) {
|
||||
console.log(`[SplitCollab] would create artist "${name}"`);
|
||||
return `dry-${name}`;
|
||||
}
|
||||
const inserted = await pg.query(
|
||||
`INSERT INTO artists (name, canonical_name, sort_name)
|
||||
VALUES ($1, $1, $1) RETURNING id`,
|
||||
[name]
|
||||
);
|
||||
createdArtists++;
|
||||
return inserted.rows[0].id;
|
||||
}
|
||||
|
||||
for (const c of combined) {
|
||||
const components = splitArtistNames(c.name);
|
||||
console.log(`[SplitCollab] "${c.name}" -> [${components.join(' | ')}]`);
|
||||
|
||||
const componentIds: string[] = [];
|
||||
for (const name of components) componentIds.push(await resolveArtistId(name));
|
||||
const primaryId = componentIds[0];
|
||||
|
||||
if (DRY_RUN) continue;
|
||||
|
||||
// Tracks currently attributed to the combined row.
|
||||
const { rows: links } = await pg.query<{ track_id: string }>(
|
||||
'SELECT DISTINCT track_id FROM track_artists WHERE artist_id = $1',
|
||||
[c.id]
|
||||
);
|
||||
|
||||
for (const { track_id } of links) {
|
||||
// primary = main, the rest = featured. Skip self-conflicts.
|
||||
await pg.query(
|
||||
`INSERT INTO track_artists (track_id, artist_id, role)
|
||||
VALUES ($1, $2, 'main') ON CONFLICT (track_id, artist_id, role) DO NOTHING`,
|
||||
[track_id, primaryId]
|
||||
);
|
||||
for (const featId of componentIds.slice(1)) {
|
||||
if (featId === primaryId) continue;
|
||||
await pg.query(
|
||||
`INSERT INTO track_artists (track_id, artist_id, role)
|
||||
VALUES ($1, $2, 'featured') ON CONFLICT (track_id, artist_id, role) DO NOTHING`,
|
||||
[track_id, featId]
|
||||
);
|
||||
}
|
||||
tracksRelinked++;
|
||||
}
|
||||
|
||||
// Reassign the combined row's albums to the primary artist, but only where
|
||||
// that wouldn't collide with an existing (primary, title) album. Colliding
|
||||
// albums are dropped (the primary already has that album).
|
||||
await pg.query(
|
||||
`UPDATE albums a SET artist_id = $1
|
||||
WHERE a.artist_id = $2
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM albums b WHERE b.artist_id = $1 AND b.title = a.title
|
||||
)`,
|
||||
[primaryId, c.id]
|
||||
);
|
||||
|
||||
// Drop the combined artist (cascades remaining track_artists/aliases/similar
|
||||
// and any colliding albums via ON DELETE CASCADE).
|
||||
await pg.query('DELETE FROM artists WHERE id = $1', [c.id]);
|
||||
}
|
||||
|
||||
// Final dedupe: collapse any artists sharing a normalized_name into one,
|
||||
// preferring the row with an MBID, then a canonical_name, then oldest.
|
||||
let dedupMerged = 0;
|
||||
if (!DRY_RUN) {
|
||||
const { rows: dups } = await pg.query<{ ids: string[] }>(
|
||||
`SELECT array_agg(id ORDER BY
|
||||
CASE WHEN mbid IS NOT NULL THEN 0 ELSE 1 END,
|
||||
CASE WHEN canonical_name IS NOT NULL THEN 0 ELSE 1 END,
|
||||
created_at) AS ids
|
||||
FROM artists
|
||||
GROUP BY normalized_name
|
||||
HAVING COUNT(*) > 1`
|
||||
);
|
||||
for (const { ids } of dups) {
|
||||
const keepId = ids[0];
|
||||
for (const mergeId of ids.slice(1)) {
|
||||
await pg.query(
|
||||
`INSERT INTO track_artists (track_id, artist_id, role)
|
||||
SELECT track_id, $1, role FROM track_artists WHERE artist_id = $2
|
||||
ON CONFLICT (track_id, artist_id, role) DO NOTHING`,
|
||||
[keepId, mergeId]
|
||||
);
|
||||
await pg.query('DELETE FROM track_artists WHERE artist_id = $1', [mergeId]);
|
||||
await pg.query(
|
||||
`UPDATE albums a SET artist_id = $1
|
||||
WHERE a.artist_id = $2
|
||||
AND NOT EXISTS (SELECT 1 FROM albums b WHERE b.artist_id = $1 AND b.title = a.title)`,
|
||||
[keepId, mergeId]
|
||||
);
|
||||
await pg.query('DELETE FROM artists WHERE id = $1', [mergeId]);
|
||||
dedupMerged++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Drop redundant 'featured' links where the same artist is already 'main' on
|
||||
// that track (can happen when a stale pre-split 'main' link coexists with a
|
||||
// 'featured' link added from a collaboration row).
|
||||
let redundantRemoved = 0;
|
||||
if (!DRY_RUN) {
|
||||
const res = await pg.query(
|
||||
`DELETE FROM track_artists f
|
||||
WHERE f.role = 'featured'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM track_artists m
|
||||
WHERE m.track_id = f.track_id AND m.artist_id = f.artist_id AND m.role = 'main'
|
||||
)`
|
||||
);
|
||||
redundantRemoved = res.rowCount ?? 0;
|
||||
}
|
||||
|
||||
if (!DRY_RUN) await pg.query('COMMIT');
|
||||
|
||||
console.log(
|
||||
`[SplitCollab] Done: split=${combined.length} relinkedTracks=${tracksRelinked} ` +
|
||||
`createdArtists=${createdArtists} dedupMerged=${dedupMerged} redundantFeaturedRemoved=${redundantRemoved}`
|
||||
);
|
||||
await pg.end();
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => process.exit(0))
|
||||
.catch(async (err) => {
|
||||
console.error('[SplitCollab] Failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
export interface MetadataRefreshJob {
|
||||
trackId: string;
|
||||
refreshType: 'full' | 'partial';
|
||||
}
|
||||
|
||||
export interface AudioAnalysisJob {
|
||||
trackId: string;
|
||||
features: string[];
|
||||
}
|
||||
|
||||
export interface CleanupJob {
|
||||
reason: 'expired' | 'manual';
|
||||
targetFiles: string[];
|
||||
}
|
||||
|
||||
export interface LibraryScanJob {
|
||||
directory: string;
|
||||
}
|
||||
|
||||
// Periodic integrity sweep. No payload fields required; the worker reads
|
||||
// MUSIC_DIR from env. Kept as an object for forward-compatibility.
|
||||
export interface IntegritySweepJob {
|
||||
// Reserved for future scoping options (e.g. specific track ids).
|
||||
reason?: 'scheduled' | 'manual';
|
||||
}
|
||||
|
||||
// Refresh Last.fm similar-artist data for a single artist (feeds Vibe discovery).
|
||||
export interface ArtistSimilarityJob {
|
||||
artistId: string;
|
||||
}
|
||||
|
||||
// Fetch/refresh a single artist's image via the fallback chain. Decoupled from
|
||||
// track enrichment so image lookups don't run inline while processing tracks.
|
||||
export interface ArtistImageJob {
|
||||
artistId: string;
|
||||
}
|
||||
|
||||
// Fetch/refresh a single album's cover art (Discogs + Cover Art Archive).
|
||||
// Decoupled from track enrichment for the same reason.
|
||||
export interface AlbumCoverJob {
|
||||
albumId: string;
|
||||
}
|
||||
|
||||
// Re-process all artists through the new identity pipeline.
|
||||
// Useful after schema migrations or when identity logic changes.
|
||||
export interface ReprocessArtistsJob {
|
||||
batchSize?: number;
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export type JobPayload =
|
||||
| MetadataRefreshJob
|
||||
| ArtistSimilarityJob
|
||||
| ArtistImageJob
|
||||
| AlbumCoverJob
|
||||
| AudioAnalysisJob
|
||||
| CleanupJob
|
||||
| LibraryScanJob
|
||||
| IntegritySweepJob
|
||||
| ReprocessArtistsJob;
|
||||
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* Shared artist-name parsing.
|
||||
*
|
||||
* Music tags routinely cram several artists into one string using a variety of
|
||||
* separators ("A & B", "A; B", "A, B", "A / B", "A x B") and feature markers
|
||||
* ("A feat. B", "A ft. B", "Song (feat. B)"). Historically these were stored as
|
||||
* a single combined artist row (e.g. "Booker & ЗАМАЙ"), which both duplicated
|
||||
* the real artists and produced bogus two-artists-in-one entries.
|
||||
*
|
||||
* This module is the single source of truth for breaking such a string into the
|
||||
* individual artists it actually names. The FIRST artist is treated as the main
|
||||
* artist; the rest are collaborators/features. Used by the scanner (at ingest)
|
||||
* and by the split-collab-artists migration (to clean existing rows).
|
||||
*/
|
||||
|
||||
// Collaboration separators that join co-billed artists: ; / & , and the
|
||||
// space-delimited word "x" ("Баста x Capella"). NOTE: matched case-insensitively
|
||||
// and only as a standalone token so it doesn't chop names that merely contain x.
|
||||
const SEPARATOR_RE = /\s*(?:;|\/|&|,)\s*|\s+x\s+/i;
|
||||
|
||||
// Feature/versus markers. Everything after the marker is a featured artist (or a
|
||||
// list of them, which is split again on SEPARATOR_RE by the caller).
|
||||
const FEAT_SPLIT_RE = /\s+(?:feat(?:uring)?\.?|ft\.?|vs\.?)\s+/i;
|
||||
|
||||
// Parenthesized feature marker inside a track title: "Song (feat. Guest)".
|
||||
const TITLE_FEAT_RE = /[\(\[]\s*(?:feat(?:uring)?\.?|ft\.?|vs\.?)\s+([^\)\]]+)[\)\]]/i;
|
||||
|
||||
/** Case-insensitive, order-preserving dedupe. */
|
||||
function dedupe(names: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
for (const raw of names) {
|
||||
const n = raw.trim();
|
||||
if (!n) continue;
|
||||
const key = n.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(n);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a raw artist string into the individual artists it names, in billing
|
||||
* order. Handles feature markers and all collaboration separators. Returns a
|
||||
* deduped list; a plain single artist yields a one-element array.
|
||||
*/
|
||||
export function splitArtistNames(rawArtist: string): string[] {
|
||||
if (!rawArtist) return [];
|
||||
// Feature markers first: "A feat. B & C" -> ["A", "B & C"], then each chunk is
|
||||
// split on collaboration separators.
|
||||
const featChunks = rawArtist.split(FEAT_SPLIT_RE);
|
||||
const parts: string[] = [];
|
||||
for (const chunk of featChunks) {
|
||||
parts.push(...chunk.split(SEPARATOR_RE));
|
||||
}
|
||||
return dedupe(parts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the main + featured artists for a track from its raw artist string and
|
||||
* (optionally) its title. First named artist is main; everything else is
|
||||
* featured. `featured` excludes the main artist and is deduped.
|
||||
*/
|
||||
export function parseArtists(
|
||||
rawArtist: string,
|
||||
rawTitle = ''
|
||||
): { main: string; featured: string[] } {
|
||||
const fromArtist = splitArtistNames(rawArtist || 'Unknown Artist');
|
||||
|
||||
// Pull any "(feat. X)" group out of the title and split it too.
|
||||
const titleMatch = rawTitle.match(TITLE_FEAT_RE);
|
||||
const fromTitle = titleMatch ? splitArtistNames(titleMatch[1]) : [];
|
||||
|
||||
const all = dedupe([...fromArtist, ...fromTitle]);
|
||||
if (all.length === 0) return { main: rawArtist.trim() || 'Unknown Artist', featured: [] };
|
||||
|
||||
const main = all[0];
|
||||
const mainKey = main.toLowerCase();
|
||||
const featured = all.slice(1).filter((a) => a.toLowerCase() !== mainKey);
|
||||
return { main, featured };
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// Fuzzy string matching utilities for artist deduplication.
|
||||
// Implements Levenshtein distance and Jaro-Winkler similarity.
|
||||
|
||||
/**
|
||||
* Levenshtein distance between two strings.
|
||||
* Returns the minimum number of single-character edits (insertions, deletions, substitutions)
|
||||
* required to change one string into the other.
|
||||
*/
|
||||
export function levenshteinDistance(a: string, b: string): number {
|
||||
if (a === b) return 0;
|
||||
if (a.length === 0) return b.length;
|
||||
if (b.length === 0) return a.length;
|
||||
|
||||
const matrix = new Array(a.length + 1).fill(null).map(() => new Array(b.length + 1).fill(0));
|
||||
|
||||
for (let i = 0; i <= a.length; i++) matrix[i][0] = i;
|
||||
for (let j = 0; j <= b.length; j++) matrix[0][j] = j;
|
||||
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
for (let j = 1; j <= b.length; j++) {
|
||||
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
||||
matrix[i][j] = Math.min(
|
||||
matrix[i - 1][j] + 1, // deletion
|
||||
matrix[i][j - 1] + 1, // insertion
|
||||
matrix[i - 1][j - 1] + cost // substitution
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return matrix[a.length][b.length];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalized Levenshtein similarity (0..1).
|
||||
* 1 = identical, 0 = completely different.
|
||||
*/
|
||||
export function levenshteinSimilarity(a: string, b: string): number {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 1;
|
||||
const distance = levenshteinDistance(a, b);
|
||||
return 1 - distance / maxLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jaro similarity between two strings.
|
||||
* Returns a value between 0 (no similarity) and 1 (identical).
|
||||
*/
|
||||
export function jaroSimilarity(a: string, b: string): number {
|
||||
if (a === b) return 1;
|
||||
if (a.length === 0 || b.length === 0) return 0;
|
||||
|
||||
const matchWindow = Math.floor(Math.max(a.length, b.length) / 2) - 1;
|
||||
const aMatches = new Array(a.length).fill(false);
|
||||
const bMatches = new Array(b.length).fill(false);
|
||||
|
||||
let matches = 0;
|
||||
let transpositions = 0;
|
||||
|
||||
// Find matches
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const start = Math.max(0, i - matchWindow);
|
||||
const end = Math.min(i + matchWindow + 1, b.length);
|
||||
for (let j = start; j < end; j++) {
|
||||
if (bMatches[j] || a[i] !== b[j]) continue;
|
||||
aMatches[i] = bMatches[j] = true;
|
||||
matches++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (matches === 0) return 0;
|
||||
|
||||
// Count transpositions
|
||||
let k = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
if (!aMatches[i]) continue;
|
||||
while (!bMatches[k]) k++;
|
||||
if (a[i] !== b[k]) transpositions++;
|
||||
k++;
|
||||
}
|
||||
|
||||
const m = matches;
|
||||
return (m / a.length + m / b.length + (m - transpositions / 2) / m) / 3;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jaro-Winkler similarity.
|
||||
* Gives more weight to common prefixes.
|
||||
* Returns a value between 0 (no similarity) and 1 (identical).
|
||||
*/
|
||||
export function jaroWinklerSimilarity(a: string, b: string, prefixScale = 0.1): number {
|
||||
const jaro = jaroSimilarity(a, b);
|
||||
|
||||
if (jaro < 0.7) return jaro; // Only boost if already somewhat similar
|
||||
|
||||
// Calculate common prefix length (max 4)
|
||||
let prefixLen = 0;
|
||||
const minLen = Math.min(a.length, b.length, 4);
|
||||
for (let i = 0; i < minLen; i++) {
|
||||
if (a[i] === b[i]) prefixLen++;
|
||||
else break;
|
||||
}
|
||||
|
||||
return jaro + prefixLen * prefixScale * (1 - jaro);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize artist name for matching.
|
||||
* Lowercase, strip diacritics, remove collaboration suffixes.
|
||||
*/
|
||||
export function normalizeForMatching(name: string): string {
|
||||
return name
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036f]/g, '') // Remove diacritics
|
||||
.replace(/\s*\(?\s*(feat(?:uring)?\.?|ft\.?|vs\.?|x)\s+.*$/i, '')
|
||||
.replace(/,\s*$/, '')
|
||||
.replace(/^([^,]+),.*$/, '$1')
|
||||
.replace(/[&+]/g, 'and')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Find best match for a name among candidates.
|
||||
* Returns { match: string, score: number } or null if no good match.
|
||||
*/
|
||||
export function findBestMatch<T extends { name: string }>(
|
||||
query: string,
|
||||
candidates: T[],
|
||||
threshold = 0.9
|
||||
): { match: T; score: number } | null {
|
||||
const normalizedQuery = normalizeForMatching(query);
|
||||
let bestMatch: { match: T; score: number } | null = null;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const normalizedCandidate = normalizeForMatching(candidate.name);
|
||||
|
||||
// Exact normalized match
|
||||
if (normalizedQuery === normalizedCandidate) {
|
||||
return { match: candidate, score: 1 };
|
||||
}
|
||||
|
||||
// Jaro-Winkler similarity
|
||||
const score = jaroWinklerSimilarity(normalizedQuery, normalizedCandidate);
|
||||
|
||||
if (score >= threshold && (!bestMatch || score > bestMatch.score)) {
|
||||
bestMatch = { match: candidate, score };
|
||||
}
|
||||
}
|
||||
|
||||
return bestMatch;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if two artist names likely represent the same artist.
|
||||
*/
|
||||
export function areSameArtist(name1: string, name2: string, threshold = 0.95): boolean {
|
||||
const norm1 = normalizeForMatching(name1);
|
||||
const norm2 = normalizeForMatching(name2);
|
||||
|
||||
if (norm1 === norm2) return true;
|
||||
|
||||
const score = jaroWinklerSimilarity(norm1, norm2);
|
||||
return score >= threshold;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"outDir": "./dist"
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
Reference in New Issue
Block a user