import fs from 'fs/promises'; import { createReadStream } from 'fs'; import { createHash } from 'crypto'; import path from 'path'; import mm from 'music-metadata'; import type { Queryable } from './db.js'; import { Queue } from 'bullmq'; import { MetadataRefreshJob, ArtistSimilarityJob, ArtistImageJob, AlbumCoverJob } from './types.js'; import { splitArtistNames, parseArtists } from './utils/artist-names.js'; /** * Parse main + featured artists from music-metadata. Prefers the structured * `artists[]` array when the tag provides it (each entry already one artist), * otherwise falls back to splitting the single `artist` string. Feature markers * in the title ("Song (feat. X)") are folded in either way. First artist is the * main artist; the rest are featured. See ./utils/artist-names for the rules. */ function parseArtistsFromMetadata(common: any): { main: string; featured: string[] } { const rawTitle = common.title || ''; // Structured array present: still split each entry (tags sometimes put a whole // "A & B" collaboration in one slot) and fold in any title feature. if (common.artists && Array.isArray(common.artists) && common.artists.length > 0) { const fromArray = common.artists.flatMap((a: string) => splitArtistNames(a)); const fromTitleParse = parseArtists('', rawTitle); const all: string[] = []; const seen = new Set(); for (const name of [...fromArray, ...fromTitleParse.featured]) { const n = name.trim(); if (!n || seen.has(n.toLowerCase())) continue; seen.add(n.toLowerCase()); all.push(n); } if (all.length > 0) { return { main: all[0], featured: all.slice(1) }; } } return parseArtists(common.artist || 'Unknown Artist', rawTitle); } function hashFile(filePath: string): Promise { return new Promise((resolve, reject) => { const hash = createHash('md5'); const stream = createReadStream(filePath); stream.on('data', (d) => hash.update(d)); stream.on('end', () => resolve(hash.digest('hex'))); stream.on('error', reject); }); } /** * Resolve the album artist from metadata tags. The `albumartist` tag is the * standard way to identify who "owns" the album as a whole — "Various Artists" * for compilations, "Soundtrack" for OSTs, or the primary artist for normal * albums. Without it, compilations where each track has a different artist * would create duplicate album rows (one per track artist). * * Falls back to the track's main artist when the tag is absent (single-artist * albums where the tag is redundant). * * Also splits the albumartist string via splitArtistNames so that "A & B" as * albumartist uses just the first-billed artist as the album's identity — * consistent with how normalize_artist treats track artists. */ function resolveAlbumArtist(common: any, fallback: string): string { // music-metadata exposes albumartist as either a string or an array. const raw = common.albumartist ?? common.albumArtist; if (!raw) return fallback; const str = Array.isArray(raw) ? raw[0] : String(raw); if (str.trim() === '') return fallback; const parts = splitArtistNames(str); return parts.length > 0 ? parts[0] : fallback; } export class ScannerService { // Per-scan dedupe sets for per-artist / per-album jobs. Belt-and-braces // alongside the BullMQ jobId dedupe (jobId dedupes across overlapping scans / // pending jobs; these avoid even issuing the redundant add() within one scan). private enqueuedArtists = new Set(); private enqueuedAlbums = new Set(); constructor(private pgClient: Queryable, private queue: Queue) {} async scanDirectory(directory: string) { console.log(`[Scanner] Starting scan in: ${directory}`); this.enqueuedArtists.clear(); this.enqueuedAlbums.clear(); await this.walk(directory); console.log(`[Scanner] Scan completed.`); } private async walk(dir: string) { const entries = await fs.readdir(dir, { withFileTypes: true }); for (const entry of entries) { const fullPath = path.join(dir, entry.name); if (entry.isDirectory()) { await this.walk(fullPath); } else if (this.isMusicFile(entry.name)) { await this.processFile(fullPath); } } } private isMusicFile(fileName: string): boolean { const extensions = ['.mp3', '.flac', '.m4a', '.wav', '.ogg']; return extensions.includes(path.extname(fileName).toLowerCase()); } private async processFile(filePath: string) { try { console.log(`[Scanner] Processing: ${filePath}`); const metadata = await mm.parseFile(filePath); const { common, format } = metadata; // 1. Ensure Artist(s) exist. const trackTitle = common.title || path.basename(filePath); const { main: mainArtistRaw, featured: featuredArtistNames } = parseArtistsFromMetadata(common); const { id: artistId, name: resolvedArtist } = await this.resolveOrCreateArtist(mainArtistRaw); // Upsert featured artists into artists table and collect their ids. const featuredIds: string[] = []; for (const featName of featuredArtistNames) { const feat = await this.resolveOrCreateArtist(featName); featuredIds.push(feat.id); } // 2. Ensure Album exists. // The album is keyed by (album_artist_id, title), NOT (track_artist_id, // title). This prevents duplicate album rows for compilations where // each track has a different artist but they all belong to one album. // We resolve the ALBUM artist from the `albumartist` tag when present // ("Various Artists", "Soundtrack", or the primary artist), falling // back to the track's main artist when the tag is absent. const albumTitle = common.album || 'Unknown Album'; const albumArtistRaw = resolveAlbumArtist(common, mainArtistRaw); const albumArtistId = albumArtistRaw === mainArtistRaw ? artistId : (await this.resolveOrCreateArtist(albumArtistRaw)).id; const existingAlbum = await this.pgClient.query( 'SELECT id FROM albums WHERE artist_id = $1 AND lower(title) = lower($2) LIMIT 1', [albumArtistId, albumTitle] ); let albumId: string; if (existingAlbum.rows.length > 0) { albumId = String(existingAlbum.rows[0].id); } else { const albumRes = await this.pgClient.query( 'INSERT INTO albums (artist_id, title) VALUES ($1, $2) ON CONFLICT (artist_id, title) DO UPDATE SET title = EXCLUDED.title RETURNING id', [albumArtistId, albumTitle] ); albumId = String(albumRes.rows[0].id); } // 3. Upsert Track const duration = format.duration || 0; const fileHash = await hashFile(filePath); const trackRes = await this.pgClient.query( `INSERT INTO tracks (path, hash, title, artist, album_id, duration, state) VALUES ($1, $2, $3, $4, $5, $6, 'LIBRARY') ON CONFLICT (path) DO UPDATE SET hash = EXCLUDED.hash, title = EXCLUDED.title, artist = EXCLUDED.artist, album_id = EXCLUDED.album_id, duration = EXCLUDED.duration, mtime = EXTRACT(EPOCH FROM NOW()) RETURNING id `, [filePath, fileHash, trackTitle, resolvedArtist, albumId, duration] ); const trackId = String(trackRes.rows[0].id); // 4. Populate track_artists junction table. await this.pgClient.query( `INSERT INTO track_artists (track_id, artist_id, role) VALUES ($1, $2, 'main') ON CONFLICT (track_id, artist_id, role) DO NOTHING`, [trackId, artistId] ); for (const featId of featuredIds) { await this.pgClient.query( `INSERT INTO track_artists (track_id, artist_id, role) VALUES ($1, $2, 'featured') ON CONFLICT (track_id, artist_id, role) DO NOTHING`, [trackId, featId] ); } console.log(`[Scanner] Successfully processed: ${trackTitle}`); // Trigger external-API enrichment for this track + artist + album. // Best-effort: an enqueue failure must never abort the scan of remaining files. await this.enqueueEnrichment(trackId, String(artistId), String(albumId)); } catch (err) { console.error(`[Scanner] Error processing ${filePath}:`, err); } } /** * Resolve an artist by its case-insensitive normalized identity, creating the * row only when no existing artist matches. This is what prevents case / * punctuation variants ("Booker" vs "BOOKER", "Acryl madness" vs "Acryl * Madness") from becoming separate rows (and duplicate albums). Returns the * artist id and the stored display name. */ private async resolveOrCreateArtist(rawName: string): Promise<{ id: string; name: string }> { const norm = await this.pgClient.query('SELECT normalize_artist($1) AS name', [rawName]); let name: string = norm.rows[0].name; // Comma heuristic: prefer the pre-comma part if it already exists. if (name.includes(',')) { const before = name.split(',')[0].trim(); const existing = await this.pgClient.query( 'SELECT id FROM artists WHERE lower(normalized_name) = lower(normalize_artist($1)) LIMIT 1', [before] ); if (existing.rows.length > 0) name = before; } // Case-insensitive lookup by normalized identity; prefer the most "complete" // row when several somehow match. const found = await this.pgClient.query( `SELECT id, name FROM artists WHERE lower(normalized_name) = lower(normalize_artist($1)) ORDER BY CASE WHEN mbid IS NOT NULL THEN 0 ELSE 1 END, CASE WHEN image_path IS NOT NULL AND image_path <> '' THEN 0 ELSE 1 END, created_at LIMIT 1`, [name] ); if (found.rows.length > 0) { return { id: String(found.rows[0].id), name: String(found.rows[0].name) }; } // `canonical_name` is NOT NULL in schema.sql with no default, so it MUST be // supplied here — omitting it makes every artist insert fail on a fresh // volume (and processFile swallows the error, so the scan silently yields an // empty library). It holds the DISPLAY name: we store the raw tag name, not // the normalize_artist() output, because that function truncates on `/` and // a standalone `x` ("AC/DC" -> "AC", "Felix Mendelssohn" -> "Feli"). The // enrichment path later overwrites canonical_name with the MusicBrainz name; // until then the raw tag is the most faithful display value we have. const inserted = await this.pgClient.query( 'INSERT INTO artists (name, canonical_name) VALUES ($1, $2) RETURNING id, name', [name, rawName.trim() || name] ); return { id: String(inserted.rows[0].id), name: String(inserted.rows[0].name) }; } private async enqueueEnrichment(trackId: string, artistId: string, albumId: string) { const keep = { removeOnComplete: { age: 86400, count: 5000 }, removeOnFail: { age: 86400, count: 5000 }, } as const; // metadata_refresh per track. jobId `meta-` collapses duplicate // pending jobs across re-scans; the handler (enrichTrack) is idempotent so // re-enqueues are always safe. BullMQ 5.x rejects colons in custom ids. try { const payload: MetadataRefreshJob = { trackId, refreshType: 'full' }; await this.queue.add('metadata_refresh', payload, { jobId: `meta-${trackId}`, ...keep }); } catch (err) { console.error(`[Scanner] Failed to enqueue metadata_refresh for track ${trackId}:`, err); } // album_cover per album, deduped per scan (Set) + jobId across scans. if (!this.enqueuedAlbums.has(albumId)) { this.enqueuedAlbums.add(albumId); try { const payload: AlbumCoverJob = { albumId }; await this.queue.add('album_cover', payload, { jobId: `album-cover-${albumId}`, ...keep }); } catch (err) { console.error(`[Scanner] Failed to enqueue album_cover for album ${albumId}:`, err); } } // artist_similarity + artist_image per artist, deduped per scan (Set) AND // across overlapping scans / pending jobs via their jobIds. if (this.enqueuedArtists.has(artistId)) { return; } this.enqueuedArtists.add(artistId); try { const payload: ArtistSimilarityJob = { artistId }; await this.queue.add('artist_similarity', payload, { jobId: `artist-sim-${artistId}`, ...keep }); } catch (err) { console.error(`[Scanner] Failed to enqueue artist_similarity for artist ${artistId}:`, err); } try { const payload: ArtistImageJob = { artistId }; await this.queue.add('artist_image', payload, { jobId: `artist-image-${artistId}`, ...keep }); } catch (err) { console.error(`[Scanner] Failed to enqueue artist_image for artist ${artistId}:`, err); } } }