/** * AudioFeaturesService — two-pass audio feature extraction. * * Pass 1 (fast): read BPM and key from embedded file tags via music-metadata. * Many files tagged by beets / MusicBrainz Picard / Mixed In Key already * carry these, so we get the answer for free. * * Pass 2 (compute): run essentia.js (WebAssembly) on the decoded PCM signal * for anything still missing. Audio decoding is done by a ffmpeg subprocess * so every format ffmpeg understands (MP3, FLAC, AAC, OGG, …) is supported. * Computed features: BPM, key, energy, danceability, dynamic complexity. * * The two passes are combined via COALESCE so that embedded-tag values are * never overwritten once they exist. */ import { spawn } from 'child_process'; import mm from 'music-metadata'; import { Client as PgClient } from 'pg'; // Lazy WASM singleton — heavy to load (~2.4 MB), so we initialise once and // reuse across all enrichment jobs within the same worker process. let essentiaReady: Promise<{ essentia: any }> | null = null; async function getEssentia() { if (!essentiaReady) { essentiaReady = (async () => { const mod = await import('essentia.js'); // essentia.js v0.1.3: default export contains { Essentia, EssentiaWASM, ... } // EssentiaWASM is already the instantiated WASM module (not a function) const EssentiaWASM = mod.default?.EssentiaWASM ?? mod.EssentiaWASM; const Essentia = mod.default?.Essentia ?? mod.Essentia; if (!EssentiaWASM || !Essentia) { throw new Error('essentia.js exports missing EssentiaWASM or Essentia'); } const essentia = new Essentia(EssentiaWASM); return { essentia }; })(); } return essentiaReady; } /** Decode any audio file to a mono Float32Array at 44100 Hz via ffmpeg. */ function decodeAudioToFloat32(filePath: string): Promise { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; const ff = spawn('ffmpeg', [ '-i', filePath, '-vn', '-acodec', 'pcm_f32le', '-ar', '44100', '-ac', '1', '-f', 'f32le', 'pipe:1', ], { stdio: ['ignore', 'pipe', 'ignore'] }); ff.stdout.on('data', (chunk: Buffer) => chunks.push(chunk)); ff.stdout.on('end', () => { const buf = Buffer.concat(chunks); resolve(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4)); }); ff.on('error', reject); ff.on('close', (code) => { if (code !== 0 && chunks.length === 0) { reject(new Error(`ffmpeg exited with code ${code} for ${filePath}`)); } }); }); } export interface AudioFeatures { bpm: number | null; key: string | null; energy: number | null; danceability: number | null; dynamicComplexity: number | null; } export class AudioFeaturesService { constructor(private pgClient: PgClient) {} async ensureSchema(): Promise { await this.pgClient.query( `CREATE TABLE IF NOT EXISTS track_audio_features ( track_id UUID PRIMARY KEY REFERENCES tracks(id) ON DELETE CASCADE, bpm REAL, key TEXT, energy REAL, danceability REAL, valence REAL, acousticness REAL, instrumentalness REAL, liveness REAL, valence_score REAL, tempo REAL )` ); } // ── Pass 1: embedded tags ────────────────────────────────────────────────── private async readEmbeddedTags(filePath: string): Promise<{ bpm: number | null; key: string | null; replayGainDb: number | null }> { try { const { common } = await mm.parseFile(filePath, { duration: false }); const bpm = common.bpm && Number.isFinite(common.bpm) && common.bpm > 0 ? common.bpm : null; const key = common.key?.trim() || null; const rgRaw = (common as any).replaygain_track_gain; let replayGainDb: number | null = null; if (rgRaw != null) { const val = typeof rgRaw === 'object' ? rgRaw.dB : Number(String(rgRaw).replace(/[^\d.-]/g, '')); if (Number.isFinite(val)) replayGainDb = val; } return { bpm, key, replayGainDb }; } catch { return { bpm: null, key: null, replayGainDb: null }; } } // ── Pass 2: essentia.js DSP ──────────────────────────────────────────────── private async analyseWithEssentia(filePath: string, needBpm: boolean, needKey: boolean): Promise { const result: AudioFeatures = { bpm: null, key: null, energy: null, danceability: null, dynamicComplexity: null }; const { essentia } = await getEssentia(); const signal = await decodeAudioToFloat32(filePath); if (signal.length === 0) return result; const vec = essentia.arrayToVector(signal); if (needBpm) { try { const rhythm = essentia.RhythmExtractor2013(vec); if (rhythm.bpm > 0) result.bpm = Math.round(rhythm.bpm * 10) / 10; } catch (err) { console.warn('[AudioFeatures] RhythmExtractor2013 failed:', (err as Error).message); } } if (needKey) { try { const keyResult = essentia.KeyExtractor(vec); if (keyResult.key) result.key = `${keyResult.key} ${keyResult.scale}`; } catch (err) { console.warn('[AudioFeatures] KeyExtractor failed:', (err as Error).message); } } // Energy and danceability are only available from signal analysis — always compute. try { const energy = essentia.Energy(vec); // Essentia Energy returns the sum of squared samples, normalised to [0,1] // by dividing by signal length. Map to a sensible 0-1 display value. const rawEnergy = energy.energy / signal.length; result.energy = Math.min(1, rawEnergy * 1000); // typical values << 0.001 } catch (err) { console.warn('[AudioFeatures] Energy failed:', (err as Error).message); } try { const dance = essentia.Danceability(vec); // Danceability output range is 0-3; normalise to 0-1. result.danceability = Math.min(1, dance.danceability / 3); } catch (err) { console.warn('[AudioFeatures] Danceability failed:', (err as Error).message); } try { const dynComp = essentia.DynamicComplexity(vec); // DynamicComplexity is a measure of loudness variation (0 = flat/compressed, // higher = dynamic). Scale to 0-1 (typical max ~10 dB). result.dynamicComplexity = Math.min(1, dynComp.dynamicComplexity / 10); } catch (err) { console.warn('[AudioFeatures] DynamicComplexity failed:', (err as Error).message); } vec.delete(); // free WASM memory return result; } // ── Public entry point ───────────────────────────────────────────────────── async extractAndPersist(trackId: string): Promise { const pathRes = await this.pgClient.query<{ path: string }>( 'SELECT path FROM tracks WHERE id = $1', [trackId] ); const row = pathRes.rows[0]; if (!row) { console.warn(`[AudioFeatures] track not found: ${trackId}`); return; } const tags = await this.readEmbeddedTags(row.path); // Rough energy proxy from ReplayGain when available (free, no DSP). let tagEnergy: number | null = null; if (tags.replayGainDb !== null) { tagEnergy = Math.min(1, Math.max(0, (tags.replayGainDb + 18) / 21)); } // Only spin up Essentia when embedded tags are missing something useful. const needBpm = tags.bpm === null; const needKey = tags.key === null; // Always run Essentia for energy/danceability since no tag source covers those. const dsq = await this.analyseWithEssentia(row.path, needBpm, needKey); const bpm = tags.bpm ?? dsq.bpm; const key = tags.key ?? dsq.key; const energy = tagEnergy ?? dsq.energy; const danceability = dsq.danceability; const tempo = bpm; await this.pgClient.query( `INSERT INTO track_audio_features (track_id, bpm, key, energy, danceability, tempo) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (track_id) DO UPDATE SET bpm = COALESCE(EXCLUDED.bpm, track_audio_features.bpm), key = COALESCE(EXCLUDED.key, track_audio_features.key), energy = COALESCE(EXCLUDED.energy, track_audio_features.energy), danceability = COALESCE(EXCLUDED.danceability, track_audio_features.danceability), tempo = COALESCE(EXCLUDED.tempo, track_audio_features.tempo)`, [trackId, bpm, key, energy, danceability, tempo] ); const parts: string[] = []; if (bpm) parts.push(`bpm:${bpm.toFixed(1)}${tags.bpm ? '(tag)' : '(dsp)'}`); if (key) parts.push(`key:${key}${tags.key ? '(tag)' : '(dsp)'}`); if (energy !== null) parts.push(`energy:${energy.toFixed(2)}`); if (danceability !== null) parts.push(`dance:${danceability.toFixed(2)}`); console.log( `[AudioFeatures] ${trackId}: ${parts.length ? parts.join(', ') : 'no features extracted'}` ); } }