kami
234 lines
9.1 KiB
TypeScript
234 lines
9.1 KiB
TypeScript
/**
|
|
* AudioFeaturesService — two-pass audio feature extraction.
|
|
*
|
|
* Pass 1 (fast): read BPM and key from embedded file tags via music-metadata.
|
|
* Many files tagged by beets / MusicBrainz Picard / Mixed In Key already
|
|
* carry these, so we get the answer for free.
|
|
*
|
|
* Pass 2 (compute): run essentia.js (WebAssembly) on the decoded PCM signal
|
|
* for anything still missing. Audio decoding is done by a ffmpeg subprocess
|
|
* so every format ffmpeg understands (MP3, FLAC, AAC, OGG, …) is supported.
|
|
* Computed features: BPM, key, energy, danceability, dynamic complexity.
|
|
*
|
|
* The two passes are combined via COALESCE so that embedded-tag values are
|
|
* never overwritten once they exist.
|
|
*/
|
|
import { spawn } from 'child_process';
|
|
import mm from 'music-metadata';
|
|
import { Client as PgClient } from 'pg';
|
|
|
|
// Lazy WASM singleton — heavy to load (~2.4 MB), so we initialise once and
|
|
// reuse across all enrichment jobs within the same worker process.
|
|
let essentiaReady: Promise<{ essentia: any }> | null = null;
|
|
|
|
async function getEssentia() {
|
|
if (!essentiaReady) {
|
|
essentiaReady = (async () => {
|
|
const mod = await import('essentia.js');
|
|
// essentia.js v0.1.3: default export contains { Essentia, EssentiaWASM, ... }
|
|
// EssentiaWASM is already the instantiated WASM module (not a function)
|
|
const EssentiaWASM = mod.default?.EssentiaWASM ?? mod.EssentiaWASM;
|
|
const Essentia = mod.default?.Essentia ?? mod.Essentia;
|
|
if (!EssentiaWASM || !Essentia) {
|
|
throw new Error('essentia.js exports missing EssentiaWASM or Essentia');
|
|
}
|
|
const essentia = new Essentia(EssentiaWASM);
|
|
return { essentia };
|
|
})();
|
|
}
|
|
return essentiaReady;
|
|
}
|
|
|
|
/** Decode any audio file to a mono Float32Array at 44100 Hz via ffmpeg. */
|
|
function decodeAudioToFloat32(filePath: string): Promise<Float32Array> {
|
|
return new Promise((resolve, reject) => {
|
|
const chunks: Buffer[] = [];
|
|
const ff = spawn('ffmpeg', [
|
|
'-i', filePath,
|
|
'-vn',
|
|
'-acodec', 'pcm_f32le',
|
|
'-ar', '44100',
|
|
'-ac', '1',
|
|
'-f', 'f32le',
|
|
'pipe:1',
|
|
], { stdio: ['ignore', 'pipe', 'ignore'] });
|
|
|
|
ff.stdout.on('data', (chunk: Buffer) => chunks.push(chunk));
|
|
ff.stdout.on('end', () => {
|
|
const buf = Buffer.concat(chunks);
|
|
resolve(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
|
|
});
|
|
ff.on('error', reject);
|
|
ff.on('close', (code) => {
|
|
if (code !== 0 && chunks.length === 0) {
|
|
reject(new Error(`ffmpeg exited with code ${code} for ${filePath}`));
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
export interface AudioFeatures {
|
|
bpm: number | null;
|
|
key: string | null;
|
|
energy: number | null;
|
|
danceability: number | null;
|
|
dynamicComplexity: number | null;
|
|
}
|
|
|
|
export class AudioFeaturesService {
|
|
constructor(private pgClient: PgClient) {}
|
|
|
|
async ensureSchema(): Promise<void> {
|
|
await this.pgClient.query(
|
|
`CREATE TABLE IF NOT EXISTS track_audio_features (
|
|
track_id UUID PRIMARY KEY REFERENCES tracks(id) ON DELETE CASCADE,
|
|
bpm REAL,
|
|
key TEXT,
|
|
energy REAL,
|
|
danceability REAL,
|
|
valence REAL,
|
|
acousticness REAL,
|
|
instrumentalness REAL,
|
|
liveness REAL,
|
|
valence_score REAL,
|
|
tempo REAL
|
|
)`
|
|
);
|
|
}
|
|
|
|
// ── Pass 1: embedded tags ──────────────────────────────────────────────────
|
|
|
|
private async readEmbeddedTags(filePath: string): Promise<{ bpm: number | null; key: string | null; replayGainDb: number | null }> {
|
|
try {
|
|
const { common } = await mm.parseFile(filePath, { duration: false });
|
|
const bpm = common.bpm && Number.isFinite(common.bpm) && common.bpm > 0 ? common.bpm : null;
|
|
const key = common.key?.trim() || null;
|
|
const rgRaw = (common as any).replaygain_track_gain;
|
|
let replayGainDb: number | null = null;
|
|
if (rgRaw != null) {
|
|
const val = typeof rgRaw === 'object' ? rgRaw.dB : Number(String(rgRaw).replace(/[^\d.-]/g, ''));
|
|
if (Number.isFinite(val)) replayGainDb = val;
|
|
}
|
|
return { bpm, key, replayGainDb };
|
|
} catch {
|
|
return { bpm: null, key: null, replayGainDb: null };
|
|
}
|
|
}
|
|
|
|
// ── Pass 2: essentia.js DSP ────────────────────────────────────────────────
|
|
|
|
private async analyseWithEssentia(filePath: string, needBpm: boolean, needKey: boolean): Promise<AudioFeatures> {
|
|
const result: AudioFeatures = { bpm: null, key: null, energy: null, danceability: null, dynamicComplexity: null };
|
|
|
|
const { essentia } = await getEssentia();
|
|
const signal = await decodeAudioToFloat32(filePath);
|
|
if (signal.length === 0) return result;
|
|
|
|
const vec = essentia.arrayToVector(signal);
|
|
|
|
if (needBpm) {
|
|
try {
|
|
const rhythm = essentia.RhythmExtractor2013(vec);
|
|
if (rhythm.bpm > 0) result.bpm = Math.round(rhythm.bpm * 10) / 10;
|
|
} catch (err) {
|
|
console.warn('[AudioFeatures] RhythmExtractor2013 failed:', (err as Error).message);
|
|
}
|
|
}
|
|
|
|
if (needKey) {
|
|
try {
|
|
const keyResult = essentia.KeyExtractor(vec);
|
|
if (keyResult.key) result.key = `${keyResult.key} ${keyResult.scale}`;
|
|
} catch (err) {
|
|
console.warn('[AudioFeatures] KeyExtractor failed:', (err as Error).message);
|
|
}
|
|
}
|
|
|
|
// Energy and danceability are only available from signal analysis — always compute.
|
|
try {
|
|
const energy = essentia.Energy(vec);
|
|
// Essentia Energy returns the sum of squared samples, normalised to [0,1]
|
|
// by dividing by signal length. Map to a sensible 0-1 display value.
|
|
const rawEnergy = energy.energy / signal.length;
|
|
result.energy = Math.min(1, rawEnergy * 1000); // typical values << 0.001
|
|
} catch (err) {
|
|
console.warn('[AudioFeatures] Energy failed:', (err as Error).message);
|
|
}
|
|
|
|
try {
|
|
const dance = essentia.Danceability(vec);
|
|
// Danceability output range is 0-3; normalise to 0-1.
|
|
result.danceability = Math.min(1, dance.danceability / 3);
|
|
} catch (err) {
|
|
console.warn('[AudioFeatures] Danceability failed:', (err as Error).message);
|
|
}
|
|
|
|
try {
|
|
const dynComp = essentia.DynamicComplexity(vec);
|
|
// DynamicComplexity is a measure of loudness variation (0 = flat/compressed,
|
|
// higher = dynamic). Scale to 0-1 (typical max ~10 dB).
|
|
result.dynamicComplexity = Math.min(1, dynComp.dynamicComplexity / 10);
|
|
} catch (err) {
|
|
console.warn('[AudioFeatures] DynamicComplexity failed:', (err as Error).message);
|
|
}
|
|
|
|
vec.delete(); // free WASM memory
|
|
return result;
|
|
}
|
|
|
|
// ── Public entry point ─────────────────────────────────────────────────────
|
|
|
|
async extractAndPersist(trackId: string): Promise<void> {
|
|
const pathRes = await this.pgClient.query<{ path: string }>(
|
|
'SELECT path FROM tracks WHERE id = $1',
|
|
[trackId]
|
|
);
|
|
const row = pathRes.rows[0];
|
|
if (!row) {
|
|
console.warn(`[AudioFeatures] track not found: ${trackId}`);
|
|
return;
|
|
}
|
|
|
|
const tags = await this.readEmbeddedTags(row.path);
|
|
|
|
// Rough energy proxy from ReplayGain when available (free, no DSP).
|
|
let tagEnergy: number | null = null;
|
|
if (tags.replayGainDb !== null) {
|
|
tagEnergy = Math.min(1, Math.max(0, (tags.replayGainDb + 18) / 21));
|
|
}
|
|
|
|
// Only spin up Essentia when embedded tags are missing something useful.
|
|
const needBpm = tags.bpm === null;
|
|
const needKey = tags.key === null;
|
|
// Always run Essentia for energy/danceability since no tag source covers those.
|
|
const dsq = await this.analyseWithEssentia(row.path, needBpm, needKey);
|
|
|
|
const bpm = tags.bpm ?? dsq.bpm;
|
|
const key = tags.key ?? dsq.key;
|
|
const energy = tagEnergy ?? dsq.energy;
|
|
const danceability = dsq.danceability;
|
|
const tempo = bpm;
|
|
|
|
await this.pgClient.query(
|
|
`INSERT INTO track_audio_features (track_id, bpm, key, energy, danceability, tempo)
|
|
VALUES ($1, $2, $3, $4, $5, $6)
|
|
ON CONFLICT (track_id) DO UPDATE SET
|
|
bpm = COALESCE(EXCLUDED.bpm, track_audio_features.bpm),
|
|
key = COALESCE(EXCLUDED.key, track_audio_features.key),
|
|
energy = COALESCE(EXCLUDED.energy, track_audio_features.energy),
|
|
danceability = COALESCE(EXCLUDED.danceability, track_audio_features.danceability),
|
|
tempo = COALESCE(EXCLUDED.tempo, track_audio_features.tempo)`,
|
|
[trackId, bpm, key, energy, danceability, tempo]
|
|
);
|
|
|
|
const parts: string[] = [];
|
|
if (bpm) parts.push(`bpm:${bpm.toFixed(1)}${tags.bpm ? '(tag)' : '(dsp)'}`);
|
|
if (key) parts.push(`key:${key}${tags.key ? '(tag)' : '(dsp)'}`);
|
|
if (energy !== null) parts.push(`energy:${energy.toFixed(2)}`);
|
|
if (danceability !== null) parts.push(`dance:${danceability.toFixed(2)}`);
|
|
console.log(
|
|
`[AudioFeatures] ${trackId}: ${parts.length ? parts.join(', ') : 'no features extracted'}`
|
|
);
|
|
}
|
|
}
|