initial state: muzick music player + recommendation engine
This commit is contained in:
kami
@@ -0,0 +1,233 @@
|
||||
/**
|
||||
* AudioFeaturesService — two-pass audio feature extraction.
|
||||
*
|
||||
* Pass 1 (fast): read BPM and key from embedded file tags via music-metadata.
|
||||
* Many files tagged by beets / MusicBrainz Picard / Mixed In Key already
|
||||
* carry these, so we get the answer for free.
|
||||
*
|
||||
* Pass 2 (compute): run essentia.js (WebAssembly) on the decoded PCM signal
|
||||
* for anything still missing. Audio decoding is done by a ffmpeg subprocess
|
||||
* so every format ffmpeg understands (MP3, FLAC, AAC, OGG, …) is supported.
|
||||
* Computed features: BPM, key, energy, danceability, dynamic complexity.
|
||||
*
|
||||
* The two passes are combined via COALESCE so that embedded-tag values are
|
||||
* never overwritten once they exist.
|
||||
*/
|
||||
import { spawn } from 'child_process';
|
||||
import mm from 'music-metadata';
|
||||
import { Client as PgClient } from 'pg';
|
||||
|
||||
// Lazy WASM singleton — heavy to load (~2.4 MB), so we initialise once and
|
||||
// reuse across all enrichment jobs within the same worker process.
|
||||
let essentiaReady: Promise<{ essentia: any }> | null = null;
|
||||
|
||||
async function getEssentia() {
|
||||
if (!essentiaReady) {
|
||||
essentiaReady = (async () => {
|
||||
const mod = await import('essentia.js');
|
||||
// essentia.js v0.1.3: default export contains { Essentia, EssentiaWASM, ... }
|
||||
// EssentiaWASM is already the instantiated WASM module (not a function)
|
||||
const EssentiaWASM = mod.default?.EssentiaWASM ?? mod.EssentiaWASM;
|
||||
const Essentia = mod.default?.Essentia ?? mod.Essentia;
|
||||
if (!EssentiaWASM || !Essentia) {
|
||||
throw new Error('essentia.js exports missing EssentiaWASM or Essentia');
|
||||
}
|
||||
const essentia = new Essentia(EssentiaWASM);
|
||||
return { essentia };
|
||||
})();
|
||||
}
|
||||
return essentiaReady;
|
||||
}
|
||||
|
||||
/** Decode any audio file to a mono Float32Array at 44100 Hz via ffmpeg. */
|
||||
function decodeAudioToFloat32(filePath: string): Promise<Float32Array> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
const ff = spawn('ffmpeg', [
|
||||
'-i', filePath,
|
||||
'-vn',
|
||||
'-acodec', 'pcm_f32le',
|
||||
'-ar', '44100',
|
||||
'-ac', '1',
|
||||
'-f', 'f32le',
|
||||
'pipe:1',
|
||||
], { stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
|
||||
ff.stdout.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
ff.stdout.on('end', () => {
|
||||
const buf = Buffer.concat(chunks);
|
||||
resolve(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
|
||||
});
|
||||
ff.on('error', reject);
|
||||
ff.on('close', (code) => {
|
||||
if (code !== 0 && chunks.length === 0) {
|
||||
reject(new Error(`ffmpeg exited with code ${code} for ${filePath}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export interface AudioFeatures {
|
||||
bpm: number | null;
|
||||
key: string | null;
|
||||
energy: number | null;
|
||||
danceability: number | null;
|
||||
dynamicComplexity: number | null;
|
||||
}
|
||||
|
||||
export class AudioFeaturesService {
|
||||
constructor(private pgClient: PgClient) {}
|
||||
|
||||
async ensureSchema(): Promise<void> {
|
||||
await this.pgClient.query(
|
||||
`CREATE TABLE IF NOT EXISTS track_audio_features (
|
||||
track_id UUID PRIMARY KEY REFERENCES tracks(id) ON DELETE CASCADE,
|
||||
bpm REAL,
|
||||
key TEXT,
|
||||
energy REAL,
|
||||
danceability REAL,
|
||||
valence REAL,
|
||||
acousticness REAL,
|
||||
instrumentalness REAL,
|
||||
liveness REAL,
|
||||
valence_score REAL,
|
||||
tempo REAL
|
||||
)`
|
||||
);
|
||||
}
|
||||
|
||||
// ── Pass 1: embedded tags ──────────────────────────────────────────────────
|
||||
|
||||
private async readEmbeddedTags(filePath: string): Promise<{ bpm: number | null; key: string | null; replayGainDb: number | null }> {
|
||||
try {
|
||||
const { common } = await mm.parseFile(filePath, { duration: false });
|
||||
const bpm = common.bpm && Number.isFinite(common.bpm) && common.bpm > 0 ? common.bpm : null;
|
||||
const key = common.key?.trim() || null;
|
||||
const rgRaw = (common as any).replaygain_track_gain;
|
||||
let replayGainDb: number | null = null;
|
||||
if (rgRaw != null) {
|
||||
const val = typeof rgRaw === 'object' ? rgRaw.dB : Number(String(rgRaw).replace(/[^\d.-]/g, ''));
|
||||
if (Number.isFinite(val)) replayGainDb = val;
|
||||
}
|
||||
return { bpm, key, replayGainDb };
|
||||
} catch {
|
||||
return { bpm: null, key: null, replayGainDb: null };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Pass 2: essentia.js DSP ────────────────────────────────────────────────
|
||||
|
||||
private async analyseWithEssentia(filePath: string, needBpm: boolean, needKey: boolean): Promise<AudioFeatures> {
|
||||
const result: AudioFeatures = { bpm: null, key: null, energy: null, danceability: null, dynamicComplexity: null };
|
||||
|
||||
const { essentia } = await getEssentia();
|
||||
const signal = await decodeAudioToFloat32(filePath);
|
||||
if (signal.length === 0) return result;
|
||||
|
||||
const vec = essentia.arrayToVector(signal);
|
||||
|
||||
if (needBpm) {
|
||||
try {
|
||||
const rhythm = essentia.RhythmExtractor2013(vec);
|
||||
if (rhythm.bpm > 0) result.bpm = Math.round(rhythm.bpm * 10) / 10;
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] RhythmExtractor2013 failed:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
if (needKey) {
|
||||
try {
|
||||
const keyResult = essentia.KeyExtractor(vec);
|
||||
if (keyResult.key) result.key = `${keyResult.key} ${keyResult.scale}`;
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] KeyExtractor failed:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// Energy and danceability are only available from signal analysis — always compute.
|
||||
try {
|
||||
const energy = essentia.Energy(vec);
|
||||
// Essentia Energy returns the sum of squared samples, normalised to [0,1]
|
||||
// by dividing by signal length. Map to a sensible 0-1 display value.
|
||||
const rawEnergy = energy.energy / signal.length;
|
||||
result.energy = Math.min(1, rawEnergy * 1000); // typical values << 0.001
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] Energy failed:', (err as Error).message);
|
||||
}
|
||||
|
||||
try {
|
||||
const dance = essentia.Danceability(vec);
|
||||
// Danceability output range is 0-3; normalise to 0-1.
|
||||
result.danceability = Math.min(1, dance.danceability / 3);
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] Danceability failed:', (err as Error).message);
|
||||
}
|
||||
|
||||
try {
|
||||
const dynComp = essentia.DynamicComplexity(vec);
|
||||
// DynamicComplexity is a measure of loudness variation (0 = flat/compressed,
|
||||
// higher = dynamic). Scale to 0-1 (typical max ~10 dB).
|
||||
result.dynamicComplexity = Math.min(1, dynComp.dynamicComplexity / 10);
|
||||
} catch (err) {
|
||||
console.warn('[AudioFeatures] DynamicComplexity failed:', (err as Error).message);
|
||||
}
|
||||
|
||||
vec.delete(); // free WASM memory
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Public entry point ─────────────────────────────────────────────────────
|
||||
|
||||
async extractAndPersist(trackId: string): Promise<void> {
|
||||
const pathRes = await this.pgClient.query<{ path: string }>(
|
||||
'SELECT path FROM tracks WHERE id = $1',
|
||||
[trackId]
|
||||
);
|
||||
const row = pathRes.rows[0];
|
||||
if (!row) {
|
||||
console.warn(`[AudioFeatures] track not found: ${trackId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const tags = await this.readEmbeddedTags(row.path);
|
||||
|
||||
// Rough energy proxy from ReplayGain when available (free, no DSP).
|
||||
let tagEnergy: number | null = null;
|
||||
if (tags.replayGainDb !== null) {
|
||||
tagEnergy = Math.min(1, Math.max(0, (tags.replayGainDb + 18) / 21));
|
||||
}
|
||||
|
||||
// Only spin up Essentia when embedded tags are missing something useful.
|
||||
const needBpm = tags.bpm === null;
|
||||
const needKey = tags.key === null;
|
||||
// Always run Essentia for energy/danceability since no tag source covers those.
|
||||
const dsq = await this.analyseWithEssentia(row.path, needBpm, needKey);
|
||||
|
||||
const bpm = tags.bpm ?? dsq.bpm;
|
||||
const key = tags.key ?? dsq.key;
|
||||
const energy = tagEnergy ?? dsq.energy;
|
||||
const danceability = dsq.danceability;
|
||||
const tempo = bpm;
|
||||
|
||||
await this.pgClient.query(
|
||||
`INSERT INTO track_audio_features (track_id, bpm, key, energy, danceability, tempo)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (track_id) DO UPDATE SET
|
||||
bpm = COALESCE(EXCLUDED.bpm, track_audio_features.bpm),
|
||||
key = COALESCE(EXCLUDED.key, track_audio_features.key),
|
||||
energy = COALESCE(EXCLUDED.energy, track_audio_features.energy),
|
||||
danceability = COALESCE(EXCLUDED.danceability, track_audio_features.danceability),
|
||||
tempo = COALESCE(EXCLUDED.tempo, track_audio_features.tempo)`,
|
||||
[trackId, bpm, key, energy, danceability, tempo]
|
||||
);
|
||||
|
||||
const parts: string[] = [];
|
||||
if (bpm) parts.push(`bpm:${bpm.toFixed(1)}${tags.bpm ? '(tag)' : '(dsp)'}`);
|
||||
if (key) parts.push(`key:${key}${tags.key ? '(tag)' : '(dsp)'}`);
|
||||
if (energy !== null) parts.push(`energy:${energy.toFixed(2)}`);
|
||||
if (danceability !== null) parts.push(`dance:${danceability.toFixed(2)}`);
|
||||
console.log(
|
||||
`[AudioFeatures] ${trackId}: ${parts.length ? parts.join(', ') : 'no features extracted'}`
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user