// HTTP base shared by every external-integration client. // // Provides: // - configurable User-Agent (REQUIRED by MusicBrainz, format: // `muzick/0.1 ( )`) // - SOCKS5 proxy support via SOCKS_PROXY_URL env var // (e.g. `socks5://127.0.0.1:10808`; no proxy when unset/empty) // - exponential backoff with jitter on 429 / 5xx / network errors, // honouring a `Retry-After` header when present // - a simple per-host rate limit (min interval between requests to the same // host) so e.g. MusicBrainz's <=1 req/sec rule is respected // - typed JSON parsing via generics and a typed error on persistent failure. // // Why not global fetch? Node's built-in fetch doesn't support proxy agents. // When SOCKS_PROXY_URL is set we build a proxy-aware fetcher via // socks-proxy-agent + native https.request. Otherwise we use the global fetch // (faster path, no overhead). import * as http from 'http'; import * as https from 'https'; import { SocksProxyAgent } from 'socks-proxy-agent'; /** Error thrown when a request ultimately fails (non-2xx after all retries). */ export class HttpError extends Error { constructor( message: string, readonly status: number, readonly url: string, readonly body?: string ) { super(message); this.name = 'HttpError'; } } // ── Proxy-aware fetch factory ──────────────────────────────────────────────── const SOCKS_PROXY_URL = process.env.SOCKS_PROXY_URL || ''; /** * Return a `fetch`-compatible function that routes through the configured * SOCKS5 proxy (if any), or the bare global fetch when no proxy is needed. */ function createFetch(): typeof fetch { if (!SOCKS_PROXY_URL) return globalThis.fetch.bind(globalThis); const agent = new SocksProxyAgent(SOCKS_PROXY_URL); // Internal helper: issue a single request (no redirect following). function requestOnce( url: string, method: string, headers: Record | undefined, body: BodyInit | undefined, signal: AbortSignal | undefined, ): Promise { const urlObj = new URL(url); const mod = urlObj.protocol === 'https:' ? https : http; return new Promise((resolve, reject) => { const req = mod.request(url, { agent, method, headers, signal }, (res) => { const chunks: Buffer[] = []; res.on('data', (chunk: Buffer) => chunks.push(chunk)); res.on('end', () => { resolve( new Response(Buffer.concat(chunks), { status: res.statusCode, statusText: res.statusMessage || '', headers: new Headers( Object.entries(res.headers).reduce( (acc, [k, v]) => { acc[k] = Array.isArray(v) ? v.join(', ') : String(v ?? ''); return acc; }, {} as Record, ) ), }) ); }); }); req.on('error', reject); if (body) req.write(body); req.end(); }); } const MAX_REDIRECTS = 5; return async (input, init): Promise => { let url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; const method = init?.method || 'GET'; const headers = init?.headers as Record | undefined; const body = init?.body ?? undefined; const signal = init?.signal ?? undefined; for (let i = 0; i <= MAX_REDIRECTS; i++) { const res = await requestOnce(url, method, headers, body, signal); const code = res.status; // Follow redirect (301, 302, 307, 308) — max 5 hops. if (code === 301 || code === 302 || code === 307 || code === 308) { const location = res.headers.get('location'); if (!location) return res; url = new URL(location, url).href; continue; } return res; } // Too many redirects. return new Response(null, { status: 508, statusText: 'Loop Detected' }); }; } // Singleton fetcher — created once at module load. const fetchFn = createFetch(); export interface RequestJsonOptions { /** Full User-Agent header value. Required for MusicBrainz; recommended always. */ userAgent: string; /** Minimum interval (ms) between requests to the same host. Default 1000. */ minIntervalMs?: number; /** Max retry attempts on transient failures. Default 5. */ maxRetries?: number; /** Base backoff delay (ms). Default 500. */ baseDelayMs?: number; /** Cap on a single backoff delay (ms). Default 20000. */ maxDelayMs?: number; /** Extra request headers (e.g. Accept). */ headers?: Record; /** Per-request timeout (ms). Default 15000. */ timeoutMs?: number; } // Per-host rate-limit state. Module-scoped so all clients sharing a host // coordinate automatically. // // `tail` is the promise chain for a host: every throttle() call links onto the // host's current tail, so slot acquisition is strictly serialized. Without this // chain the old implementation was a TOCTOU race — under the worker's // `concurrency: 10`, ten jobs read the same `lastRequestAt`, slept the same // duration and fired in the same tick, i.e. ~10 req/s against MusicBrainz's // 1 req/s policy. Serialization is deliberately PER HOST (not global) so a slow // MusicBrainz queue cannot starve Last.fm, Discogs, cover art, etc. interface HostLimiter { /** Timestamp of the last granted request slot. */ lastRequestAt: number; /** Tail of the serialization chain; resolves when the previous waiter is done. */ tail: Promise; } const hostLimiters = new Map(); const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); function hostOf(url: string): string { try { return new URL(url).host; } catch { return url; } } /** * Acquire this host's next request slot, blocking until at least * `minIntervalMs` has elapsed since the previously granted slot. * * Concurrent callers for the same host queue strictly in arrival order: each * one appends to `limiter.tail` and only computes its wait once every earlier * waiter has already claimed its timestamp, so N concurrent callers are spaced * `minIntervalMs` apart rather than all firing at once. */ function throttle(host: string, minIntervalMs: number): Promise { let limiter = hostLimiters.get(host); if (!limiter) { limiter = { lastRequestAt: 0, tail: Promise.resolve() }; hostLimiters.set(host, limiter); } const lim = limiter; // Link onto the tail. The critical section (read lastRequestAt → sleep → // write lastRequestAt) runs only after the previous waiter finished it. const slot = lim.tail.then(async () => { const wait = minIntervalMs - (Date.now() - lim.lastRequestAt); if (wait > 0) await delay(wait); lim.lastRequestAt = Date.now(); }); // The next caller waits for this slot. Swallow rejections on the chain itself // so one failure can never poison the queue for subsequent requests. lim.tail = slot.catch(() => {}); return slot; } /** * Test-only handle on the per-host throttle, so its serialization can be * asserted without issuing real network requests. Not used in production code. */ export const __throttleForTest = throttle; /** Parse a Retry-After header (delta-seconds or HTTP date) into ms, or null. */ function parseRetryAfter(value: string | null): number | null { if (!value) return null; const secs = Number(value); if (Number.isFinite(secs)) return Math.max(0, secs * 1000); const when = Date.parse(value); if (!Number.isNaN(when)) return Math.max(0, when - Date.now()); return null; } /** Exponential backoff with full jitter, capped at maxDelayMs. */ function backoffDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number { const exp = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt); return Math.floor(Math.random() * exp); } /** * Fetch a URL and parse the JSON body as `T`. Retries transient failures with * backoff + jitter; enforces the per-host rate limit. Throws `HttpError` on a * persistent non-2xx response and rethrows the last network error if all * attempts fail. */ export async function requestJson( url: string, opts: RequestJsonOptions ): Promise { const { userAgent, minIntervalMs = 1000, maxRetries = 5, baseDelayMs = 500, maxDelayMs = 20000, headers = {}, timeoutMs = 15000, } = opts; const host = hostOf(url); let lastError: unknown; for (let attempt = 0; attempt <= maxRetries; attempt++) { await throttle(host, minIntervalMs); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { const res = await fetchFn(url, { headers: { 'User-Agent': userAgent, Accept: 'application/json', ...headers, }, signal: controller.signal, }); // Retry on 429 (rate limited) and 5xx (transient server errors). if (res.status === 429 || res.status >= 500) { if (attempt < maxRetries) { const retryAfter = parseRetryAfter(res.headers.get('retry-after')); await delay(retryAfter ?? backoffDelay(attempt, baseDelayMs, maxDelayMs)); continue; } const body = await res.text().catch(() => undefined); throw new HttpError( `Request failed after ${maxRetries} retries: ${res.status} ${res.statusText}`, res.status, url, body ); } if (!res.ok) { const body = await res.text().catch(() => undefined); throw new HttpError( `Request failed: ${res.status} ${res.statusText}`, res.status, url, body ); } return (await res.json()) as T; } catch (err) { // Non-retryable application errors propagate immediately. if (err instanceof HttpError) throw err; // Network / abort errors: retry with backoff, otherwise rethrow. lastError = err; if (attempt < maxRetries) { await delay(backoffDelay(attempt, baseDelayMs, maxDelayMs)); continue; } throw err; } finally { clearTimeout(timer); } } // Unreachable in practice; satisfies the type checker. throw lastError instanceof Error ? lastError : new Error('requestJson: exhausted retries'); } /** Build the MusicBrainz-compliant User-Agent string. */ export function buildUserAgent(contact: string): string { const c = contact && contact.trim() !== '' ? contact.trim() : 'no-contact-configured'; return `muzick/0.1 ( ${c} )`; }