From bfe22745bcad8e4403c6e47087f917f5db33b3a1 Mon Sep 17 00:00:00 2001 From: kami Date: Sat, 8 Aug 2026 18:43:58 +0400 Subject: [PATCH] feat(discovery): acquire recommendations that keep their names Acquisition ran yt-dlp without --embed-metadata, so every download arrived untagged. The scanner then stored the video id as the title and "Unknown Artist" as the artist, the vetted-candidate tag check rejected the mismatch, and all 18 acquired tracks were hidden and retired. - Pass --embed-metadata so downloads carry real tags. - Let a scan take fallback title/artist from the candidate, for sources that still ship untagged files. - Install Deno alongside yt-dlp: YouTube guards some formats with a JS challenge yt-dlp must execute, and no other runtime is enabled. - Dedupe candidates by artist and title. The (source, external_id) key misses the same song reaching us under two Deezer release ids. Also carries the in-flight discovery work this builds on: the Recommendations page replacing Discover, the discovery source service, and the acquisition spec tests. Co-Authored-By: Claude Opus 5 --- backend/src/app.ts | 34 +++ backend/src/db/migrations.ts | 60 +++++ backend/src/db/schema.sql | 10 +- backend/src/routes/discovery.routes.ts | 48 ++++ backend/src/routes/history.routes.ts | 11 +- backend/src/services/db.service.test.ts | 26 ++ backend/src/services/db.service.ts | 45 +++- backend/src/services/discovery.service.ts | 17 +- docker-compose.yml | 25 +- frontend/src/components/CommandPalette.tsx | 4 +- frontend/src/components/NavRail.tsx | 2 +- frontend/src/components/TopBar.tsx | 2 +- frontend/src/components/VibeAura.tsx | 46 +++- frontend/src/index.css | 92 +++++-- frontend/src/pages/Discover.tsx | 137 ---------- frontend/src/pages/Recommendations.tsx | 239 +++++++++++++++++ frontend/src/router.tsx | 10 +- frontend/src/services/discoveryService.ts | 41 +++ frontend/src/services/historyService.ts | 6 +- workers/Dockerfile | 17 +- workers/src/acquisition-spec.test.ts | 40 +++ workers/src/acquisition.service.ts | 98 +++++-- workers/src/discovery-sources.service.ts | 288 +++++++++++++++++++++ workers/src/index.ts | 56 +++- workers/src/integrations/deezer.client.ts | 75 ++++++ workers/src/integrations/index.ts | 2 +- workers/src/scanner.service.ts | 16 +- workers/src/types.ts | 15 +- 28 files changed, 1241 insertions(+), 221 deletions(-) delete mode 100644 frontend/src/pages/Discover.tsx create mode 100644 frontend/src/pages/Recommendations.tsx create mode 100644 frontend/src/services/discoveryService.ts create mode 100644 workers/src/acquisition-spec.test.ts create mode 100644 workers/src/discovery-sources.service.ts diff --git a/backend/src/app.ts b/backend/src/app.ts index 594abd4..a90385c 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -18,6 +18,11 @@ import vibeSessionsRoutes from './routes/vibe-sessions.routes.js'; import discoveryRoutes from './routes/discovery.routes.js'; import imagesRoutes from './routes/images.routes.js'; import { VibeSessionCoordinator } from './services/vibe-session-coordinator.service.js'; +import { DiscoveryService } from './services/discovery.service.js'; + +// Single-listener deployment: the same placeholder the vibe-session routes use +// when no x-user-id header is supplied. +const DEFAULT_USER_ID = '00000000-0000-0000-0000-000000000000'; export interface AppConfig { port: number; @@ -91,6 +96,34 @@ export async function buildApp(config: AppConfig) { dbService.decayBeliefs().catch(() => {}); dbService.deriveForgottenProfile().catch(() => {}); + // Close the discovery loop without an operator in it. The worker's cron jobs + // generate candidates; this timer is what decides which of them earn a + // probation slot and enqueues the download. Evaluation is cheap SQL against a + // capped batch, and every acquisition gate (relevance, novelty tolerance, + // backlog, per-artist diversity) lives inside evalCandidates. + const discoveryService = new DiscoveryService(dbService); + const DISCOVERY_EVAL_INTERVAL_MS = 30 * 60 * 1000; + const runDiscoveryEval = async () => { + const results = await discoveryService.evalCandidates(DEFAULT_USER_ID); + let enqueued = 0; + for (const result of results) { + if (!result.shouldAcquire) continue; + try { + await jobService.enqueueDiscoveryAcquisition(result.candidateId); + enqueued++; + } catch (err) { + const reason = err instanceof Error ? err.message : 'failed to enqueue acquisition'; + await discoveryService.markEnqueueFailed(result.candidateId, reason); + } + } + if (results.length > 0) { + console.log(`[Discovery] Evaluated ${results.length} candidates, enqueued ${enqueued}`); + } + }; + const discoveryEvalTimer = setInterval(() => { + runDiscoveryEval().catch((e) => console.error('[Discovery] eval failed:', e)); + }, DISCOVERY_EVAL_INTERVAL_MS); + // Ensure the Typesense 'tracks' collection schema exists on boot so that // the first search request doesn't hit a 404. await searchService.ensureCollection(); @@ -176,6 +209,7 @@ export async function buildApp(config: AppConfig) { clearInterval(fusionTimer); clearInterval(decayTimer); clearInterval(forgottenTimer); + clearInterval(discoveryEvalTimer); } catch (err) { fastify.log.error(err); } diff --git a/backend/src/db/migrations.ts b/backend/src/db/migrations.ts index 7ea215e..1794695 100644 --- a/backend/src/db/migrations.ts +++ b/backend/src/db/migrations.ts @@ -759,4 +759,64 @@ export const MIGRATIONS: Migration[] = [ ON CONFLICT (session_id) DO NOTHING; `, }, + { + // Two external candidate strategies, kept as separate source_trust keys so + // meta-learning can compare their retention independently: "artists you + // already play just released something" is a much stronger prior than + // "Last.fm thinks this sounds similar", and the trust values say so. + id: '20260806_external_discovery_sources', + sql: ` + INSERT INTO source_trust (key, trust, description) VALUES + ('new_release', 0.60, + 'New release by an artist already played from the local library.'), + ('similar_recommendation', 0.45, + 'External similarity (Last.fm) seeded from local play history.') + ON CONFLICT (key) DO NOTHING; + `, + }, + { + // play_history is the only durable record of what was listened to and when, + // and it had two holes that only show up when you try to read a year back: + // + // 1. ON DELETE CASCADE meant the gated cleanup sweep silently erased the + // plays of every file it removed. A play happened; deleting the file + // later does not un-happen it. The FK becomes SET NULL and the track's + // identity is denormalised onto the row so it stays readable. + // 2. No duration, so listening time was only ever inferable from the + // track's current duration — itself gone once the file is. + id: '20260806_play_history_durable_facts', + sql: ` + ALTER TABLE play_history ADD COLUMN IF NOT EXISTS listened_ms INTEGER; + ALTER TABLE play_history ADD COLUMN IF NOT EXISTS track_title TEXT; + ALTER TABLE play_history ADD COLUMN IF NOT EXISTS track_artist TEXT; + + DO $$ + DECLARE fk_name TEXT; + BEGIN + SELECT con.conname INTO fk_name + FROM pg_constraint con + JOIN pg_class rel ON rel.oid = con.conrelid + JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = con.conkey[1] + WHERE rel.relname = 'play_history' + AND con.contype = 'f' + AND att.attname = 'track_id' + AND con.confdeltype = 'c' + LIMIT 1; + IF fk_name IS NOT NULL THEN + EXECUTE format('ALTER TABLE play_history DROP CONSTRAINT %I', fk_name); + ALTER TABLE play_history + ADD CONSTRAINT play_history_track_id_fkey + FOREIGN KEY (track_id) REFERENCES tracks(id) ON DELETE SET NULL; + END IF; + END $$; + + -- Backfill identity for rows written before the columns existed. Rows whose + -- track was already cascade-deleted are unrecoverable; this at least stops + -- the bleeding from here on. + UPDATE play_history ph + SET track_title = t.title, track_artist = t.artist + FROM tracks t + WHERE t.id = ph.track_id AND ph.track_title IS NULL; + `, + }, ]; diff --git a/backend/src/db/schema.sql b/backend/src/db/schema.sql index dcbba8b..c3b85ce 100644 --- a/backend/src/db/schema.sql +++ b/backend/src/db/schema.sql @@ -302,12 +302,20 @@ CREATE TABLE IF NOT EXISTS recommendation_batch_track ( -- "Success-Driven Center" rule (a completed play moves the active batch's center) -- and the feedback learning loop. Created with IF NOT EXISTS so it can be -- self-provisioned on databases that predate this schema change. +-- A play is a historical fact: it stays true after the file is gone. Hence +-- ON DELETE SET NULL rather than CASCADE, plus the denormalised title/artist so +-- a row whose track was hard-deleted is still readable. listened_ms is what was +-- actually heard, recorded at play time; tracks.duration is not a substitute +-- because it disappears with the track. CREATE TABLE IF NOT EXISTS play_history ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), user_id UUID NOT NULL, - track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, + track_id UUID REFERENCES tracks(id) ON DELETE SET NULL, batch_id UUID REFERENCES recommendation_batch(id) ON DELETE SET NULL, completed BOOLEAN NOT NULL DEFAULT false, + listened_ms INTEGER, + track_title TEXT, + track_artist TEXT, played_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); diff --git a/backend/src/routes/discovery.routes.ts b/backend/src/routes/discovery.routes.ts index 4f35200..871687f 100644 --- a/backend/src/routes/discovery.routes.ts +++ b/backend/src/routes/discovery.routes.ts @@ -35,6 +35,54 @@ export default async function discoveryRoutes(fastify: FastifyInstance, options: return reply.send({ candidates: res.rows }); }); + /** + * GET /api/discovery/overview — every recommendation and what became of it. + * + * One row per candidate, acquired or not, so a stalled candidate is as + * visible as a retained track. The play/skip counts come from `evidence` + * rather than tracks.play_count because the probation sweep decides on + * exactly those two signals: showing anything else would be showing a number + * that does not drive the outcome. + */ + fastify.get('/discovery/overview', async (request, reply) => { + const query = request.query as { limit?: string }; + const limit = Math.min(Math.max(parseInt(query.limit || '200', 10) || 200, 1), 500); + + const rows = await dbService.pgClient.query( + `SELECT dc.id, dc.source, dc.status, dc.title, dc.artist_credit, dc.notes, + dc.first_seen_at, dc.acquired_at, dc.last_error, dc.acquisition_attempts, + t.id AS track_id, t.title AS track_title, t.artist AS track_artist, + t.probation_status, t.probation_entered_at, + (SELECT COUNT(*)::int FROM evidence e + WHERE e.entity_type = 'track' AND e.entity_id = t.id + AND e.signal = 'playback_completed') AS completed_plays, + (SELECT COUNT(*)::int FROM evidence e + WHERE e.entity_type = 'track' AND e.entity_id = t.id + AND e.signal = 'skip_quick') AS quick_skips + FROM discovery_candidates dc + LEFT JOIN tracks t ON t.id = dc.acquired_track_id + ORDER BY COALESCE(dc.acquired_at, dc.first_seen_at) DESC + LIMIT $1`, + [limit] + ); + + const summary = await dbService.pgClient.query( + `SELECT dc.source, + COUNT(*)::int AS candidates, + COUNT(t.id) FILTER (WHERE t.probation_status = 'probation')::int AS probation, + COUNT(t.id) FILTER (WHERE t.probation_status = 'retained')::int AS retained, + COUNT(t.id) FILTER (WHERE t.probation_status = 'retired')::int AS retired, + COUNT(*) FILTER (WHERE dc.acquired_track_id IS NULL + AND dc.status <> 'candidate')::int AS stalled + FROM discovery_candidates dc + LEFT JOIN tracks t ON t.id = dc.acquired_track_id + GROUP BY dc.source + ORDER BY dc.source` + ); + + return reply.send({ rows: rows.rows, summary: summary.rows }); + }); + /** * POST /api/discovery/eval — evaluate pending candidates for acquisition */ diff --git a/backend/src/routes/history.routes.ts b/backend/src/routes/history.routes.ts index 4aabb98..5f4789f 100644 --- a/backend/src/routes/history.routes.ts +++ b/backend/src/routes/history.routes.ts @@ -6,16 +6,23 @@ export default async function historyRoutes(fastify: FastifyInstance, options: { // Record a playback event. completed defaults to false. fastify.post('/history', async (request, reply) => { - const { trackId, completed, batchId } = request.body as { + const { trackId, completed, batchId, listenedMs } = request.body as { trackId: string; completed?: boolean; batchId?: string; + listenedMs?: number; }; if (!trackId) { return reply.code(400).send({ error: 'trackId is required' }); } + // Clamped rather than rejected: a bogus duration must not cost the caller a + // play record, and a 24h ceiling keeps one bad client from dominating any + // listening-time total. + const listened = typeof listenedMs === 'number' && Number.isFinite(listenedMs) + ? Math.max(0, Math.min(Math.round(listenedMs), 24 * 60 * 60 * 1000)) + : undefined; const userId = (request.headers['x-user-id'] as string) || '00000000-0000-0000-0000-000000000000'; - const historyId = await dbService.recordPlay(userId, trackId, completed === true, batchId); + const historyId = await dbService.recordPlay(userId, trackId, completed === true, batchId, listened); return reply.send({ historyId }); }); diff --git a/backend/src/services/db.service.test.ts b/backend/src/services/db.service.test.ts index 5b08a36..9b017ee 100644 --- a/backend/src/services/db.service.test.ts +++ b/backend/src/services/db.service.test.ts @@ -606,6 +606,32 @@ describe('DbService v2 methods', () => { }); }); + describe('recordPlay durability', () => { + it('copies the track identity onto the row and records what was heard', async () => { + const { service, mockQuery } = makeService(); + mockQuery.mockResolvedValue({ rows: [{ id: 'history-1' }] }); + + await service.recordPlay('user-1', 'track-1', false, undefined, 91_000); + + const [sql, params] = mockQuery.mock.calls[0]; + // A LEFT JOIN, not a plain VALUES: the insert must still happen when the + // track id resolves to nothing. + expect(sql).toContain('LEFT JOIN tracks'); + expect(sql).toContain('t.title'); + expect(sql).toContain('t.artist'); + expect(params).toEqual(['user-1', 'track-1', null, false, 91_000]); + }); + + it('leaves listened_ms null when the caller has no duration to report', async () => { + const { service, mockQuery } = makeService(); + mockQuery.mockResolvedValue({ rows: [{ id: 'history-1' }] }); + + await service.recordPlay('user-1', 'track-1', false); + + expect(mockQuery.mock.calls[0][1][4]).toBeNull(); + }); + }); + describe('upsertClaim', () => { it('calls INSERT ... ON CONFLICT with correct parameters', async () => { const { service, mockQuery } = makeService(); diff --git a/backend/src/services/db.service.ts b/backend/src/services/db.service.ts index 46fb3a0..3ab7a5e 100644 --- a/backend/src/services/db.service.ts +++ b/backend/src/services/db.service.ts @@ -69,6 +69,22 @@ import type { import { AUDIO_PREFERENCE_BUCKETS, FEEDBACK_ACTIONS } from '../db/types.js'; export * from '../db/types.js'; +/** + * Every play_history write goes through this statement. + * + * The title/artist copy is taken in the same round trip via LEFT JOIN, so a + * play stays readable after its track is hard-deleted, and the join never + * suppresses the insert when the track id is unknown. + */ +const PLAY_HISTORY_INSERT = ` + INSERT INTO play_history ( + user_id, track_id, batch_id, completed, listened_ms, track_title, track_artist + ) + SELECT $1::uuid, $2::uuid, $3::uuid, $4::boolean, $5::int, t.title, t.artist + FROM (SELECT 1) AS always + LEFT JOIN tracks t ON t.id = $2::uuid + RETURNING id`; + export class DbService { /** Exposed so route handlers (e.g. settings) can query the database directly. */ readonly pgClient: Pool; @@ -483,11 +499,17 @@ export class DbService { // (permanentlyDeleteTrack, below) and exactly one unlink site in the whole // system (workers/src/cleanup.service.ts). - async recordPlay(userId: string, trackId: string, completed: boolean, batchId?: string): Promise { + async recordPlay( + userId: string, + trackId: string, + completed: boolean, + batchId?: string, + listenedMs?: number, + ): Promise { if (!completed) { const res = await this.pgClient.query( - 'INSERT INTO play_history (user_id, track_id, batch_id, completed) VALUES ($1, $2, $3, $4) RETURNING id', - [userId, trackId, batchId ?? null, false] + PLAY_HISTORY_INSERT, + [userId, trackId, batchId ?? null, false, listenedMs ?? null] ); return res.rows[0].id as string; } @@ -497,8 +519,8 @@ export class DbService { return this.withTransaction(async (client) => { // 1. Record play history const insertRes = await client.query( - 'INSERT INTO play_history (user_id, track_id, batch_id, completed) VALUES ($1, $2, $3, $4) RETURNING id', - [userId, trackId, batchId ?? null, true] + PLAY_HISTORY_INSERT, + [userId, trackId, batchId ?? null, true, listenedMs ?? null] ); const historyId = insertRes.rows[0].id as string; @@ -1936,10 +1958,17 @@ export class DbService { const occurredAt = event.occurred_at?.toISOString?.() ?? new Date().toISOString(); switch (event.type) { case 'completed': + // The Vibe player is the one caller that knows how much was actually + // heard. position_ms is where playback reached; duration_ms is the + // fallback for a client that only reported the track length. await client.query( - `INSERT INTO play_history (user_id, track_id, completed, played_at) - VALUES ($1, $2, true, $3::timestamptz)`, - [event.user_id, event.track_id, occurredAt], + `INSERT INTO play_history ( + user_id, track_id, completed, played_at, listened_ms, track_title, track_artist + ) + SELECT $1::uuid, $2::uuid, true, $3::timestamptz, $4::int, t.title, t.artist + FROM (SELECT 1) AS always + LEFT JOIN tracks t ON t.id = $2::uuid`, + [event.user_id, event.track_id, occurredAt, event.position_ms ?? event.duration_ms ?? null], ); await client.query( `UPDATE tracks diff --git a/backend/src/services/discovery.service.ts b/backend/src/services/discovery.service.ts index dbe40a7..219a9dc 100644 --- a/backend/src/services/discovery.service.ts +++ b/backend/src/services/discovery.service.ts @@ -148,17 +148,20 @@ export class DiscoveryService { const relevance = claimRes.rows[0]?.fused_value ?? 0; const candidateArtistId = claimRes.rows[0]?.object_id; - // Graph walks identify artists, not a legal/downloadable recording. A - // resolver (or a human) must attach a vetted HTTPS source before the - // worker can acquire anything. Do not guess a search query and download - // an arbitrary track under an artist's name. + // A candidate must name what to fetch: either a vetted HTTPS URL, or a + // track-level search phrase the worker resolves against its allow-listed + // hosts. An artist-only candidate (what a bare graph walk produces) names + // neither, and parks here rather than downloading something arbitrary + // under that artist's name. const notes = typeof row.notes === 'string' ? safeJson(row.notes) : row.notes; - const resolvedUrl = (notes as { acquisition?: { url?: unknown } } | null)?.acquisition?.url; - if (typeof resolvedUrl !== 'string' || resolvedUrl.trim() === '') { + const acquisition = (notes as { acquisition?: { url?: unknown; query?: unknown } } | null)?.acquisition; + const hasUrl = typeof acquisition?.url === 'string' && acquisition.url.trim() !== ''; + const hasQuery = typeof acquisition?.query === 'string' && acquisition.query.trim() !== ''; + if (!hasUrl && !hasQuery) { await this.db.pgClient.query( `UPDATE discovery_candidates SET status = 'awaiting_resolution', last_eval_at = NOW(), - last_error = 'no vetted acquisition source attached' + last_error = 'no acquisition url or search query attached' WHERE id = $1`, [row.id] ); diff --git a/docker-compose.yml b/docker-compose.yml index eb5ccf4..cc822ea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,9 +59,10 @@ services: worker: build: context: ./workers - # Keep the downloader absent unless an operator intentionally opts in. + # Enabled deliberately: the auto-seed discovery loop cannot acquire a + # candidate without the downloader present. args: - INSTALL_YTDLP: "false" + INSTALL_YTDLP: "true" restart: unless-stopped network_mode: host environment: @@ -73,13 +74,19 @@ services: DISCOGS_TOKEN: ${DISCOGS_TOKEN} SOCKS_PROXY_URL: ${SOCKS_PROXY_URL} MUSIC_DIR: /music - # System E acquisition is intentionally disabled by default. To enable it - # an operator must build with INSTALL_YTDLP=true and set all three runtime - # gates below; no acquisition happens merely from graph discovery. - # MUZICK_ACQUISITION_ENABLED: "true" - # MUZICK_ACQUISITION_YTDLP_PATH: /usr/bin/yt-dlp - # MUZICK_ACQUISITION_ALLOWED_HOSTS: example.org - # MUZICK_ACQUISITION_DIR: .recommendations + # System E acquisition, enabled. The allow-list is the only thing deciding + # where audio may come from, and it binds search-resolved URLs too. + # `www.youtube.com` is what yt-dlp reports as webpage_url for a ytsearch + # hit. Narrow this list to shrink the blast radius; empty it to stop + # acquisition without a rebuild. + MUZICK_ACQUISITION_ENABLED: "true" + MUZICK_ACQUISITION_YTDLP_PATH: /usr/bin/yt-dlp + MUZICK_ACQUISITION_ALLOWED_HOSTS: www.youtube.com + MUZICK_ACQUISITION_DIR: .recommendations + # + # Candidate generation crons (defaults shown, both daily). + # NEW_RELEASE_CRON: "30 5 * * *" + # RECOMMENDATION_CRON: "30 6 * * *" # Hard-deletion gates for the dislike lifecycle (invariant §C). All three # default to the safe value inside cleanup.service.ts; they are listed here # as documentation and are intentionally left unset. diff --git a/frontend/src/components/CommandPalette.tsx b/frontend/src/components/CommandPalette.tsx index 848fc54..f2d8f33 100644 --- a/frontend/src/components/CommandPalette.tsx +++ b/frontend/src/components/CommandPalette.tsx @@ -32,7 +32,7 @@ const NAV_COMMANDS: CommandItem[] = [ { id: 'nav-artists', label: 'Artists', icon: Users, action: () => {}, keywords: ['artists', 'bands'] }, { id: 'nav-genres', label: 'Genres', icon: Tag, action: () => {}, keywords: ['genres', 'tags', 'categories'] }, { id: 'nav-vibe', label: 'Vibe', description: 'Endless recommendations', icon: Zap, action: () => {}, keywords: ['vibe', 'recommendations', 'radio'] }, - { id: 'nav-discover', label: 'Discover', description: 'Browse by genre', icon: Compass, action: () => {}, keywords: ['discover', 'explore'] }, + { id: 'nav-recommendations', label: 'Found', description: 'Discovered tracks and their fate', icon: Compass, action: () => {}, keywords: ['found', 'discovery', 'recommendations', 'probation', 'new releases'] }, { id: 'nav-quarantine', label: 'Quarantine', icon: ShieldAlert, action: () => {}, keywords: ['quarantine', 'disliked', 'trash'] }, { id: 'nav-jobs', label: 'Jobs', description: 'Background tasks', icon: Terminal, action: () => {}, keywords: ['jobs', 'tasks', 'queue'] }, { id: 'nav-settings', label: 'Settings', icon: Settings, action: () => {}, keywords: ['settings', 'preferences', 'config'] }, @@ -63,7 +63,7 @@ export function CommandPalette({ open, onClose }: CommandPaletteProps) { 'nav-artists': '/artists', 'nav-genres': '/genres', 'nav-vibe': '/vibe', - 'nav-discover': '/discover', + 'nav-recommendations': '/recommendations', 'nav-quarantine': '/quarantine', 'nav-jobs': '/jobs', 'nav-settings': '/settings', diff --git a/frontend/src/components/NavRail.tsx b/frontend/src/components/NavRail.tsx index e2051a1..ba95ad3 100644 --- a/frontend/src/components/NavRail.tsx +++ b/frontend/src/components/NavRail.tsx @@ -35,7 +35,7 @@ const NAV_GROUPS: NavGroup[] = [ label: 'AI', items: [ { to: '/vibe', icon: Zap, label: 'Vibe' }, - { to: '/discover', icon: Compass, label: 'Discover' }, + { to: '/recommendations', icon: Compass, label: 'Found' }, ], }, { diff --git a/frontend/src/components/TopBar.tsx b/frontend/src/components/TopBar.tsx index 01b2b4d..6805ac0 100644 --- a/frontend/src/components/TopBar.tsx +++ b/frontend/src/components/TopBar.tsx @@ -17,8 +17,8 @@ const PAGE_TITLES: Record = { '/albums': 'Albums', '/artists': 'Artists', '/genres': 'Genres', + '/recommendations': 'Found', '/vibe': 'Vibe', - '/discover': 'Discover', '/search': 'Search', '/settings': 'Settings', '/quarantine': 'Quarantine', diff --git a/frontend/src/components/VibeAura.tsx b/frontend/src/components/VibeAura.tsx index d0e29f4..2d380e7 100644 --- a/frontend/src/components/VibeAura.tsx +++ b/frontend/src/components/VibeAura.tsx @@ -22,9 +22,11 @@ export function VibeAura({ profile, ambient = false }: { profile: VibeProfile; a const energy = clamp(profile.energy, 0.5); const novelty = clamp(profile.noveltyHunger ?? profile.explorationCoefficient, 0.3); const { energyLabel, discoveryLabel } = describeProfile(energy, novelty); - // Warm range only (amber → gold → orange). The old 205 base put a teal-blue - // glow in a warm brown room, which read as a different app's accent. - const hue = Math.round(28 + novelty * 30 - energy * 12); + // Hue is discovery alone: ember red when the Vibe stays familiar, gold when it + // reaches. Energy is deliberately absent — it already drives four motion + // channels below, and mixing it in here cancelled half the novelty swing. + // Warm range only; the old 205 base put a teal-blue glow in a warm brown room. + const hue = Math.round(15 + novelty * 40); const style = { '--vibe-hue': String(hue), '--vibe-pulse': `${(4.8 - energy * 2.3).toFixed(2)}s`, @@ -53,13 +55,47 @@ export function VibeAura({ profile, ambient = false }: { profile: VibeProfile; a ); if (ambient) { + // Displacing clean gradients by animated fractal noise is what separates a + // plasma from a blurred blob. baseFrequency is animated in SMIL rather than + // CSS because no CSS property reaches inside an SVG filter primitive. return (
+ > + + + + + + + + +
+
+
+
+
+
); } diff --git a/frontend/src/index.css b/frontend/src/index.css index e4a73c4..9d4220d 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -244,27 +244,87 @@ html { scroll-behavior: smooth; } .vibe-aura-spark { width: 4px; height: 4px; animation: vibe-aura-spark 2.1s ease-in-out infinite; } .vibe-aura-spark-one { left: -4px; top: 5px; } .vibe-aura-spark-two { right: -3px; bottom: 7px; animation-delay: -1s; } -/* Ambient variant: one soft organic blob of warm light behind the page. The - scaled-up creature read as concentric rings at that size, which fought the - content instead of sitting behind it. Gradient goes on the sized element — - no inset-0 fill layer. */ -@keyframes vibe-aura-drift { - from { transform: scale(var(--vibe-scale)) translate3d(-2%, -1%, 0) rotate(-4deg); } - to { transform: scale(calc(var(--vibe-scale) * 1.12)) translate3d(3%, 2%, 0) rotate(5deg); } -} +/* Ambient variant: a plasma in the page background, in the lineage of a media + player visualizer. Three layers — a hot core, a cooler counter-rotating + companion, and a fan of rays — stacked and pushed through one SVG turbulence + displacement pass (see VibeAura.tsx), which is what turns clean gradients + into churning flame. Centering lives on the wrapper, never here: these + keyframes animate `transform` and would overwrite it. */ +/* Anchor lives here, not in utility classes: the offsets differ per breakpoint + and the -50% centering pair sits on the same element, so keeping both in one + place is what stops the two from fighting. Off to the right on purpose — the + reading column stays clear and the plasma bleeds past the track rows. */ .vibe-aura-blob { - border-radius: 46% 54% 62% 38% / 55% 43% 57% 45%; - background: - radial-gradient(closest-side at 40% 36%, hsl(var(--vibe-hue) 92% 60% / .55), transparent 72%), - radial-gradient(closest-side at 66% 64%, hsl(calc(var(--vibe-hue) + 28) 86% 52% / .4), transparent 74%); - filter: blur(44px); - animation: vibe-aura-drift var(--vibe-orbit) ease-in-out infinite alternate; + left: 62%; + top: 52%; } @media (min-width: 640px) { - .vibe-aura-blob { filter: blur(72px); } + .vibe-aura-blob { left: 80%; top: 56%; } +} +.vibe-aura-stack { + position: absolute; + inset: 0; + filter: url(#vibe-plasma) blur(14px); +} +.vibe-aura-layer { + position: absolute; + inset: 0; + border-radius: 50%; + mix-blend-mode: screen; +} +@keyframes vibe-aura-churn { + 0% { transform: scale(var(--vibe-scale)) translate3d(-7%, -4%, 0) rotate(0deg); } + 33% { transform: scale(calc(var(--vibe-scale) * 1.22)) translate3d(6%, -8%, 0) rotate(120deg); } + 66% { transform: scale(calc(var(--vibe-scale) * 0.9)) translate3d(8%, 7%, 0) rotate(240deg); } + 100% { transform: scale(var(--vibe-scale)) translate3d(-7%, -4%, 0) rotate(360deg); } +} +@keyframes vibe-aura-spin { + from { transform: rotate(0deg) scale(var(--vibe-scale)); } + to { transform: rotate(-360deg) scale(var(--vibe-scale)); } +} +@keyframes vibe-aura-breathe { + from { opacity: .55; transform: scale(.86); } + to { opacity: 1; transform: scale(1.12); } +} +/* Hot core — near-white centre falling to the profile hue. */ +.vibe-aura-core-layer { + background: + radial-gradient(closest-side at 47% 44%, hsl(calc(var(--vibe-hue) + 24) 100% 82% / .95), hsl(var(--vibe-hue) 100% 58% / .75) 38%, transparent 70%), + radial-gradient(closest-side at 58% 58%, hsl(calc(var(--vibe-hue) - 12) 100% 50% / .8), transparent 66%); + animation: vibe-aura-churn var(--vibe-orbit) ease-in-out infinite; +} +/* Companion — turns the other way so the two shear against each other. */ +.vibe-aura-swirl-layer { + background: + radial-gradient(closest-side at 62% 40%, hsl(calc(var(--vibe-hue) - 20) 100% 54% / .7), transparent 68%), + radial-gradient(closest-side at 36% 66%, hsl(calc(var(--vibe-hue) - 44) 96% 50% / .58), transparent 72%); + animation: vibe-aura-churn calc(var(--vibe-orbit) * 1.6) ease-in-out infinite reverse; +} +/* Ray fan — the spikes the displacement pass bends into filaments. Masked to a + ring so the centre stays a clean hot core. */ +.vibe-aura-rays-layer { + background: repeating-conic-gradient( + from 0deg, + transparent 0deg 5deg, + hsl(calc(var(--vibe-hue) + 8) 100% 66% / .5) 5deg 7.5deg + ); + -webkit-mask-image: radial-gradient(closest-side, transparent 18%, #000 46%, transparent 82%); + mask-image: radial-gradient(closest-side, transparent 18%, #000 46%, transparent 82%); + animation: vibe-aura-spin calc(var(--vibe-orbit) * 2.4) linear infinite; +} +/* Pulse at the profile's tempo. This lives on the stack, not on the outer + .vibe-aura-blob: that element carries the -translate-x-1/2 -translate-y-1/2 + centering, and an animation setting `transform` overwrites it, which anchors + the plasma by its top-left corner instead of its middle. */ +.vibe-aura-stack { + animation: vibe-aura-breathe var(--vibe-pulse) ease-in-out infinite alternate; +} +@media (min-width: 640px) { + .vibe-aura-stack { filter: url(#vibe-plasma) blur(20px); } } @media (prefers-reduced-motion: reduce) { - .vibe-aura-halo, .vibe-aura-core, .vibe-aura-orbit, .vibe-aura-spark, .vibe-aura-blob { animation: none; } + .vibe-aura-halo, .vibe-aura-core, .vibe-aura-orbit, .vibe-aura-spark, + .vibe-aura-stack, .vibe-aura-layer { animation: none; } } /* ── Component base classes ────────────────────────────────────────────────── */ diff --git a/frontend/src/pages/Discover.tsx b/frontend/src/pages/Discover.tsx deleted file mode 100644 index 00f6d7b..0000000 --- a/frontend/src/pages/Discover.tsx +++ /dev/null @@ -1,137 +0,0 @@ -import { useState } from 'react'; -import { useQuery } from '@tanstack/react-query'; -import { Disc3, Sparkles, AlertCircle } from 'lucide-react'; -import { useNavigate } from '@tanstack/react-router'; -import { genreService } from '../services/genreService'; -import { startVibeSession } from '../services/vibeSession'; -import { TrackRow } from '../components/TrackRow'; -import { PageContainer } from '../components/PageContainer'; -import { PageHeader } from '../components/PageHeader'; -import { EmptyState } from '../components/EmptyState'; -import { SkeletonGrid, SkeletonRows } from '../components/LoadingState'; -import type { Genre, Track } from '../types'; - -export default function Discover() { - const [selected, setSelected] = useState(null); - const [startingVibe, setStartingVibe] = useState(false); - const [vibeError, setVibeError] = useState(null); - const navigate = useNavigate(); - - const genres = useQuery({ - queryKey: ['genres'], - queryFn: () => genreService.listGenres(), - }); - - const genreTracks = useQuery({ - queryKey: ['genre-tracks', selected?.id], - queryFn: () => genreService.getGenreTracks(selected!.id), - enabled: !!selected, - }); - - const startGenreVibe = async () => { - const tracks = genreTracks.data; - if (!tracks || tracks.length === 0 || startingVibe) return; - const seed = tracks[0]; - setStartingVibe(true); - setVibeError(null); - try { - await startVibeSession(seed); - await navigate({ to: '/vibe' }); - } catch { - setVibeError('Could not start a Vibe from this genre. Please try again.'); - } finally { - setStartingVibe(false); - } - }; - - return ( - - - - {genres.isLoading ? ( - - ) : genres.isError ? ( - } title="Couldn't load genres" subtitle="Something went wrong. Try reloading the page." /> - ) : (genres.data ?? []).length === 0 ? ( - } title="No genres yet" subtitle="Genres appear after you scan and enrich your library." /> - ) : ( -
- {(genres.data ?? []).map((genre) => { - const active = selected?.id === genre.id; - return ( - - ); - })} -
- )} - - {selected && ( -
-
-

- {selected.name} - - {(genreTracks.data?.length ?? 0).toLocaleString()} tracks - -

- -
- - {vibeError && ( -

- {vibeError} -

- )} - - {genreTracks.isLoading ? ( - - ) : genreTracks.isError ? ( - } title="Couldn't load tracks" subtitle="Something went wrong. Try reloading the page." /> - ) : (genreTracks.data ?? []).length === 0 ? ( - } title="No tracks in this genre" /> - ) : ( -
- {(genreTracks.data ?? []).map((track, i) => ( - - ))} -
- )} -
- )} - - ); -} diff --git a/frontend/src/pages/Recommendations.tsx b/frontend/src/pages/Recommendations.tsx new file mode 100644 index 0000000..75b070b --- /dev/null +++ b/frontend/src/pages/Recommendations.tsx @@ -0,0 +1,239 @@ +import { useMemo, useState } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { AlertCircle, Compass } from 'lucide-react'; +import { discoveryService } from '../services/discoveryService'; +import type { DiscoveryRow, DiscoverySourceSummary } from '../services/discoveryService'; +import { Badge } from '../components/ethos/Badge'; +import { PageContainer } from '../components/PageContainer'; +import { PageHeader } from '../components/PageHeader'; +import { EmptyState } from '../components/EmptyState'; +import { SkeletonRows } from '../components/LoadingState'; + +// Mirrors the probation sweep in workers/src/index.ts. Shown rather than hidden: +// a track on probation is N plays away from staying, and the operator should be +// able to see exactly how far. If the sweep's thresholds change, change these. +const KEEP_AFTER_PLAYS = 3; +const DROP_AFTER_SKIPS = 3; + +type Filter = 'all' | 'probation' | 'retained' | 'retired' | 'stalled'; + +const FILTERS: { id: Filter; label: string }[] = [ + { id: 'all', label: 'All' }, + { id: 'probation', label: 'On probation' }, + { id: 'retained', label: 'Kept' }, + { id: 'retired', label: 'Removed' }, + { id: 'stalled', label: 'Never arrived' }, +]; + +/** A candidate that was evaluated but never became a track. */ +const isStalled = (row: DiscoveryRow) => !row.track_id && row.status !== 'candidate'; + +function matches(row: DiscoveryRow, filter: Filter): boolean { + if (filter === 'all') return true; + if (filter === 'stalled') return isStalled(row); + return row.probation_status === filter; +} + +function sourceLabel(source: string): string { + if (source === 'new_release') return 'New release'; + if (source === 'similar_recommendation') return 'Similar'; + if (source === 'graph_exploration') return 'Graph'; + return source; +} + +function statusBadge(row: DiscoveryRow) { + if (row.probation_status === 'retained') return Kept; + if (row.probation_status === 'retired') return Removed; + if (row.probation_status === 'probation') return On probation; + if (isStalled(row)) return {row.status.replace(/_/g, ' ')}; + return Queued; +} + +function shortDate(value: string | null): string { + if (!value) return '—'; + return new Date(value).toISOString().slice(0, 10); +} + +/** Where this candidate came from, in one line, without a second request. */ +function provenance(row: DiscoveryRow): string { + const notes = row.notes ?? {}; + if (notes.album) return `from ${notes.album}`; + if (notes.seed_artist) return `because you play ${notes.seed_artist}`; + return sourceLabel(row.source); +} + +function displayName(row: DiscoveryRow): { title: string; artist: string } { + const credited = row.artist_credit?.[0]?.name ?? ''; + return { + title: row.track_title ?? row.title ?? 'Untitled', + artist: row.track_artist ?? credited, + }; +} + +function SummaryStrip({ summary }: { summary: DiscoverySourceSummary[] }) { + if (summary.length === 0) return null; + return ( +
+ {summary.map((s) => ( +
+
+ {sourceLabel(s.source)} + + {s.candidates} found + +
+
+
+
kept
+
{s.retained}
+
+
+
trying
+
{s.probation}
+
+
+
removed
+
{s.retired}
+
+
+
never arrived
+
0 ? 'text-red' : 'text-machine'}>{s.stalled}
+
+
+
+ ))} +
+ ); +} + +export default function Recommendations() { + const [filter, setFilter] = useState('all'); + const { data, isLoading, isError } = useQuery({ + queryKey: ['discovery-overview'], + queryFn: () => discoveryService.overview(), + refetchInterval: 60_000, + }); + + const rows = data?.rows ?? []; + const visible = useMemo(() => rows.filter((row) => matches(row, filter)), [rows, filter]); + + const totals = data?.summary.reduce( + (acc, s) => ({ + candidates: acc.candidates + s.candidates, + retained: acc.retained + s.retained, + probation: acc.probation + s.probation, + }), + { candidates: 0, retained: 0, probation: 0 } + ); + + return ( + + + + {isLoading ? ( + + ) : isError ? ( + } + title="Couldn't load recommendations" + subtitle="Something went wrong. Try reloading the page." + /> + ) : rows.length === 0 ? ( + } + title="Nothing discovered yet" + subtitle="The new-release and similarity scans run daily. Acquisition also has to be enabled on the worker before a candidate can become a track." + /> + ) : ( + <> + + +
+ {FILTERS.map((f) => { + const count = rows.filter((row) => matches(row, f.id)).length; + const active = filter === f.id; + return ( + + ); + })} +
+ + {visible.length === 0 ? ( + } title="Nothing in this state" /> + ) : ( +
    + {visible.map((row) => { + const { title, artist } = displayName(row); + const plays = row.completed_plays ?? 0; + const skips = row.quick_skips ?? 0; + // Stacked below 640px: at that width the name, the badge and the + // mono readout cannot share a row without the title collapsing + // to two characters. + return ( +
  • +
    +
    + {title} + {statusBadge(row)} +
    +
    + {artist ? `${artist} · ` : ''} + {provenance(row)} +
    + {row.last_error && ( +
    + {row.last_error} +
    + )} +
    +
    + {row.probation_status === 'probation' ? ( + + {plays}/{KEEP_AFTER_PLAYS} plays · {skips}/{DROP_AFTER_SKIPS} skips + + ) : row.track_id ? ( + + {plays} plays · {skips} skips + + ) : ( + + {row.acquisition_attempts} attempts + + )} + + {shortDate(row.acquired_at ?? row.first_seen_at)} + +
    +
    • + ); + })} +
+ )} + + )} + + ); +} diff --git a/frontend/src/router.tsx b/frontend/src/router.tsx index 3d89dd5..874dfb3 100644 --- a/frontend/src/router.tsx +++ b/frontend/src/router.tsx @@ -13,7 +13,7 @@ import AlbumDetail from './pages/AlbumDetail'; import Tracks from './pages/Tracks'; import Genres from './pages/Genres'; import Vibe from './pages/Vibe'; -import Discover from './pages/Discover'; +import Recommendations from './pages/Recommendations'; import Search from './pages/Search'; import Settings from './pages/Settings'; import Quarantine from './pages/Quarantine'; @@ -73,10 +73,10 @@ export const vibeRoute = createRoute({ component: Vibe, }); -export const discoverRoute = createRoute({ +export const recommendationsRoute = createRoute({ getParentRoute: () => rootRoute, - path: '/discover', - component: Discover, + path: '/recommendations', + component: Recommendations, }); export const searchRoute = createRoute({ @@ -115,7 +115,7 @@ const routeTree = rootRoute.addChildren([ tracksRoute, genresRoute, vibeRoute, - discoverRoute, + recommendationsRoute, searchRoute, settingsRoute, quarantineRoute, diff --git a/frontend/src/services/discoveryService.ts b/frontend/src/services/discoveryService.ts new file mode 100644 index 0000000..edc7a7a --- /dev/null +++ b/frontend/src/services/discoveryService.ts @@ -0,0 +1,41 @@ +import api from './api'; + +/** One candidate and whatever became of it. `track_id` is null until acquired. */ +export interface DiscoveryRow { + id: string; + source: string; + status: string; + title: string | null; + artist_credit: { name?: string; artist_id?: string }[] | null; + notes: { acquisition?: { query?: string; url?: string }; album?: string; seed_artist?: string; seed_title?: string } | null; + first_seen_at: string; + acquired_at: string | null; + last_error: string | null; + acquisition_attempts: number; + track_id: string | null; + track_title: string | null; + track_artist: string | null; + probation_status: 'probation' | 'retained' | 'retired' | null; + probation_entered_at: string | null; + completed_plays: number | null; + quick_skips: number | null; +} + +export interface DiscoverySourceSummary { + source: string; + candidates: number; + probation: number; + retained: number; + retired: number; + stalled: number; +} + +export const discoveryService = { + // GET /api/discovery/overview -> every candidate plus per-source totals + async overview(): Promise<{ rows: DiscoveryRow[]; summary: DiscoverySourceSummary[] }> { + const res = await api.get<{ rows: DiscoveryRow[]; summary: DiscoverySourceSummary[] }>( + '/discovery/overview' + ); + return res.data; + }, +}; diff --git a/frontend/src/services/historyService.ts b/frontend/src/services/historyService.ts index 9559d7b..e54a92f 100644 --- a/frontend/src/services/historyService.ts +++ b/frontend/src/services/historyService.ts @@ -2,16 +2,18 @@ import api from './api'; import type { FeedbackAction, HistoryEntry } from '../types'; export const historyService = { - // POST /api/history { trackId, completed?, batchId? } -> { historyId } + // POST /api/history { trackId, completed?, batchId?, listenedMs? } -> { historyId } async recordPlay( trackId: string, completed?: boolean, - batchId?: string + batchId?: string, + listenedMs?: number ): Promise<{ historyId: string }> { const res = await api.post<{ historyId: string }>('/history', { trackId, completed, batchId, + listenedMs, }); return res.data; }, diff --git a/workers/Dockerfile b/workers/Dockerfile index 11696af..5c9e0dd 100644 --- a/workers/Dockerfile +++ b/workers/Dockerfile @@ -2,8 +2,21 @@ FROM node:20-slim # yt-dlp is intentionally opt-in. Enabling its image build alone does not make # acquisition live: the worker additionally requires explicit runtime gates. ARG INSTALL_YTDLP=false -RUN apt-get update && apt-get install -y ffmpeg \ - && if [ "$INSTALL_YTDLP" = "true" ]; then apt-get install -y yt-dlp; fi \ +# Debian's yt-dlp package lags years behind and fails on current YouTube, so +# take the self-contained upstream binary instead. It bundles its own Python. +# ponytail: tracks latest; pin the tag here if a release ever breaks the loop. +RUN apt-get update && apt-get install -y ffmpeg curl unzip \ + && if [ "$INSTALL_YTDLP" = "true" ]; then \ + curl -fsSL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux \ + -o /usr/bin/yt-dlp \ + && chmod 755 /usr/bin/yt-dlp \ + && /usr/bin/yt-dlp --version \ + # YouTube guards some formats with an obfuscated JS challenge that + # yt-dlp must execute. Deno is the only runtime it enables by default, + # and without one those videos fail extraction. + && curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- --yes \ + && deno --version; \ + fi \ && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY package*.json ./ diff --git a/workers/src/acquisition-spec.test.ts b/workers/src/acquisition-spec.test.ts new file mode 100644 index 0000000..55a039e --- /dev/null +++ b/workers/src/acquisition-spec.test.ts @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { parseAcquisitionSpec, validateAcquisitionUrl } from './acquisition.service.js'; + +const HOSTS = new Set(['www.youtube.com']); + +test('accepts an allow-listed HTTPS url', () => { + const spec = parseAcquisitionSpec( + { acquisition: { url: 'https://www.youtube.com/watch?v=abc', expectedTitle: 'T', expectedArtist: 'A' } }, + HOSTS + ); + assert.equal(spec.url, 'https://www.youtube.com/watch?v=abc'); + assert.equal(spec.query, undefined); + assert.equal(spec.expectedArtist, 'A'); +}); + +test('accepts a search query and strips newlines that could forge print output', () => { + const spec = parseAcquisitionSpec( + { acquisition: { query: 'Boards of Canada\nRoygbiv', expectedTitle: 'Roygbiv' } }, + HOSTS + ); + assert.equal(spec.query, 'Boards of Canada Roygbiv'); + assert.equal(spec.url, undefined); +}); + +test('rejects a candidate naming neither a url nor a query', () => { + assert.throws(() => parseAcquisitionSpec({ acquisition: { expectedTitle: 'T' } }, HOSTS), /url or a query/); + assert.throws(() => parseAcquisitionSpec({ acquisition: { query: ' ' } }, HOSTS), /url or a query/); + assert.throws(() => parseAcquisitionSpec({}, HOSTS), /no resolved acquisition source/); +}); + +test('search-resolved urls face the same gates as supplied ones', () => { + assert.throws(() => validateAcquisitionUrl('https://evil.example/x', HOSTS), /not allow-listed/); + assert.throws(() => validateAcquisitionUrl('http://www.youtube.com/x', HOSTS), /must use HTTPS/); + assert.throws(() => validateAcquisitionUrl('https://u:p@www.youtube.com/x', HOSTS), /must not contain credentials/); + assert.equal( + validateAcquisitionUrl('https://WWW.YouTube.com/watch?v=1', HOSTS), + 'https://www.youtube.com/watch?v=1' + ); +}); diff --git a/workers/src/acquisition.service.ts b/workers/src/acquisition.service.ts index d1ce0f8..94cb601 100644 --- a/workers/src/acquisition.service.ts +++ b/workers/src/acquisition.service.ts @@ -13,7 +13,10 @@ type CandidateRow = { }; type AcquisitionSpec = { - url: string; + /** A vetted, allow-listed HTTPS URL. Mutually exclusive with `query`. */ + url?: string; + /** A search phrase to resolve into such a URL at download time. */ + query?: string; expectedTitle?: string; expectedArtist?: string; }; @@ -72,18 +75,17 @@ function matchesExpected(actual: string, expected: string | undefined): boolean return left === right || left.includes(right) || right.includes(left); } -/** Parse only a deliberately supplied HTTPS source URL; never accept argv/query strings. */ -export function parseAcquisitionSpec(notes: unknown, allowedHosts: Set): AcquisitionSpec { - const notesValue = typeof notes === 'string' ? JSON.parse(notes) : notes; - const candidate = (notesValue as { acquisition?: unknown } | null)?.acquisition; - if (!candidate || typeof candidate !== 'object') { - throw new Error('candidate has no resolved acquisition source'); - } - const { url, expectedTitle, expectedArtist } = candidate as Record; - if (typeof url !== 'string') throw new Error('acquisition source URL is required'); +/** + * Every URL that reaches the downloader passes through here: HTTPS only, no + * embedded credentials, host on the allow-list. Applied to operator-supplied + * URLs and to search-resolved ones alike, so a search cannot widen the hosts a + * download may come from. + */ +export function validateAcquisitionUrl(raw: unknown, allowedHosts: Set): string { + if (typeof raw !== 'string') throw new Error('acquisition source URL is required'); let parsed: URL; try { - parsed = new URL(url); + parsed = new URL(raw); } catch { throw new Error('acquisition source URL is invalid'); } @@ -92,11 +94,41 @@ export function parseAcquisitionSpec(notes: unknown, allowedHosts: Set): if (!allowedHosts.has(parsed.hostname.toLowerCase())) { throw new Error(`acquisition host is not allow-listed: ${parsed.hostname}`); } - return { - url: parsed.toString(), + return parsed.toString(); +} + +/** + * Parse either a vetted HTTPS URL or a search phrase to resolve into one. + * + * The search form is what automated discovery emits: a graph walk or a + * similarity lookup names an artist and a title, never a downloadable file. A + * resolved search result is trusted no further than an operator-supplied URL — + * same host allow-list, and `matchesExpected` still rejects the download if the + * file's tags disagree with the candidate. + */ +export function parseAcquisitionSpec(notes: unknown, allowedHosts: Set): AcquisitionSpec { + const notesValue = typeof notes === 'string' ? JSON.parse(notes) : notes; + const candidate = (notesValue as { acquisition?: unknown } | null)?.acquisition; + if (!candidate || typeof candidate !== 'object') { + throw new Error('candidate has no resolved acquisition source'); + } + const { url, query, expectedTitle, expectedArtist } = candidate as Record; + const expected = { expectedTitle: typeof expectedTitle === 'string' ? expectedTitle.slice(0, 500) : undefined, expectedArtist: typeof expectedArtist === 'string' ? expectedArtist.slice(0, 500) : undefined, }; + + if (url !== undefined) { + return { url: validateAcquisitionUrl(url, allowedHosts), ...expected }; + } + if (typeof query === 'string' && query.trim() !== '') { + // Newlines would let a crafted candidate forge extra --print output lines + // when the resolver parses stdout. + const cleaned = query.replace(/[\r\n]+/g, ' ').trim().slice(0, 300); + if (cleaned === '') throw new Error('acquisition search query is empty'); + return { query: cleaned, ...expected }; + } + throw new Error('acquisition source needs either a url or a query'); } async function runDownloader(executable: string, args: string[], timeoutMs: number): Promise { @@ -153,6 +185,24 @@ export class AcquisitionService { ); } + /** + * Turn a search phrase into one allow-listed URL, without downloading. + * + * ponytail: first search result only. Ranking alternatives needs a quality + * signal this system does not have, and the tag check downstream already + * rejects a wrong hit. Revisit if mismatches become common. + */ + private async resolveQueryToUrl(query: string): Promise { + const stdout = await runDownloader(this.config.ytDlpPath, [ + '--no-playlist', '--no-progress', '--skip-download', + '--print', 'webpage_url', + '--', `ytsearch1:${query}`, + ], this.config.timeoutMs); + const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + if (lines.length !== 1) throw new Error('search did not resolve to exactly one result'); + return validateAcquisitionUrl(lines[0], this.config.allowedHosts); + } + async acquire(candidateId: string): Promise { const rowResult = await this.pgPool.query( `SELECT id, source, notes, status FROM discovery_candidates WHERE id = $1`, [candidateId] @@ -212,13 +262,19 @@ export class AcquisitionService { // Arguments are fixed by us. The sole untrusted value is the validated URL // and spawn() is invoked with shell:false, so no command interpolation is // possible. One URL / one item is deliberate: playlists are out of scope. + const sourceUrl = spec.url ?? await this.resolveQueryToUrl(spec.query as string); + const outputTemplate = path.join(candidateDir, '%(id)s.%(ext)s'); const stdout = await runDownloader(this.config.ytDlpPath, [ '--no-playlist', '--no-progress', '--restrict-filenames', '--extract-audio', '--audio-format', 'mp3', '--audio-quality', '5', + // Without this the mp3 carries no tags at all, the scanner falls back to + // "