feat(discovery): acquire recommendations that keep their names

Acquisition ran yt-dlp without --embed-metadata, so every download
arrived untagged. The scanner then stored the video id as the title and
"Unknown Artist" as the artist, the vetted-candidate tag check rejected
the mismatch, and all 18 acquired tracks were hidden and retired.

- Pass --embed-metadata so downloads carry real tags.
- Let a scan take fallback title/artist from the candidate, for sources
  that still ship untagged files.
- Install Deno alongside yt-dlp: YouTube guards some formats with a JS
  challenge yt-dlp must execute, and no other runtime is enabled.
- Dedupe candidates by artist and title. The (source, external_id) key
  misses the same song reaching us under two Deezer release ids.

Also carries the in-flight discovery work this builds on: the
Recommendations page replacing Discover, the discovery source service,
and the acquisition spec tests.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

workers/Dockerfile

@@ -2,8 +2,21 @@ FROM node:20-slim
 # yt-dlp is intentionally opt-in. Enabling its image build alone does not make
 # acquisition live: the worker additionally requires explicit runtime gates.
 ARG INSTALL_YTDLP=false
-RUN apt-get update && apt-get install -y ffmpeg \
-    && if [ "$INSTALL_YTDLP" = "true" ]; then apt-get install -y yt-dlp; fi \
+# Debian's yt-dlp package lags years behind and fails on current YouTube, so
+# take the self-contained upstream binary instead. It bundles its own Python.
+# ponytail: tracks latest; pin the tag here if a release ever breaks the loop.
+RUN apt-get update && apt-get install -y ffmpeg curl unzip \
+    && if [ "$INSTALL_YTDLP" = "true" ]; then \
+         curl -fsSL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux \
+           -o /usr/bin/yt-dlp \
+         && chmod 755 /usr/bin/yt-dlp \
+         && /usr/bin/yt-dlp --version \
+         # YouTube guards some formats with an obfuscated JS challenge that
+         # yt-dlp must execute. Deno is the only runtime it enables by default,
+         # and without one those videos fail extraction.
+         && curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- --yes \
+         && deno --version; \
+       fi \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /app
 COPY package*.json ./

workers/src/acquisition-spec.test.ts

@@ -0,0 +1,40 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { parseAcquisitionSpec, validateAcquisitionUrl } from './acquisition.service.js';
+
+const HOSTS = new Set(['www.youtube.com']);
+
+test('accepts an allow-listed HTTPS url', () => {
+  const spec = parseAcquisitionSpec(
+    { acquisition: { url: 'https://www.youtube.com/watch?v=abc', expectedTitle: 'T', expectedArtist: 'A' } },
+    HOSTS
+  );
+  assert.equal(spec.url, 'https://www.youtube.com/watch?v=abc');
+  assert.equal(spec.query, undefined);
+  assert.equal(spec.expectedArtist, 'A');
+});
+
+test('accepts a search query and strips newlines that could forge print output', () => {
+  const spec = parseAcquisitionSpec(
+    { acquisition: { query: 'Boards of Canada\nRoygbiv', expectedTitle: 'Roygbiv' } },
+    HOSTS
+  );
+  assert.equal(spec.query, 'Boards of Canada Roygbiv');
+  assert.equal(spec.url, undefined);
+});
+
+test('rejects a candidate naming neither a url nor a query', () => {
+  assert.throws(() => parseAcquisitionSpec({ acquisition: { expectedTitle: 'T' } }, HOSTS), /url or a query/);
+  assert.throws(() => parseAcquisitionSpec({ acquisition: { query: '   ' } }, HOSTS), /url or a query/);
+  assert.throws(() => parseAcquisitionSpec({}, HOSTS), /no resolved acquisition source/);
+});
+
+test('search-resolved urls face the same gates as supplied ones', () => {
+  assert.throws(() => validateAcquisitionUrl('https://evil.example/x', HOSTS), /not allow-listed/);
+  assert.throws(() => validateAcquisitionUrl('http://www.youtube.com/x', HOSTS), /must use HTTPS/);
+  assert.throws(() => validateAcquisitionUrl('https://u:p@www.youtube.com/x', HOSTS), /must not contain credentials/);
+  assert.equal(
+    validateAcquisitionUrl('https://WWW.YouTube.com/watch?v=1', HOSTS),
+    'https://www.youtube.com/watch?v=1'
+  );
+});

workers/src/acquisition.service.ts

@@ -13,7 +13,10 @@ type CandidateRow = {
 };
 
 type AcquisitionSpec = {
-  url: string;
+  /** A vetted, allow-listed HTTPS URL. Mutually exclusive with `query`. */
+  url?: string;
+  /** A search phrase to resolve into such a URL at download time. */
+  query?: string;
   expectedTitle?: string;
   expectedArtist?: string;
 };
@@ -72,18 +75,17 @@ function matchesExpected(actual: string, expected: string | undefined): boolean
   return left === right || left.includes(right) || right.includes(left);
 }
 
-/** Parse only a deliberately supplied HTTPS source URL; never accept argv/query strings. */
-export function parseAcquisitionSpec(notes: unknown, allowedHosts: Set<string>): AcquisitionSpec {
-  const notesValue = typeof notes === 'string' ? JSON.parse(notes) : notes;
-  const candidate = (notesValue as { acquisition?: unknown } | null)?.acquisition;
-  if (!candidate || typeof candidate !== 'object') {
-    throw new Error('candidate has no resolved acquisition source');
-  }
-  const { url, expectedTitle, expectedArtist } = candidate as Record<string, unknown>;
-  if (typeof url !== 'string') throw new Error('acquisition source URL is required');
+/**
+ * Every URL that reaches the downloader passes through here: HTTPS only, no
+ * embedded credentials, host on the allow-list. Applied to operator-supplied
+ * URLs and to search-resolved ones alike, so a search cannot widen the hosts a
+ * download may come from.
+ */
+export function validateAcquisitionUrl(raw: unknown, allowedHosts: Set<string>): string {
+  if (typeof raw !== 'string') throw new Error('acquisition source URL is required');
   let parsed: URL;
   try {
-    parsed = new URL(url);
+    parsed = new URL(raw);
   } catch {
     throw new Error('acquisition source URL is invalid');
   }
@@ -92,11 +94,41 @@ export function parseAcquisitionSpec(notes: unknown, allowedHosts: Set<string>):
   if (!allowedHosts.has(parsed.hostname.toLowerCase())) {
     throw new Error(`acquisition host is not allow-listed: ${parsed.hostname}`);
   }
-  return {
-    url: parsed.toString(),
+  return parsed.toString();
+}
+
+/**
+ * Parse either a vetted HTTPS URL or a search phrase to resolve into one.
+ *
+ * The search form is what automated discovery emits: a graph walk or a
+ * similarity lookup names an artist and a title, never a downloadable file. A
+ * resolved search result is trusted no further than an operator-supplied URL —
+ * same host allow-list, and `matchesExpected` still rejects the download if the
+ * file's tags disagree with the candidate.
+ */
+export function parseAcquisitionSpec(notes: unknown, allowedHosts: Set<string>): AcquisitionSpec {
+  const notesValue = typeof notes === 'string' ? JSON.parse(notes) : notes;
+  const candidate = (notesValue as { acquisition?: unknown } | null)?.acquisition;
+  if (!candidate || typeof candidate !== 'object') {
+    throw new Error('candidate has no resolved acquisition source');
+  }
+  const { url, query, expectedTitle, expectedArtist } = candidate as Record<string, unknown>;
+  const expected = {
     expectedTitle: typeof expectedTitle === 'string' ? expectedTitle.slice(0, 500) : undefined,
     expectedArtist: typeof expectedArtist === 'string' ? expectedArtist.slice(0, 500) : undefined,
   };
+
+  if (url !== undefined) {
+    return { url: validateAcquisitionUrl(url, allowedHosts), ...expected };
+  }
+  if (typeof query === 'string' && query.trim() !== '') {
+    // Newlines would let a crafted candidate forge extra --print output lines
+    // when the resolver parses stdout.
+    const cleaned = query.replace(/[\r\n]+/g, ' ').trim().slice(0, 300);
+    if (cleaned === '') throw new Error('acquisition search query is empty');
+    return { query: cleaned, ...expected };
+  }
+  throw new Error('acquisition source needs either a url or a query');
 }
 
 async function runDownloader(executable: string, args: string[], timeoutMs: number): Promise<string> {
@@ -153,6 +185,24 @@ export class AcquisitionService {
     );
   }
 
+  /**
+   * Turn a search phrase into one allow-listed URL, without downloading.
+   *
+   * ponytail: first search result only. Ranking alternatives needs a quality
+   * signal this system does not have, and the tag check downstream already
+   * rejects a wrong hit. Revisit if mismatches become common.
+   */
+  private async resolveQueryToUrl(query: string): Promise<string> {
+    const stdout = await runDownloader(this.config.ytDlpPath, [
+      '--no-playlist', '--no-progress', '--skip-download',
+      '--print', 'webpage_url',
+      '--', `ytsearch1:${query}`,
+    ], this.config.timeoutMs);
+    const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+    if (lines.length !== 1) throw new Error('search did not resolve to exactly one result');
+    return validateAcquisitionUrl(lines[0], this.config.allowedHosts);
+  }
+
   async acquire(candidateId: string): Promise<AcquisitionResult> {
     const rowResult = await this.pgPool.query<CandidateRow>(
       `SELECT id, source, notes, status FROM discovery_candidates WHERE id = $1`, [candidateId]
@@ -212,13 +262,19 @@ export class AcquisitionService {
       // Arguments are fixed by us. The sole untrusted value is the validated URL
       // and spawn() is invoked with shell:false, so no command interpolation is
       // possible. One URL / one item is deliberate: playlists are out of scope.
+      const sourceUrl = spec.url ?? await this.resolveQueryToUrl(spec.query as string);
+
       const outputTemplate = path.join(candidateDir, '%(id)s.%(ext)s');
       const stdout = await runDownloader(this.config.ytDlpPath, [
         '--no-playlist', '--no-progress', '--restrict-filenames',
         '--extract-audio', '--audio-format', 'mp3', '--audio-quality', '5',
+        // Without this the mp3 carries no tags at all, the scanner falls back to
+        // "<video id>.mp3" / "Unknown Artist", and the tag check below rejects
+        // every download.
+        '--embed-metadata',
         '--output', outputTemplate,
         '--print', 'after_move:filepath',
-        '--', spec.url,
+        '--', sourceUrl,
       ], this.config.timeoutMs);
       const reportedPaths = stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
       if (reportedPaths.length !== 1) throw new Error('downloader did not report exactly one output file');
@@ -235,6 +291,11 @@ export class AcquisitionService {
       await this.setStatus(candidate.id, 'scanning', null);
       const scan = await this.scanner.scanDirectory(candidateDir, {
         sourceType: 'RECOMMENDATION', probationStatus: 'probation', candidateId: candidate.id,
+        // A source that embeds no tags leaves nothing to verify against. The
+        // candidate's own vetted names are then both the display names and
+        // what the check below compares, so the download is accepted and the
+        // track stays on probation, where listening decides its fate.
+        fallbackTitle: spec.expectedTitle, fallbackArtist: spec.expectedArtist,
       });
       if (scan.trackIds.length !== 1) {
         throw new Error(`scanner created ${scan.trackIds.length} tracks; expected exactly one`);
@@ -270,7 +331,14 @@ export class AcquisitionService {
            ON CONFLICT (subject_type, subject_id, predicate, object_type, object_id, source, user_id)
            DO UPDATE SET confidence = EXCLUDED.confidence,
                          last_reinforced_at = NOW(), raw = EXCLUDED.raw`,
-          [trackId, candidate.id, candidate.source, JSON.stringify({ expectedTitle: spec.expectedTitle, expectedArtist: spec.expectedArtist })]
+          [trackId, candidate.id, candidate.source, JSON.stringify({
+            expectedTitle: spec.expectedTitle,
+            expectedArtist: spec.expectedArtist,
+            // Which URL a search actually landed on is the only way to audit a
+            // bad acquisition after the fact.
+            sourceUrl,
+            resolvedFromQuery: spec.query ?? null,
+          })]
         );
         await client.query('COMMIT');
       } catch (err) {

workers/src/discovery-sources.service.ts

@@ -0,0 +1,288 @@
+// External discovery sources: where recommendation candidates come from.
+//
+// Two deliberately separate strategies, one per source_trust key:
+//
+//   new_release            — an artist you actually play released something new.
+//                            Seeded from local play counts, resolved via Deezer.
+//   similar_recommendation — Last.fm's "sounds like" neighbours of the tracks
+//                            you play most. Weaker prior, wider reach.
+//
+// Both write track-level rows into discovery_candidates carrying a search spec
+// (artist + title) rather than a URL: the acquisition worker resolves that to a
+// concrete allow-listed URL at download time. Nothing here touches the
+// filesystem or the player; a candidate is only ever an intent to try a track.
+
+import type { Pool } from 'pg';
+import { DeezerClient, LastFmClient } from './integrations/index.js';
+
+/** A track we might want to try, before any acquisition source is resolved. */
+interface CandidateSeed {
+  source: 'new_release' | 'similar_recommendation';
+  /** Stable dedup key within the source. */
+  externalId: string;
+  title: string;
+  artist: string;
+  /** Local artist this candidate hangs off: the releasing or the seed artist. */
+  relatedArtistId: string;
+  /** 0..1 prior on this candidate being worth a probation slot. */
+  relevance: number;
+  raw: Record<string, unknown>;
+}
+
+interface TopArtist {
+  id: string;
+  name: string;
+  plays: number;
+}
+
+interface TopTrack {
+  title: string;
+  artist: string;
+  artistId: string;
+  plays: number;
+}
+
+export interface DiscoverySourcesResult {
+  considered: number;
+  inserted: number;
+}
+
+const NEW_RELEASE_WINDOW_DAYS = 120;
+
+export class DiscoverySourcesService {
+  constructor(
+    private readonly pgPool: Pool,
+    private readonly deezer = new DeezerClient(),
+    private readonly lastfm = new LastFmClient(),
+  ) {}
+
+  // ---------------------------------------------------------------
+  // Shared plumbing
+  // ---------------------------------------------------------------
+
+  /** Artists ranked by how much of their local catalog actually gets played. */
+  private async topArtists(limit: number): Promise<TopArtist[]> {
+    const res = await this.pgPool.query<{ id: string; name: string; plays: string }>(
+      `SELECT a.id, COALESCE(a.canonical_name, a.name) AS name, SUM(t.play_count)::text AS plays
+         FROM tracks t
+         JOIN albums al ON al.id = t.album_id
+         JOIN artists a ON a.id = al.artist_id
+        WHERE t.state = 'LIBRARY' AND t.deleted_at IS NULL AND t.play_count > 0
+        GROUP BY a.id, COALESCE(a.canonical_name, a.name)
+        ORDER BY SUM(t.play_count) DESC
+        LIMIT $1`,
+      [limit]
+    );
+    return res.rows.map((r) => ({ id: r.id, name: r.name, plays: Number(r.plays) }));
+  }
+
+  /** Most-played individual tracks — the seeds similarity lookups start from. */
+  private async topTracks(limit: number): Promise<TopTrack[]> {
+    const res = await this.pgPool.query<{ title: string; artist: string; artist_id: string; play_count: number }>(
+      `SELECT t.title, t.artist, al.artist_id, t.play_count
+         FROM tracks t
+         JOIN albums al ON al.id = t.album_id
+        WHERE t.state = 'LIBRARY' AND t.deleted_at IS NULL
+          AND t.play_count > 0 AND al.artist_id IS NOT NULL
+        ORDER BY t.play_count DESC
+        LIMIT $1`,
+      [limit]
+    );
+    return res.rows.map((r) => ({
+      title: r.title,
+      artist: r.artist,
+      artistId: r.artist_id,
+      plays: r.play_count,
+    }));
+  }
+
+  /**
+   * True when this artist+title is already a local track, in any state.
+   *
+   * Any state is deliberate: a retired or quarantined track must not be
+   * re-acquired, otherwise every sweep re-downloads what the listener already
+   * skipped away.
+   */
+  private async alreadyKnown(artist: string, title: string): Promise<boolean> {
+    const res = await this.pgPool.query<{ exists: boolean }>(
+      `SELECT EXISTS (
+         SELECT 1 FROM tracks
+          WHERE lower(artist) = lower($1) AND lower(title) = lower($2)
+       ) AS exists`,
+      [artist, title]
+    );
+    return res.rows[0]?.exists ?? false;
+  }
+
+  /**
+   * Insert a candidate plus the relevance claim `evalCandidates` reads.
+   *
+   * The claim's object is the local artist the candidate came from. For a new
+   * release that is the releasing artist; for a similarity hit it is the seed.
+   * Either way it is what the existing per-artist diversity cap should count.
+   */
+  private async insertCandidate(seed: CandidateSeed): Promise<boolean> {
+    // The (source, external_id) unique key does not catch the same song reaching
+    // us twice under different ids — a Deezer single and the album that carries
+    // it, or a Last.fm hit for something a new release already proposed. Name
+    // equality does, in any candidate state, for the same reason alreadyKnown
+    // ignores track state.
+    const duplicate = await this.pgPool.query<{ exists: boolean }>(
+      `SELECT EXISTS (
+         SELECT 1 FROM discovery_candidates
+          WHERE lower(title) = lower($1)
+            AND lower(artist_credit->0->>'name') = lower($2)
+       ) AS exists`,
+      [seed.title, seed.artist]
+    );
+    if (duplicate.rows[0]?.exists) return false;
+
+    const notes = {
+      discovery_source: seed.source,
+      ...seed.raw,
+      // The acquisition worker turns this into a concrete URL. No URL is
+      // recorded here because nothing has vetted one yet.
+      acquisition: {
+        query: `${seed.artist} ${seed.title}`,
+        expectedTitle: seed.title,
+        expectedArtist: seed.artist,
+      },
+    };
+
+    const dcRes = await this.pgPool.query<{ id: string }>(
+      `INSERT INTO discovery_candidates (source, external_id, title, artist_credit, notes)
+       VALUES ($1, $2, $3, $4::jsonb, $5::jsonb)
+       ON CONFLICT (source, external_id) DO NOTHING
+       RETURNING id`,
+      [
+        seed.source,
+        seed.externalId,
+        seed.title,
+        JSON.stringify([{ name: seed.artist, artist_id: seed.relatedArtistId }]),
+        JSON.stringify(notes),
+      ]
+    );
+    if (dcRes.rows.length === 0) return false;
+
+    await this.pgPool.query(
+      `INSERT INTO claims (
+         subject_type, subject_id, predicate, object_type, object_id,
+         source, confidence, raw
+       ) VALUES ('discovery_candidate', $1::uuid, 'discovery_candidate', 'artist', $2::uuid,
+                 $3, $4, $5::jsonb)
+       ON CONFLICT (subject_type, subject_id, predicate, object_type, object_id, source, user_id)
+       DO UPDATE SET confidence = EXCLUDED.confidence, last_reinforced_at = NOW()`,
+      [
+        dcRes.rows[0].id,
+        seed.relatedArtistId,
+        seed.source,
+        Math.max(0, Math.min(1, seed.relevance)),
+        JSON.stringify({ discovery_source: seed.source, ...seed.raw }),
+      ]
+    );
+    return true;
+  }
+
+  // ---------------------------------------------------------------
+  // new_release
+  // ---------------------------------------------------------------
+
+  /**
+   * For the `artistLimit` most-played local artists, find releases from the
+   * last NEW_RELEASE_WINDOW_DAYS whose tracks we do not own.
+   *
+   * ponytail: `tracksPerAlbum` tracks per new album, not the whole thing. A
+   * probation slot per track is the expensive resource, and one representative
+   * track is enough to learn whether the release lands. Raise it if retention
+   * on this source turns out high.
+   */
+  async discoverNewReleases(artistLimit = 15, tracksPerAlbum = 2): Promise<DiscoverySourcesResult> {
+    const since = new Date(Date.now() - NEW_RELEASE_WINDOW_DAYS * 86_400_000)
+      .toISOString()
+      .slice(0, 10);
+    let considered = 0;
+    let inserted = 0;
+
+    for (const artist of await this.topArtists(artistLimit)) {
+      const albums = await this.deezer.getArtistAlbumsSince(artist.name, since);
+      for (const album of albums) {
+        const titles = await this.deezer.getAlbumTracks(album.id);
+        let takenFromAlbum = 0;
+        for (const title of titles) {
+          if (takenFromAlbum >= tracksPerAlbum) break;
+          considered++;
+          if (await this.alreadyKnown(artist.name, title)) continue;
+          const ok = await this.insertCandidate({
+            source: 'new_release',
+            externalId: `deezer:${album.id}:${title.toLowerCase()}`,
+            title,
+            artist: artist.name,
+            relatedArtistId: artist.id,
+            // A new release by someone already in heavy rotation is the
+            // strongest prior this system has short of an explicit request.
+            relevance: 0.75,
+            raw: {
+              album: album.title,
+              release_date: album.releaseDate,
+              record_type: album.recordType,
+              seed_artist_plays: artist.plays,
+            },
+          });
+          if (ok) {
+            inserted++;
+            takenFromAlbum++;
+          }
+        }
+      }
+    }
+
+    return { considered, inserted };
+  }
+
+  // ---------------------------------------------------------------
+  // similar_recommendation
+  // ---------------------------------------------------------------
+
+  /**
+   * Ask Last.fm what sounds like the tracks played most, and keep the hits by
+   * artists other than the seed's — a different track by the same artist is a
+   * library gap, not a discovery.
+   */
+  async discoverRecommendations(seedLimit = 10, perSeed = 5): Promise<DiscoverySourcesResult> {
+    let considered = 0;
+    let inserted = 0;
+
+    for (const seed of await this.topTracks(seedLimit)) {
+      const similar = await this.lastfm.getSimilarTracks(seed.artist, seed.title, perSeed * 4);
+      let takenFromSeed = 0;
+      for (const hit of similar) {
+        if (takenFromSeed >= perSeed) break;
+        considered++;
+        if (hit.artist.toLowerCase() === seed.artist.toLowerCase()) continue;
+        if (await this.alreadyKnown(hit.artist, hit.name)) continue;
+        const ok = await this.insertCandidate({
+          source: 'similar_recommendation',
+          externalId: `lastfm:${hit.artist.toLowerCase()}:${hit.name.toLowerCase()}`,
+          title: hit.name,
+          artist: hit.artist,
+          relatedArtistId: seed.artistId,
+          // Last.fm's own match score, floored so a weak-but-present match
+          // still clears the acquisition gate's relevance > 0.3.
+          relevance: Math.max(0.35, Math.min(0.7, hit.match)),
+          raw: {
+            seed_artist: seed.artist,
+            seed_title: seed.title,
+            seed_plays: seed.plays,
+            lastfm_match: hit.match,
+          },
+        });
+        if (ok) {
+          inserted++;
+          takenFromSeed++;
+        }
+      }
+    }
+
+    return { considered, inserted };
+  }
+}

workers/src/index.ts

@@ -1,6 +1,6 @@
 import { Worker, Job } from 'bullmq';
 import { connection, QUEUE_NAME, queue } from './queue.js';
-import { MetadataRefreshJob, AudioAnalysisJob, AudioAnalysisSweepJob, LibraryScanJob, ArtistSimilarityJob, ArtistImageJob, AlbumCoverJob, ReprocessArtistsJob, AcquisitionJob } from './types.js';
+import { MetadataRefreshJob, AudioAnalysisJob, AudioAnalysisSweepJob, LibraryScanJob, ArtistSimilarityJob, ArtistImageJob, AlbumCoverJob, ReprocessArtistsJob, AcquisitionJob, DiscoverySourceJob } from './types.js';
 import { Pool } from 'pg';
 import { ScannerService } from './scanner.service.js';
 import { IntegrityService } from './integrity.service.js';
@@ -9,6 +9,7 @@ import { AudioFeaturesService } from './audio-features.service.js';
 import { CleanupSweepService } from './cleanup.service.js';
 import { reprocessArtists } from './reprocess-artists.service.js';
 import { AcquisitionService } from './acquisition.service.js';
+import { DiscoverySourcesService } from './discovery-sources.service.js';
 import { AUDIO_ANALYSIS_JOB_OPTIONS, AUDIO_ANALYSIS_VERSION, audioAnalysisJobId } from './audio-analysis.js';
 
 // Cron for the periodic integrity sweep (default: daily at 03:00). Configurable
@@ -22,6 +23,12 @@ const CLEANUP_SWEEP_CRON = process.env.CLEANUP_SWEEP_CRON || '0 */6 * * *';
 // 24h must transition to RESOLVED so returning users start fresh sessions.
 const VIBE_REAP_CRON = process.env.VIBE_REAP_CRON || '0 * * * *';
 const PROBATION_SWEEP_CRON = process.env.PROBATION_SWEEP_CRON || '15 * * * *';
+// Candidate generation runs daily, not hourly: both sources are seeded from
+// play counts, which barely move in an hour, and every extra run is external
+// API traffic that returns the same rows. New releases are checked at a
+// different hour from similarity so the two never contend for the proxy.
+const NEW_RELEASE_CRON = process.env.NEW_RELEASE_CRON || '30 5 * * *';
+const RECOMMENDATION_CRON = process.env.RECOMMENDATION_CRON || '30 6 * * *';
 // A small daily backfill is intentionally bounded. It refreshes stale v1
 // measurements over time without turning a worker restart into a library-wide
 // ffmpeg/Essentia batch.
@@ -74,6 +81,7 @@ async function initWorker() {
 
   const scannerService = new ScannerService(pgPool, queue);
   const acquisitionService = new AcquisitionService(pgPool, scannerService);
+  const discoverySources = new DiscoverySourcesService(pgPool);
   const enrichmentService = new EnrichmentService(pgPool);
   const audioFeaturesService = new AudioFeaturesService(pgPool);
   await audioFeaturesService.ensureSchema();
@@ -293,6 +301,18 @@ async function initWorker() {
         if (result.status === 'failed') throw new Error(result.reason);
         return result;
       }
+      case 'discover_new_releases': {
+        const payload = job.data as DiscoverySourceJob;
+        const result = await discoverySources.discoverNewReleases(payload.seeds, payload.perSeed);
+        console.log(`[Discovery] New releases considered=${result.considered} inserted=${result.inserted}`);
+        return result;
+      }
+      case 'discover_recommendations': {
+        const payload = job.data as DiscoverySourceJob;
+        const result = await discoverySources.discoverRecommendations(payload.seeds, payload.perSeed);
+        console.log(`[Discovery] Recommendations considered=${result.considered} inserted=${result.inserted}`);
+        return result;
+      }
       case 'probation_sweep': {
         // Keep probation moving without exposing an operator-only HTTP endpoint
         // as the sole lifecycle driver. These conditions mirror DiscoveryService.
@@ -311,8 +331,26 @@ async function initWorker() {
              AND (SELECT COUNT(*) FROM evidence e WHERE e.entity_type = 'track'
                     AND e.entity_id = t.id AND e.signal = 'skip_quick') >= 3`
         );
-        const result = { retained: retained.rowCount ?? 0, retired: retired.rowCount ?? 0 };
+        // Per-source outcomes, so which strategy is worth its bandwidth is a
+        // fact rather than a hunch. Logged every sweep because the interesting
+        // number is the trend, and the counts are a single grouped scan.
+        const bySource = await pgPool.query<{ source: string; probation: string; retained: string; retired: string }>(
+          `SELECT dc.source,
+                  COUNT(*) FILTER (WHERE t.probation_status = 'probation')::text AS probation,
+                  COUNT(*) FILTER (WHERE t.probation_status = 'retained')::text  AS retained,
+                  COUNT(*) FILTER (WHERE t.probation_status = 'retired')::text   AS retired
+             FROM discovery_candidates dc
+             JOIN tracks t ON t.id = dc.acquired_track_id
+            GROUP BY dc.source
+            ORDER BY dc.source`
+        );
+        const result = {
+          retained: retained.rowCount ?? 0,
+          retired: retired.rowCount ?? 0,
+          bySource: bySource.rows,
+        };
         console.log(`[Probation] Sweep retained=${result.retained} retired=${result.retired}`);
+        console.log(`[Probation] Lifetime by source: ${JSON.stringify(result.bySource)}`);
         return result;
       }
       default:
@@ -363,6 +401,20 @@ async function initWorker() {
   );
   console.log(`[Probation] Sweep scheduled with cron: ${PROBATION_SWEEP_CRON}`);
 
+  await queue.upsertJobScheduler(
+    'discover-new-releases',
+    { pattern: NEW_RELEASE_CRON },
+    { name: 'discover_new_releases', data: { reason: 'scheduled' } satisfies DiscoverySourceJob }
+  );
+  console.log(`[Discovery] New-release scan scheduled with cron: ${NEW_RELEASE_CRON}`);
+
+  await queue.upsertJobScheduler(
+    'discover-recommendations',
+    { pattern: RECOMMENDATION_CRON },
+    { name: 'discover_recommendations', data: { reason: 'scheduled' } satisfies DiscoverySourceJob }
+  );
+  console.log(`[Discovery] Recommendation scan scheduled with cron: ${RECOMMENDATION_CRON}`);
+
   await queue.upsertJobScheduler(
     'audio-analysis-sweep',
     { pattern: AUDIO_ANALYSIS_SWEEP_CRON },

workers/src/integrations/deezer.client.ts

@@ -22,6 +22,16 @@ export interface DeezerAlbum {
   coverBig: string;
 }
 
+/** An album in an artist's discography (only the fields we read). */
+export interface DeezerAlbumRelease {
+  id: number;
+  title: string;
+  /** ISO date, YYYY-MM-DD. */
+  releaseDate: string;
+  /** 'album' | 'single' | 'ep' | 'compilation' as reported by Deezer. */
+  recordType: string;
+}
+
 interface DeezerSearchResult {
   artist?: { name?: string };
   title?: string;
@@ -116,4 +126,69 @@ export class DeezerClient {
       return null;
     }
   }
+
+  /** Resolve an artist name to a Deezer artist id via exact fold-matched search. */
+  private async resolveArtistId(artist: string): Promise<number | null> {
+    if (artist.trim() === '') return null;
+    const qs = new URLSearchParams({ q: artist });
+    try {
+      const data = await requestJson<{ data?: { id?: number; name?: string }[] }>(
+        `${this.baseUrl}/search/artist?${qs.toString()}`,
+        { userAgent: this.userAgent, minIntervalMs: this.minIntervalMs }
+      );
+      const want = foldName(artist);
+      for (const a of data.data ?? []) {
+        if (foldName(a.name ?? '') === want && typeof a.id === 'number') return a.id;
+      }
+      return null;
+    } catch (err) {
+      console.warn('[Deezer] resolveArtistId failed:', (err as Error).message);
+      return null;
+    }
+  }
+
+  /**
+   * Albums by `artist` released on/after `sinceIso` (YYYY-MM-DD), newest first.
+   *
+   * ponytail: reads only the first page (25 albums). Deezer returns albums
+   * newest-first, so a release-date cutoff never needs page two unless an
+   * artist dropped 25 albums inside the window. Paginate if that ever happens.
+   */
+  async getArtistAlbumsSince(artist: string, sinceIso: string): Promise<DeezerAlbumRelease[]> {
+    const artistId = await this.resolveArtistId(artist);
+    if (artistId === null) return [];
+    try {
+      const data = await requestJson<{
+        data?: { id?: number; title?: string; release_date?: string; record_type?: string }[];
+      }>(`${this.baseUrl}/artist/${artistId}/albums?limit=25`, {
+        userAgent: this.userAgent,
+        minIntervalMs: this.minIntervalMs,
+      });
+      return (data.data ?? [])
+        .filter((a) => typeof a.id === 'number' && a.title && a.release_date && a.release_date >= sinceIso)
+        .map((a) => ({
+          id: a.id as number,
+          title: a.title as string,
+          releaseDate: a.release_date as string,
+          recordType: a.record_type ?? 'album',
+        }));
+    } catch (err) {
+      console.warn('[Deezer] getArtistAlbumsSince failed:', (err as Error).message);
+      return [];
+    }
+  }
+
+  /** Track titles on a Deezer album, in tracklist order. */
+  async getAlbumTracks(albumId: number): Promise<string[]> {
+    try {
+      const data = await requestJson<{ data?: { title?: string }[] }>(
+        `${this.baseUrl}/album/${albumId}/tracks?limit=50`,
+        { userAgent: this.userAgent, minIntervalMs: this.minIntervalMs }
+      );
+      return (data.data ?? []).map((t) => t.title ?? '').filter((t) => t !== '');
+    } catch (err) {
+      console.warn('[Deezer] getAlbumTracks failed:', (err as Error).message);
+      return [];
+    }
+  }
 }

workers/src/integrations/index.ts

@@ -35,7 +35,7 @@ export { ITunesClient, upscaleITunesArtwork } from './itunes.client.js';
 export type { ITunesAlbum } from './itunes.client.js';
 
 export { DeezerClient } from './deezer.client.js';
-export type { DeezerAlbum } from './deezer.client.js';
+export type { DeezerAlbum, DeezerAlbumRelease } from './deezer.client.js';
 
 export { WikimediaClient } from './wikimedia.client.js';
 

workers/src/scanner.service.ts

@@ -18,6 +18,13 @@ export interface ScanContext {
   sourceType?: 'MANUAL' | 'RECOMMENDATION';
   probationStatus?: 'probation' | 'retained' | 'retired';
   candidateId?: string;
+  /**
+   * Names to use when the file carries no title/artist tags. An acquired
+   * download often has none, and the filename ("VTKqlmCpTmQ.mp3") plus
+   * "Unknown Artist" is worse than the vetted candidate's own names.
+   */
+  fallbackTitle?: string;
+  fallbackArtist?: string;
 }
 
 export interface ScanResult {
@@ -31,7 +38,7 @@ export interface ScanResult {
  * in the title ("Song (feat. X)") are folded in either way. First artist is the
  * main artist; the rest are featured. See ./utils/artist-names for the rules.
  */
-function parseArtistsFromMetadata(common: any): { main: string; featured: string[] } {
+function parseArtistsFromMetadata(common: any, fallbackArtist = 'Unknown Artist'): { main: string; featured: string[] } {
   const rawTitle = common.title || '';
 
   // Structured array present: still split each entry (tags sometimes put a whole
@@ -52,7 +59,7 @@ function parseArtistsFromMetadata(common: any): { main: string; featured: string
     }
   }
 
-  return parseArtists(common.artist || 'Unknown Artist', rawTitle);
+  return parseArtists(common.artist || fallbackArtist, rawTitle);
 }
 
 function hashFile(filePath: string): Promise<string> {
@@ -136,8 +143,9 @@ export class ScannerService {
       const { common, format } = metadata;
 
       // 1. Ensure Artist(s) exist.
-      const trackTitle = common.title || path.basename(filePath);
-      const { main: mainArtistRaw, featured: featuredArtistNames } = parseArtistsFromMetadata(common);
+      const trackTitle = common.title || context.fallbackTitle || path.basename(filePath);
+      const { main: mainArtistRaw, featured: featuredArtistNames } =
+        parseArtistsFromMetadata(common, context.fallbackArtist || 'Unknown Artist');
 
       const { id: artistId, name: resolvedArtist } = await this.resolveOrCreateArtist(mainArtistRaw);
 

workers/src/types.ts

@@ -58,6 +58,18 @@ export interface AcquisitionJob {
   candidateId: string;
 }
 
+/**
+ * External candidate generation. Two job names share this payload because the
+ * knobs are the same shape; the strategies themselves stay separate.
+ */
+export interface DiscoverySourceJob {
+  reason?: string;
+  /** new_release: artists to check. similar_recommendation: seed tracks. */
+  seeds?: number;
+  /** Candidates to keep per album (new_release) or per seed track (similar). */
+  perSeed?: number;
+}
+
 export type JobPayload =
   | MetadataRefreshJob
   | ArtistSimilarityJob
@@ -69,4 +81,5 @@ export type JobPayload =
   | LibraryScanJob
   | IntegritySweepJob
   | ReprocessArtistsJob
-  | AcquisitionJob;
+  | AcquisitionJob
+  | DiscoverySourceJob;