Discovery harvest routes, nightly schedule and per-user refresh (BL-0051)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docker-compose.yml

@@ -58,6 +58,8 @@ services:
       READING_QUIZ_CATALOG_PATH: ${READING_QUIZ_CATALOG_PATH:-}
       SCRAPER_URL: http://scraper:8091
       SCRAPER_TOKEN_FILE: /run/secrets/scraper_token
+      DISCOVERY_HARVEST_AT: ${DISCOVERY_HARVEST_AT:-03:30}
+      DISCOVERY_ANILIST_URL: ${DISCOVERY_ANILIST_URL:-}
     networks: [backend, egress]
     healthcheck:
       test: [CMD, node, tools/healthcheck.js, reader]

docs/architecture.md

@@ -171,7 +171,7 @@ This keeps ids human-readable and derivable without a database. The price: renam
 
 | Path | Handling |
 |------|----------|
-| `/api/*` | `routes["<METHOD> <path>"]`. Handlers get `{ u, b, q, req, res }` and return an object, which is sent as JSON. Throwing `HttpError` sends `{ error }` with that status |
+| `/api/*` | `routes["<METHOD> <path>"]`. Handlers get `{ u, b, q, req, res }` and return an object, which is sent as JSON. Throwing `HttpError(code, message, extra)` sends `{ error, ...extra }` with that status (for example `nextAllowedAt` on a discovery `429`) |
 | `/lib/*` | Image files from `LIBRARY`. Requires a session, allows image extensions only, and the path must resolve inside `LIBRARY`. `ETag` + `private, max-age=86400` |
 | everything else | Static files from `public/` (`/` → `index.html`). `ETag` + `no-cache` |
 
@@ -207,6 +207,13 @@ All requests and responses are JSON. POSTs require `Content-Type: application/js
 | `POST /api/admin/user/role` `{name, admin}` | admin | Grant or revoke admin rights; keeps the password and sessions. Revoking your own rights is refused, so one admin always remains |
 | `POST /api/admin/user/delete` `{name}` | admin | Delete a user and their data |
 | `GET /api/admin/backup` · `POST /api/admin/backup` `{command:"settings"\|"run"}` | admin | Built-in backup status, schedule/retention, "back up now" ([backup.md](backup.md)) |
+| `GET /api/discovery` | user | The caller's discovery feed, settings, feedback and `refresh: { state: idle\|queued\|running, position?, nextAllowedAt?, blockedReason? }` (§6.6) |
+| `POST /api/discovery/feedback` `{candidateId, value}` | user | The caller's choice for a candidate; requires opt-in |
+| `POST /api/discovery/refresh` | user | Queue a per-user AniList harvest for the caller (opt-in required, else `403`). `{ harvestId, position }`; `409` while one is queued or running for the caller; `429 { nextAllowedAt }` within an hour of the last accepted refresh; `503 { blockedReason }` after AniList pushed back, until a nightly or admin harvest |
+| `GET /api/admin/discovery` | admin | Last scoring run, opted-in users, candidate count, plus `harvest` (last run record), `currentHarvest`, `cache: { files, bytes }`, `settings: { harvestEnabled, harvestAt }` and `nextNightlyAt` (`null` when off) |
+| `POST /api/admin/discovery/run` | admin | Re-score every user from local data; `409` while a run is active |
+| `POST /api/admin/discovery/harvest` | admin | Start a full harvest for all opted-in users: `{ harvestId, position }`, `409` while any harvest is queued or running |
+| `POST /api/admin/discovery/settings` `{harvestEnabled, harvestAt}` | admin | Exactly a boolean and an `HH:MM` time, anything else is `400`; stored in `discovery.json` `settings` |
 
 ### 6.3 Authentication and security
 
@@ -229,6 +236,26 @@ At request time: the sum of all users' votes, then `chapter.json.priority`, then
 
 In-app notifications and optional webhooks are produced by `announce()` after each scan, from the append-only `known` list. See [import.md §5](import.md#5-change-detection-and-notifications).
 
+### 6.6 Discovery harvest
+
+The reader alone talks to AniList for discovery ([discovery.md](discovery.md)). A single-flight harvester
+(`discovery/harvest.js`) fetches seeds for the opted-in users' anchors into `data/discovery-cache/`; scoring
+(`discovery/engine.js`) is local and publishes `data/discovery.json` atomically.
+
+- **Kinds and budgets:** `nightly` and `admin` (300 requests, every opted-in user), `user` (60 requests, the
+  caller only). After a nightly or admin harvest every user is re-scored; after a per-user refresh only that
+  user, and the other feeds are carried over unchanged into the same publish.
+- **Schedule:** checked every minute and once 30 s after startup. The time is `discovery.json` `settings`, else
+  `DISCOVERY_HARVEST_AT` (`HH:MM`), else `03:30`; `nightly.lastDate` is written only when the run is accepted.
+- **Run record** `discovery.json` `harvest`: `{ id, kind, user, status: running|success|stopped|interrupted,
+  startedAt, finishedAt, requests, budget, stopReason }`. `stopReason` is `budget`, `rate-limited`, `forbidden`,
+  `challenge`, `upstream-errors` or `error` (a local fault). A run cut off by a restart is marked `interrupted`
+  at startup; queued refreshes are not persisted.
+- **Published fields:** candidates found through AniList carry `tagRecords`, `authorRecords`, `seed`
+  (`recs|creator|tagTop|fresh`) and `fetchedAt`; feed items carry `group` (seed kind or period) and `reasons`
+  (all reasons, strongest first) next to `reason`; seed evidence carries `fetchedAt`.
+- `DISCOVERY_ANILIST_URL` overrides the AniList endpoint (tests point it at a local fixture server).
+
 ## 7. Frontend
 
 ### 7.1 Pages

server.js

@@ -23,7 +23,11 @@ const { secret } = require('./runtime/config');
 const { PeeringState, requirePeer } = require('./peering/state');
 const peerClient = require('./peering/client');
 const { DiscoveryStore } = require('./discovery/store');
-const { generate: generateDiscovery } = require('./discovery/engine');
+const { generateAsync: generateDiscovery, mergeUserFeed } = require('./discovery/engine');
+const { DiscoveryCache } = require('./discovery/cache');
+const { createClient: createAniListClient } = require('./discovery/anilist');
+const { anchorsFor } = require('./discovery/anchors');
+const { shouldRunNightly, Harvester } = require('./discovery/harvest');
 const {
   applyPulse, applyVisit, chapterInsight, earliestCompletion, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary,
 } = require('./reading-analytics');
@@ -44,6 +48,9 @@ const PEER_DATA = path.resolve(process.env.PEER_DATA || path.join(DATA, 'peering
 const peerStore = new PeeringState(PEER_DATA);
 const discoveryStore = new DiscoveryStore(DATA);
 const DISCOVERY_CACHE = path.join(DATA, 'scraper', 'metadata', 'anilist');
+const DISCOVERY_HARVEST_CACHE = path.join(DATA, 'discovery-cache');
+const HARVEST_TIME = /^([01]\d|2[0-3]):[0-5]\d$/;
+const DISCOVERY_HARVEST_AT = HARVEST_TIME.test(process.env.DISCOVERY_HARVEST_AT || '') ? process.env.DISCOVERY_HARVEST_AT : '03:30';
 let discoveryRun = null, discoveryError = null;
 const PUB = path.join(__dirname, 'public');
 const IMG = /\.(jpe?g|png|webp|gif|avif|bmp)$/i;
@@ -78,8 +85,9 @@ const readingAccess = createReadingAccess({ schedule: readingSchedule,
   getProgress: (user, mangaId) => db.progress[user]?.[mangaId] || {},
   getMangaTitle: mangaId => lib.byId.get(mangaId)?.title || null });
 
-class HttpError extends Error { constructor(code, msg) { super(msg); this.code = code; } }
+// `extra` fields are sent next to `error`, e.g. `nextAllowedAt` on a 429.
-function bad(code, msg) { throw new HttpError(code, msg); }
+class HttpError extends Error { constructor(code, msg, extra = {}) { super(msg); this.code = code; this.extra = extra; } }
+function bad(code, msg, extra) { throw new HttpError(code, msg, extra); }
 function peerForRoute(id) {
   try { return requirePeer(peerStore.load(), id); }
   catch { bad(404, 'Peer not found'); }
@@ -241,6 +249,26 @@ function readDiscovery() {
   try { return discoveryStore.load(); }
   catch (error) { bad(503, error.message); }
 }
+// Scoring is local: it reads the harvest cache and never calls AniList. With `onlyUser` only that
+// feed is rebuilt and the other users' feeds are carried over into the same publish.
+async function rescoreNow(onlyUser) {
+  const startedAt = new Date().toISOString();
+  let queuedTitles = [], queueAvailable = true;
+  try { queuedTitles = (await scraperRequest('/api/state')).queue?.filter(item => !['complete', 'removed'].includes(item.status)).map(item => item.title) || []; }
+  catch { queueAvailable = false; }
+  const result = await generateDiscovery({ db, library: lib.mangas, greatest: greatestCatalog,
+    cacheDir: DISCOVERY_CACHE, harvestCache, queuedTitles, onlyUser });
+  // Read the prior store only now, right before the synchronous publish, so no other publish slips between.
+  const value = onlyUser === undefined ? result : { ...result, ...mergeUserFeed(discoveryStore.load(), result, onlyUser) };
+  return { published: discoveryStore.publish({ ...value, startedAt }), queueAvailable };
+}
+// One rescore at a time: an admin run and a harvest follow-up must not publish over each other.
+let rescoring = Promise.resolve();
+function rescore({ onlyUser } = {}) {
+  const run = rescoring.then(() => rescoreNow(onlyUser));
+  rescoring = run.catch(() => {});
+  return run;
+}
 function startDiscoveryRun() {
   if (discoveryRun?.status === 'running') bad(409, 'A discovery run is already active');
   readDiscovery(); // Refuse a new run if the on-disk store is corrupt.
@@ -249,12 +277,7 @@ function startDiscoveryRun() {
   discoveryError = null;
   setImmediate(async () => {
     try {
-      let queuedTitles = [], queueAvailable = true;
+      const { published, queueAvailable } = await rescore();
-      try { queuedTitles = (await scraperRequest('/api/state')).queue?.filter(item => !['complete', 'removed'].includes(item.status)).map(item => item.title) || []; }
-      catch { queueAvailable = false; }
-      const result = generateDiscovery({ db, library: lib.mangas, greatest: greatestCatalog,
-        cacheDir: DISCOVERY_CACHE, queuedTitles });
-      const published = discoveryStore.publish({ ...result, startedAt });
       discoveryRun = { id, status: 'success', startedAt, finishedAt: published.lastRun.finishedAt,
         candidates: published.lastRun.candidateCount, queueAvailable };
     } catch (error) {
@@ -266,6 +289,56 @@ function startDiscoveryRun() {
   return discoveryRun;
 }
 
+// ---------- discovery harvest (AniList seeds, BL-0051) ----------
+const harvestCache = new DiscoveryCache(DISCOVERY_HARVEST_CACHE);
+const discoverySettings = state => ({ harvestEnabled: state.settings?.harvestEnabled ?? true,
+  harvestAt: state.settings?.harvestAt ?? DISCOVERY_HARVEST_AT });
+const localDate = date => [date.getFullYear(), String(date.getMonth() + 1).padStart(2, '0'), String(date.getDate()).padStart(2, '0')].join('-');
+function nextNightlyAt(state, now = new Date()) {
+  const { harvestEnabled, harvestAt } = discoverySettings(state);
+  if (!harvestEnabled) return null;
+  const [hours, minutes] = harvestAt.split(':').map(Number);
+  const slot = new Date(now);
+  slot.setHours(hours, minutes, 0, 0);
+  if (state.nightly?.lastDate === localDate(now)) slot.setDate(slot.getDate() + 1);
+  else if (slot < now) return now.toISOString(); // due: the next minute's check starts it
+  return slot.toISOString();
+}
+function harvestTargets(kind, userName) {
+  const users = kind === 'user' ? [db.users[userName]].filter(Boolean) : Object.values(db.users);
+  return users.filter(user => settingsOf(user).discoveryEnabled).map(user => ({ user,
+    anchors: anchorsFor({ user, db, library: lib.mangas, greatest: greatestCatalog }), mature: !!settingsOf(user).discoveryMature }));
+}
+// Queued jobs live in memory only; a run cut off by a restart is recorded as interrupted.
+try {
+  const { harvest } = discoveryStore.load();
+  if (harvest?.status === 'running') discoveryStore.patch({ harvest: { ...harvest, status: 'interrupted', finishedAt: new Date().toISOString() } });
+} catch (error) { console.error('[discovery]', error.message); }
+const harvester = new Harvester({ cache: harvestCache, client: createAniListClient(), store: discoveryStore,
+  targetsFor: harvestTargets, onDone: job => rescore(job.kind === 'user' ? { onlyUser: job.user } : {}) });
+function requestHarvest(args) {
+  readDiscovery(); // A corrupt store fails closed before any AniList call.
+  try { return harvester.request(args); }
+  catch (error) {
+    if (!error.status) throw error;
+    const extra = {};
+    for (const key of ['nextAllowedAt', 'blockedReason']) if (error[key]) extra[key] = error[key];
+    bad(error.status, error.message, extra);
+  }
+}
+function checkNightly() {
+  let state;
+  try { state = discoveryStore.load(); } catch { return; }
+  const now = new Date(), { harvestEnabled, harvestAt } = discoverySettings(state);
+  if (!shouldRunNightly({ now, harvestAt, lastNightlyDate: state.nightly?.lastDate, enabled: harvestEnabled })) return;
+  if (harvester.current()) return;
+  // The date is written only once the run was accepted; a 409 (refresh in flight) retries next minute.
+  try { harvester.request({ kind: 'nightly' }); }
+  catch (error) { if (error.status !== 409) console.error('[discovery] nightly harvest:', error.message); return; }
+  try { discoveryStore.patch({ nightly: { lastDate: localDate(now) } }); }
+  catch (error) { console.error('[discovery]', error.message); }
+}
+
 const mangaDexSearchCache = new Map();
 let lastMangaDexSearch = 0;
 async function searchGreatestMangaDex(entry) {
@@ -494,6 +567,7 @@ const routes = {
       discoveryReadingActivity: !!settingsOf(u).discoveryReadingActivity,
       discoveryMature: !!settingsOf(u).discoveryMature },
     generatedAt: feed?.generatedAt || null, revision: state.revision, lastRun: state.lastRun,
+    refresh: harvester.state(u.name),
     feedback: db.discoveryFeedback?.[u.name] || {},
     items: (feed?.items || []).map(item => ({ ...item, candidate: state.candidates[item.candidateId] || null })).filter(item => item.candidate) };
   },
@@ -507,14 +581,30 @@ const routes = {
     save();
     return { candidateId: id, value };
   },
+  'POST /api/discovery/refresh': ({ u }) => {
+    if (!settingsOf(u).discoveryEnabled) bad(403, 'Enable personalized discovery first');
+    return requestHarvest({ kind: 'user', user: u.name });
+  },
   'GET /api/admin/discovery': ({ u }) => {
     admin(u);
     const state = readDiscovery();
     return { revision: state.revision, lastRun: state.lastRun, currentRun: discoveryRun,
       lastError: discoveryError, optedInUsers: Object.values(db.users).filter(user => user.settings?.discoveryEnabled).length,
-      candidateCount: Object.keys(state.candidates).length };
+      candidateCount: Object.keys(state.candidates).length,
+      harvest: state.harvest, currentHarvest: harvester.current(), cache: harvestCache.stats(),
+      settings: discoverySettings(state), nextNightlyAt: nextNightlyAt(state) };
   },
   'POST /api/admin/discovery/run': ({ u }) => { admin(u); return startDiscoveryRun(); },
+  'POST /api/admin/discovery/harvest': ({ u }) => { admin(u); return requestHarvest({ kind: 'admin' }); },
+  'POST /api/admin/discovery/settings': ({ u, b }) => {
+    admin(u);
+    const keys = b && typeof b === 'object' && !Array.isArray(b) ? Object.keys(b).sort().join() : '';
+    if (keys !== 'harvestAt,harvestEnabled' || typeof b.harvestEnabled !== 'boolean' || typeof b.harvestAt !== 'string' || !HARVEST_TIME.test(b.harvestAt))
+      bad(400, 'Send exactly { harvestEnabled: true|false, harvestAt: "HH:MM" }');
+    readDiscovery();
+    const state = discoveryStore.patch({ settings: { harvestEnabled: b.harvestEnabled, harvestAt: b.harvestAt } });
+    return { settings: discoverySettings(state), nextNightlyAt: nextNightlyAt(state) };
+  },
 
   'GET /api/greatest': ({ u }) => ({
     title: greatestCatalog.title, updatedAt: greatestCatalog.updatedAt, periods: greatestCatalog.periods,
@@ -971,7 +1061,7 @@ http.createServer(async (req, res) => {
   } catch (e) {
     const expected = e instanceof HttpError || e instanceof CatalogError;
     if (!expected) console.error(e);
-    if (!res.headersSent) sendJson(res, expected ? e.code : 500, { error: expected ? e.message : 'Internal error' });
+    if (!res.headersSent) sendJson(res, expected ? e.code : 500, expected ? { error: e.message, ...e.extra } : { error: 'Internal error' });
   }
 }).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT}  (library: ${LIB}, data: ${DATA})`));
 
@@ -1010,3 +1100,7 @@ async function pollScraper() {
 }
 pollScraper();
 setInterval(pollScraper, 15000).unref();
+
+// Nightly discovery harvest; the delayed first check also covers a slot missed while the reader was down.
+setInterval(checkNightly, 60000).unref();
+setTimeout(checkNightly, 30000).unref();

test/discovery-server.test.js

@@ -0,0 +1,260 @@
+'use strict';
+// BL-0051: discovery harvest routes, per-user refresh and nightly state, against a fixture AniList server.
+const test = require('node:test'), assert = require('node:assert/strict');
+const fs = require('fs'), os = require('os'), path = require('path'), net = require('net'), http = require('http'), crypto = require('crypto');
+const { spawn } = require('child_process');
+
+const FIXTURES = path.join(__dirname, 'fixtures', 'anilist');
+const fixture = name => JSON.parse(fs.readFileSync(path.join(FIXTURES, `${name}.json`), 'utf8'));
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+// The anchor title carries no tags or staff, so a run stays at title + recs + fresh (the client allows 25 a minute).
+function fixtureBodies() {
+  const title = fixture('title');
+  Object.assign(title.data.Media, { genres: [], tags: [], staff: { edges: [] } });
+  const recs = fixture('recs');
+  recs.data.Media.recommendations.nodes[0].rating = 500;
+  return { title, recs, creator: fixture('creator'), tagtop: fixture('tagtop'), fresh: fixture('fresh') };
+}
+
+async function startAniList({ delayMs = 0 } = {}) {
+  const bodies = fixtureBodies(), seen = [];
+  const server = http.createServer((req, res) => {
+    let raw = '';
+    req.on('data', chunk => { raw += chunk; });
+    req.on('end', async () => {
+      const query = String(JSON.parse(raw || '{}').query || '');
+      const kind = query.includes('recommendations') ? 'recs' : query.includes('Staff(') ? 'creator'
+        : /genre_in|tag_in/.test(query) ? 'tagtop' : query.includes('startDate_greater') ? 'fresh' : 'title';
+      seen.push(kind);
+      if (delayMs) await sleep(delayMs);
+      res.writeHead(200, { 'content-type': 'application/json' });
+      res.end(JSON.stringify(bodies[kind]));
+    });
+  });
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  return { url: `http://127.0.0.1:${server.address().port}/`, seen, close: () => new Promise(resolve => server.close(resolve)) };
+}
+
+async function freePort() {
+  const listener = net.createServer();
+  await new Promise(resolve => listener.listen(0, '127.0.0.1', resolve));
+  const port = listener.address().port;
+  await new Promise(resolve => listener.close(resolve));
+  return port;
+}
+
+const hashPw = pw => { const salt = crypto.randomBytes(16).toString('hex'); return salt + ':' + crypto.scryptSync(pw, salt, 32).toString('hex'); };
+const sha = s => crypto.createHash('sha256').update(s).digest('hex');
+const USERS = { boss: { admin: true, discovery: false }, alice: { discovery: true }, bob: { discovery: true }, carol: { discovery: false } };
+
+// A disposable reader with ready-made accounts and sessions; nightly harvests are off unless the store says otherwise.
+async function startReader({ anilist, discovery = {} } = {}) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-server-'));
+  const library = path.join(root, 'library'), data = path.join(root, 'data');
+  fs.mkdirSync(path.join(library, 'Example Manga'), { recursive: true });
+  fs.writeFileSync(path.join(library, 'Example Manga', 'notes.txt'), 'not a chapter');
+  fs.mkdirSync(path.join(data, 'scraper'), { recursive: true });
+  fs.writeFileSync(path.join(data, 'scraper', 'queue.json'), JSON.stringify({ queue: [] }));
+  const users = {}, sessions = {}, cookies = {};
+  for (const [name, spec] of Object.entries(USERS)) {
+    users[name] = { name, hash: hashPw(`${name}-password`), admin: !!spec.admin, created: Date.now(),
+      settings: { discoveryEnabled: spec.discovery } };
+    const token = crypto.randomBytes(32).toString('base64url');
+    sessions[sha(token)] = { u: name, t: Date.now() };
+    cookies[name] = `sid=${token}`;
+  }
+  fs.writeFileSync(path.join(data, 'db.json'), JSON.stringify({ users, sessions,
+    discoveryFeedback: { alice: { 'anilist:101': 'interested' } } }));
+  fs.writeFileSync(path.join(data, 'discovery.json'), JSON.stringify({ schema: 1, revision: 0, candidates: {}, feeds: {},
+    lastRun: { status: 'never' }, harvest: null, settings: { harvestEnabled: false, harvestAt: '03:30' },
+    nightly: { lastDate: null }, ...discovery }));
+
+  const port = await freePort(), output = [];
+  const child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true,
+    env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0',
+      SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '',
+      DISCOVERY_ANILIST_URL: anilist ? anilist.url : 'http://127.0.0.1:9/', DISCOVERY_HARVEST_AT: '' } });
+  child.stdout.on('data', d => output.push(String(d)));
+  child.stderr.on('data', d => output.push(String(d)));
+  const base = `http://127.0.0.1:${port}`;
+  let ready = false;
+  for (let n = 0; n < 100 && !ready; n++) {
+    ready = await fetch(base + '/health').then(r => r.ok).catch(() => false);
+    if (!ready) await sleep(50);
+  }
+  assert.ok(ready, output.join(''));
+  const call = async (who, method, url, body) => {
+    const response = await fetch(base + url, { method, headers: { cookie: cookies[who], ...(body ? { 'content-type': 'application/json' } : {}) },
+      body: body === undefined ? undefined : JSON.stringify(body) });
+    return { status: response.status, body: await response.json().catch(() => null) };
+  };
+  return {
+    root, library, data, output,
+    get: (who, url) => call(who, 'GET', url),
+    post: (who, url, body = {}) => call(who, 'POST', url, body),
+    async stop() {
+      await new Promise(resolve => { if (child.exitCode !== null) return resolve(); child.once('exit', resolve); child.kill(); });
+      fs.rmSync(root, { recursive: true, force: true });
+    },
+  };
+}
+
+async function until(check, what, timeoutMs = 20000) {
+  const deadline = Date.now() + timeoutMs;
+  let last;
+  while (Date.now() < deadline) {
+    last = await check();
+    if (last) return last;
+    await sleep(100);
+  }
+  assert.fail(`Timed out waiting for ${what}`);
+}
+const idle = (reader, who) => until(async () => {
+  const { body } = await reader.get(who, '/api/discovery');
+  return body.refresh.state === 'idle' ? body : null;
+}, `${who}'s refresh to finish`);
+const harvestDone = reader => until(async () => {
+  const { body } = await reader.get('boss', '/api/admin/discovery');
+  return !body.currentHarvest && body.harvest && body.harvest.status !== 'running' ? body : null;
+}, 'the harvest to finish');
+
+function snapshot(dir) {
+  const out = {};
+  const walk = rel => {
+    for (const entry of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) {
+      const child = path.join(rel, entry.name), stat = fs.statSync(path.join(dir, child));
+      out[child] = entry.isDirectory() ? 'dir' : `${stat.size}:${stat.mtimeMs}`;
+      if (entry.isDirectory()) walk(child);
+    }
+  };
+  walk('');
+  return out;
+}
+
+test('refresh requires opt-in, queues, runs and rescores only the caller', { timeout: 60000 }, async () => {
+  const anilist = await startAniList();
+  const reader = await startReader({ anilist });
+  try {
+    assert.equal((await reader.post('boss', '/api/admin/discovery/run')).status, 200);
+    await until(async () => (await reader.get('boss', '/api/admin/discovery')).body.currentRun?.status === 'success', 'the first run');
+    const bobBefore = (await reader.get('bob', '/api/discovery')).body;
+    assert.ok(bobBefore.generatedAt);
+    assert.deepEqual(bobBefore.refresh, { state: 'idle' });
+
+    const refused = await reader.post('carol', '/api/discovery/refresh');
+    assert.equal(refused.status, 403);
+
+    const accepted = await reader.post('alice', '/api/discovery/refresh');
+    assert.equal(accepted.status, 200, JSON.stringify(accepted.body));
+    assert.match(accepted.body.harvestId, /^[0-9a-f]{16}$/);
+    assert.equal(accepted.body.position, 0);
+
+    const alice = await idle(reader, 'alice');
+    assert.ok(alice.items.some(item => item.candidateId === 'anilist:201'), JSON.stringify(alice.items.map(i => i.candidateId)));
+    const rec = alice.items.find(item => item.candidateId === 'anilist:201');
+    assert.ok(rec.reasons.some(reason => /recommend/.test(reason)));
+    assert.ok(rec.candidate.fetchedAt);
+    assert.deepEqual(anilist.seen.sort(), ['fresh', 'recs', 'title']);
+
+    const bobAfter = (await reader.get('bob', '/api/discovery')).body;
+    assert.equal(bobAfter.generatedAt, bobBefore.generatedAt);
+    assert.deepEqual(bobAfter.items.map(i => i.candidateId), bobBefore.items.map(i => i.candidateId));
+
+    const admin = (await reader.get('boss', '/api/admin/discovery')).body;
+    assert.equal(admin.harvest.kind, 'user');
+    assert.equal(admin.harvest.user, 'alice');
+    assert.equal(admin.harvest.status, 'success');
+    assert.equal(admin.harvest.requests, 3);
+    assert.equal(admin.harvest.budget, 60);
+    assert.ok(admin.cache.files >= 3 && admin.cache.bytes > 0, JSON.stringify(admin.cache));
+    assert.ok(fs.existsSync(path.join(reader.data, 'discovery-cache')));
+  } finally { await reader.stop(); await anilist.close(); }
+});
+
+test('second refresh within an hour is 429 with nextAllowedAt', { timeout: 60000 }, async () => {
+  const anilist = await startAniList({ delayMs: 400 });
+  const reader = await startReader({ anilist });
+  try {
+    assert.equal((await reader.post('bob', '/api/discovery/refresh')).status, 200);
+    const duplicate = await reader.post('bob', '/api/discovery/refresh');
+    assert.equal(duplicate.status, 409);
+    const state = await idle(reader, 'bob');
+    assert.ok(Date.parse(state.refresh.nextAllowedAt) > Date.now() + 50 * 60000);
+
+    const cooling = await reader.post('bob', '/api/discovery/refresh');
+    assert.equal(cooling.status, 429);
+    assert.equal(cooling.body.nextAllowedAt, state.refresh.nextAllowedAt);
+    assert.equal(typeof cooling.body.error, 'string');
+  } finally { await reader.stop(); await anilist.close(); }
+});
+
+test('admin harvest is admin-only and single-flight', { timeout: 60000 }, async () => {
+  const anilist = await startAniList({ delayMs: 500 });
+  const reader = await startReader({ anilist });
+  try {
+    assert.equal((await reader.post('alice', '/api/admin/discovery/harvest')).status, 403);
+    const first = await reader.post('boss', '/api/admin/discovery/harvest');
+    assert.equal(first.status, 200, JSON.stringify(first.body));
+    assert.equal(first.body.position, 0);
+    assert.equal((await reader.post('boss', '/api/admin/discovery/harvest')).status, 409);
+    const running = (await reader.get('boss', '/api/admin/discovery')).body;
+    assert.equal(running.currentHarvest.id, first.body.harvestId);
+    assert.equal(running.currentHarvest.kind, 'admin');
+    const done = await harvestDone(reader);
+    assert.equal(done.harvest.kind, 'admin');
+    assert.equal(done.harvest.budget, 300);
+    // Afterwards every opted-in user was rescored.
+    for (const who of ['alice', 'bob']) assert.ok((await reader.get(who, '/api/discovery')).body.generatedAt, who);
+  } finally { await reader.stop(); await anilist.close(); }
+});
+
+test('settings validation', { timeout: 30000 }, async () => {
+  const reader = await startReader();
+  try {
+    const before = (await reader.get('boss', '/api/admin/discovery')).body;
+    assert.deepEqual(before.settings, { harvestEnabled: false, harvestAt: '03:30' });
+    assert.equal(before.nextNightlyAt, null);
+    assert.equal((await reader.post('alice', '/api/admin/discovery/settings', { harvestEnabled: true, harvestAt: '04:00' })).status, 403);
+    for (const body of [{ harvestEnabled: true, harvestAt: '25:00' }, { harvestEnabled: true, harvestAt: '4:00' },
+      { harvestEnabled: 'yes', harvestAt: '04:00' }, { harvestEnabled: true }, { harvestEnabled: true, harvestAt: '04:00', extra: 1 }]) {
+      const response = await reader.post('boss', '/api/admin/discovery/settings', body);
+      assert.equal(response.status, 400, JSON.stringify(body));
+    }
+    const saved = await reader.post('boss', '/api/admin/discovery/settings', { harvestEnabled: true, harvestAt: '23:59' });
+    assert.equal(saved.status, 200);
+    const after = (await reader.get('boss', '/api/admin/discovery')).body;
+    assert.deepEqual(after.settings, { harvestEnabled: true, harvestAt: '23:59' });
+    const next = new Date(after.nextNightlyAt);
+    // The next slot today at 23:59, or "due now" when the test happens to run inside that minute.
+    const atSlot = next.getHours() === 23 && next.getMinutes() === 59 && next.getTime() > Date.now() - 120000;
+    assert.ok(atSlot || Math.abs(next.getTime() - Date.now()) < 120000, after.nextNightlyAt);
+  } finally { await reader.stop(); }
+});
+
+test('interrupted harvest is marked on startup', { timeout: 30000 }, async () => {
+  const reader = await startReader({ discovery: { harvest: { id: 'abc', kind: 'nightly', user: null, status: 'running',
+    startedAt: '2026-10-24T03:30:00.000Z', requests: 12, budget: 300 } } });
+  try {
+    const { body } = await reader.get('boss', '/api/admin/discovery');
+    assert.equal(body.harvest.status, 'interrupted');
+    assert.equal(body.harvest.id, 'abc');
+    assert.equal(body.harvest.requests, 12);
+    assert.ok(Date.parse(body.harvest.finishedAt));
+    assert.equal(body.currentHarvest, null);
+  } finally { await reader.stop(); }
+});
+
+test('harvest never touches scraper queue or library', { timeout: 60000 }, async () => {
+  const anilist = await startAniList();
+  const reader = await startReader({ anilist });
+  try {
+    const libraryBefore = snapshot(reader.library), scraperBefore = snapshot(path.join(reader.data, 'scraper'));
+    assert.equal((await reader.post('alice', '/api/discovery/refresh')).status, 200);
+    await idle(reader, 'alice');
+    assert.equal((await reader.post('boss', '/api/admin/discovery/harvest')).status, 200);
+    await harvestDone(reader);
+    assert.deepEqual(snapshot(reader.library), libraryBefore);
+    assert.deepEqual(snapshot(path.join(reader.data, 'scraper')), scraperBefore);
+  } finally { await reader.stop(); await anilist.close(); }
+});