diff --git a/docker-compose.yml b/docker-compose.yml index 3a2cb32..03eb556 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,6 +58,8 @@ services: READING_QUIZ_CATALOG_PATH: ${READING_QUIZ_CATALOG_PATH:-} SCRAPER_URL: http://scraper:8091 SCRAPER_TOKEN_FILE: /run/secrets/scraper_token + DISCOVERY_HARVEST_AT: ${DISCOVERY_HARVEST_AT:-03:30} + DISCOVERY_ANILIST_URL: ${DISCOVERY_ANILIST_URL:-} networks: [backend, egress] healthcheck: test: [CMD, node, tools/healthcheck.js, reader] diff --git a/docs/architecture.md b/docs/architecture.md index 9d1bb98..9e67fa8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -171,7 +171,7 @@ This keeps ids human-readable and derivable without a database. The price: renam | Path | Handling | |------|----------| -| `/api/*` | `routes[" "]`. Handlers get `{ u, b, q, req, res }` and return an object, which is sent as JSON. Throwing `HttpError` sends `{ error }` with that status | +| `/api/*` | `routes[" "]`. Handlers get `{ u, b, q, req, res }` and return an object, which is sent as JSON. Throwing `HttpError(code, message, extra)` sends `{ error, ...extra }` with that status (for example `nextAllowedAt` on a discovery `429`) | | `/lib/*` | Image files from `LIBRARY`. Requires a session, allows image extensions only, and the path must resolve inside `LIBRARY`. `ETag` + `private, max-age=86400` | | everything else | Static files from `public/` (`/` → `index.html`). `ETag` + `no-cache` | @@ -207,6 +207,13 @@ All requests and responses are JSON. POSTs require `Content-Type: application/js | `POST /api/admin/user/role` `{name, admin}` | admin | Grant or revoke admin rights; keeps the password and sessions. Revoking your own rights is refused, so one admin always remains | | `POST /api/admin/user/delete` `{name}` | admin | Delete a user and their data | | `GET /api/admin/backup` · `POST /api/admin/backup` `{command:"settings"\|"run"}` | admin | Built-in backup status, schedule/retention, "back up now" ([backup.md](backup.md)) | +| `GET /api/discovery` | user | The caller's discovery feed, settings, feedback and `refresh: { state: idle\|queued\|running, position?, nextAllowedAt?, blockedReason? }` (§6.6) | +| `POST /api/discovery/feedback` `{candidateId, value}` | user | The caller's choice for a candidate; requires opt-in | +| `POST /api/discovery/refresh` | user | Queue a per-user AniList harvest for the caller (opt-in required, else `403`). `{ harvestId, position }`; `409` while one is queued or running for the caller; `429 { nextAllowedAt }` within an hour of the last accepted refresh; `503 { blockedReason }` after AniList pushed back, until a nightly or admin harvest | +| `GET /api/admin/discovery` | admin | Last scoring run, opted-in users, candidate count, plus `harvest` (last run record), `currentHarvest`, `cache: { files, bytes }`, `settings: { harvestEnabled, harvestAt }` and `nextNightlyAt` (`null` when off) | +| `POST /api/admin/discovery/run` | admin | Re-score every user from local data; `409` while a run is active | +| `POST /api/admin/discovery/harvest` | admin | Start a full harvest for all opted-in users: `{ harvestId, position }`, `409` while any harvest is queued or running | +| `POST /api/admin/discovery/settings` `{harvestEnabled, harvestAt}` | admin | Exactly a boolean and an `HH:MM` time, anything else is `400`; stored in `discovery.json` `settings` | ### 6.3 Authentication and security @@ -229,6 +236,26 @@ At request time: the sum of all users' votes, then `chapter.json.priority`, then In-app notifications and optional webhooks are produced by `announce()` after each scan, from the append-only `known` list. See [import.md §5](import.md#5-change-detection-and-notifications). +### 6.6 Discovery harvest + +The reader alone talks to AniList for discovery ([discovery.md](discovery.md)). A single-flight harvester +(`discovery/harvest.js`) fetches seeds for the opted-in users' anchors into `data/discovery-cache/`; scoring +(`discovery/engine.js`) is local and publishes `data/discovery.json` atomically. + +- **Kinds and budgets:** `nightly` and `admin` (300 requests, every opted-in user), `user` (60 requests, the + caller only). After a nightly or admin harvest every user is re-scored; after a per-user refresh only that + user, and the other feeds are carried over unchanged into the same publish. +- **Schedule:** checked every minute and once 30 s after startup. The time is `discovery.json` `settings`, else + `DISCOVERY_HARVEST_AT` (`HH:MM`), else `03:30`; `nightly.lastDate` is written only when the run is accepted. +- **Run record** `discovery.json` `harvest`: `{ id, kind, user, status: running|success|stopped|interrupted, + startedAt, finishedAt, requests, budget, stopReason }`. `stopReason` is `budget`, `rate-limited`, `forbidden`, + `challenge`, `upstream-errors` or `error` (a local fault). A run cut off by a restart is marked `interrupted` + at startup; queued refreshes are not persisted. +- **Published fields:** candidates found through AniList carry `tagRecords`, `authorRecords`, `seed` + (`recs|creator|tagTop|fresh`) and `fetchedAt`; feed items carry `group` (seed kind or period) and `reasons` + (all reasons, strongest first) next to `reason`; seed evidence carries `fetchedAt`. +- `DISCOVERY_ANILIST_URL` overrides the AniList endpoint (tests point it at a local fixture server). + ## 7. Frontend ### 7.1 Pages diff --git a/server.js b/server.js index dad2ca3..ade2f72 100644 --- a/server.js +++ b/server.js @@ -23,7 +23,11 @@ const { secret } = require('./runtime/config'); const { PeeringState, requirePeer } = require('./peering/state'); const peerClient = require('./peering/client'); const { DiscoveryStore } = require('./discovery/store'); -const { generate: generateDiscovery } = require('./discovery/engine'); +const { generateAsync: generateDiscovery, mergeUserFeed } = require('./discovery/engine'); +const { DiscoveryCache } = require('./discovery/cache'); +const { createClient: createAniListClient } = require('./discovery/anilist'); +const { anchorsFor } = require('./discovery/anchors'); +const { shouldRunNightly, Harvester } = require('./discovery/harvest'); const { applyPulse, applyVisit, chapterInsight, earliestCompletion, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary, } = require('./reading-analytics'); @@ -44,6 +48,9 @@ const PEER_DATA = path.resolve(process.env.PEER_DATA || path.join(DATA, 'peering const peerStore = new PeeringState(PEER_DATA); const discoveryStore = new DiscoveryStore(DATA); const DISCOVERY_CACHE = path.join(DATA, 'scraper', 'metadata', 'anilist'); +const DISCOVERY_HARVEST_CACHE = path.join(DATA, 'discovery-cache'); +const HARVEST_TIME = /^([01]\d|2[0-3]):[0-5]\d$/; +const DISCOVERY_HARVEST_AT = HARVEST_TIME.test(process.env.DISCOVERY_HARVEST_AT || '') ? process.env.DISCOVERY_HARVEST_AT : '03:30'; let discoveryRun = null, discoveryError = null; const PUB = path.join(__dirname, 'public'); const IMG = /\.(jpe?g|png|webp|gif|avif|bmp)$/i; @@ -78,8 +85,9 @@ const readingAccess = createReadingAccess({ schedule: readingSchedule, getProgress: (user, mangaId) => db.progress[user]?.[mangaId] || {}, getMangaTitle: mangaId => lib.byId.get(mangaId)?.title || null }); -class HttpError extends Error { constructor(code, msg) { super(msg); this.code = code; } } -function bad(code, msg) { throw new HttpError(code, msg); } +// `extra` fields are sent next to `error`, e.g. `nextAllowedAt` on a 429. +class HttpError extends Error { constructor(code, msg, extra = {}) { super(msg); this.code = code; this.extra = extra; } } +function bad(code, msg, extra) { throw new HttpError(code, msg, extra); } function peerForRoute(id) { try { return requirePeer(peerStore.load(), id); } catch { bad(404, 'Peer not found'); } @@ -241,6 +249,26 @@ function readDiscovery() { try { return discoveryStore.load(); } catch (error) { bad(503, error.message); } } +// Scoring is local: it reads the harvest cache and never calls AniList. With `onlyUser` only that +// feed is rebuilt and the other users' feeds are carried over into the same publish. +async function rescoreNow(onlyUser) { + const startedAt = new Date().toISOString(); + let queuedTitles = [], queueAvailable = true; + try { queuedTitles = (await scraperRequest('/api/state')).queue?.filter(item => !['complete', 'removed'].includes(item.status)).map(item => item.title) || []; } + catch { queueAvailable = false; } + const result = await generateDiscovery({ db, library: lib.mangas, greatest: greatestCatalog, + cacheDir: DISCOVERY_CACHE, harvestCache, queuedTitles, onlyUser }); + // Read the prior store only now, right before the synchronous publish, so no other publish slips between. + const value = onlyUser === undefined ? result : { ...result, ...mergeUserFeed(discoveryStore.load(), result, onlyUser) }; + return { published: discoveryStore.publish({ ...value, startedAt }), queueAvailable }; +} +// One rescore at a time: an admin run and a harvest follow-up must not publish over each other. +let rescoring = Promise.resolve(); +function rescore({ onlyUser } = {}) { + const run = rescoring.then(() => rescoreNow(onlyUser)); + rescoring = run.catch(() => {}); + return run; +} function startDiscoveryRun() { if (discoveryRun?.status === 'running') bad(409, 'A discovery run is already active'); readDiscovery(); // Refuse a new run if the on-disk store is corrupt. @@ -249,12 +277,7 @@ function startDiscoveryRun() { discoveryError = null; setImmediate(async () => { try { - let queuedTitles = [], queueAvailable = true; - try { queuedTitles = (await scraperRequest('/api/state')).queue?.filter(item => !['complete', 'removed'].includes(item.status)).map(item => item.title) || []; } - catch { queueAvailable = false; } - const result = generateDiscovery({ db, library: lib.mangas, greatest: greatestCatalog, - cacheDir: DISCOVERY_CACHE, queuedTitles }); - const published = discoveryStore.publish({ ...result, startedAt }); + const { published, queueAvailable } = await rescore(); discoveryRun = { id, status: 'success', startedAt, finishedAt: published.lastRun.finishedAt, candidates: published.lastRun.candidateCount, queueAvailable }; } catch (error) { @@ -266,6 +289,56 @@ function startDiscoveryRun() { return discoveryRun; } +// ---------- discovery harvest (AniList seeds, BL-0051) ---------- +const harvestCache = new DiscoveryCache(DISCOVERY_HARVEST_CACHE); +const discoverySettings = state => ({ harvestEnabled: state.settings?.harvestEnabled ?? true, + harvestAt: state.settings?.harvestAt ?? DISCOVERY_HARVEST_AT }); +const localDate = date => [date.getFullYear(), String(date.getMonth() + 1).padStart(2, '0'), String(date.getDate()).padStart(2, '0')].join('-'); +function nextNightlyAt(state, now = new Date()) { + const { harvestEnabled, harvestAt } = discoverySettings(state); + if (!harvestEnabled) return null; + const [hours, minutes] = harvestAt.split(':').map(Number); + const slot = new Date(now); + slot.setHours(hours, minutes, 0, 0); + if (state.nightly?.lastDate === localDate(now)) slot.setDate(slot.getDate() + 1); + else if (slot < now) return now.toISOString(); // due: the next minute's check starts it + return slot.toISOString(); +} +function harvestTargets(kind, userName) { + const users = kind === 'user' ? [db.users[userName]].filter(Boolean) : Object.values(db.users); + return users.filter(user => settingsOf(user).discoveryEnabled).map(user => ({ user, + anchors: anchorsFor({ user, db, library: lib.mangas, greatest: greatestCatalog }), mature: !!settingsOf(user).discoveryMature })); +} +// Queued jobs live in memory only; a run cut off by a restart is recorded as interrupted. +try { + const { harvest } = discoveryStore.load(); + if (harvest?.status === 'running') discoveryStore.patch({ harvest: { ...harvest, status: 'interrupted', finishedAt: new Date().toISOString() } }); +} catch (error) { console.error('[discovery]', error.message); } +const harvester = new Harvester({ cache: harvestCache, client: createAniListClient(), store: discoveryStore, + targetsFor: harvestTargets, onDone: job => rescore(job.kind === 'user' ? { onlyUser: job.user } : {}) }); +function requestHarvest(args) { + readDiscovery(); // A corrupt store fails closed before any AniList call. + try { return harvester.request(args); } + catch (error) { + if (!error.status) throw error; + const extra = {}; + for (const key of ['nextAllowedAt', 'blockedReason']) if (error[key]) extra[key] = error[key]; + bad(error.status, error.message, extra); + } +} +function checkNightly() { + let state; + try { state = discoveryStore.load(); } catch { return; } + const now = new Date(), { harvestEnabled, harvestAt } = discoverySettings(state); + if (!shouldRunNightly({ now, harvestAt, lastNightlyDate: state.nightly?.lastDate, enabled: harvestEnabled })) return; + if (harvester.current()) return; + // The date is written only once the run was accepted; a 409 (refresh in flight) retries next minute. + try { harvester.request({ kind: 'nightly' }); } + catch (error) { if (error.status !== 409) console.error('[discovery] nightly harvest:', error.message); return; } + try { discoveryStore.patch({ nightly: { lastDate: localDate(now) } }); } + catch (error) { console.error('[discovery]', error.message); } +} + const mangaDexSearchCache = new Map(); let lastMangaDexSearch = 0; async function searchGreatestMangaDex(entry) { @@ -494,6 +567,7 @@ const routes = { discoveryReadingActivity: !!settingsOf(u).discoveryReadingActivity, discoveryMature: !!settingsOf(u).discoveryMature }, generatedAt: feed?.generatedAt || null, revision: state.revision, lastRun: state.lastRun, + refresh: harvester.state(u.name), feedback: db.discoveryFeedback?.[u.name] || {}, items: (feed?.items || []).map(item => ({ ...item, candidate: state.candidates[item.candidateId] || null })).filter(item => item.candidate) }; }, @@ -507,14 +581,30 @@ const routes = { save(); return { candidateId: id, value }; }, + 'POST /api/discovery/refresh': ({ u }) => { + if (!settingsOf(u).discoveryEnabled) bad(403, 'Enable personalized discovery first'); + return requestHarvest({ kind: 'user', user: u.name }); + }, 'GET /api/admin/discovery': ({ u }) => { admin(u); const state = readDiscovery(); return { revision: state.revision, lastRun: state.lastRun, currentRun: discoveryRun, lastError: discoveryError, optedInUsers: Object.values(db.users).filter(user => user.settings?.discoveryEnabled).length, - candidateCount: Object.keys(state.candidates).length }; + candidateCount: Object.keys(state.candidates).length, + harvest: state.harvest, currentHarvest: harvester.current(), cache: harvestCache.stats(), + settings: discoverySettings(state), nextNightlyAt: nextNightlyAt(state) }; }, 'POST /api/admin/discovery/run': ({ u }) => { admin(u); return startDiscoveryRun(); }, + 'POST /api/admin/discovery/harvest': ({ u }) => { admin(u); return requestHarvest({ kind: 'admin' }); }, + 'POST /api/admin/discovery/settings': ({ u, b }) => { + admin(u); + const keys = b && typeof b === 'object' && !Array.isArray(b) ? Object.keys(b).sort().join() : ''; + if (keys !== 'harvestAt,harvestEnabled' || typeof b.harvestEnabled !== 'boolean' || typeof b.harvestAt !== 'string' || !HARVEST_TIME.test(b.harvestAt)) + bad(400, 'Send exactly { harvestEnabled: true|false, harvestAt: "HH:MM" }'); + readDiscovery(); + const state = discoveryStore.patch({ settings: { harvestEnabled: b.harvestEnabled, harvestAt: b.harvestAt } }); + return { settings: discoverySettings(state), nextNightlyAt: nextNightlyAt(state) }; + }, 'GET /api/greatest': ({ u }) => ({ title: greatestCatalog.title, updatedAt: greatestCatalog.updatedAt, periods: greatestCatalog.periods, @@ -971,7 +1061,7 @@ http.createServer(async (req, res) => { } catch (e) { const expected = e instanceof HttpError || e instanceof CatalogError; if (!expected) console.error(e); - if (!res.headersSent) sendJson(res, expected ? e.code : 500, { error: expected ? e.message : 'Internal error' }); + if (!res.headersSent) sendJson(res, expected ? e.code : 500, expected ? { error: e.message, ...e.extra } : { error: 'Internal error' }); } }).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT} (library: ${LIB}, data: ${DATA})`)); @@ -1010,3 +1100,7 @@ async function pollScraper() { } pollScraper(); setInterval(pollScraper, 15000).unref(); + +// Nightly discovery harvest; the delayed first check also covers a slot missed while the reader was down. +setInterval(checkNightly, 60000).unref(); +setTimeout(checkNightly, 30000).unref(); diff --git a/test/discovery-server.test.js b/test/discovery-server.test.js new file mode 100644 index 0000000..88b536a --- /dev/null +++ b/test/discovery-server.test.js @@ -0,0 +1,260 @@ +'use strict'; +// BL-0051: discovery harvest routes, per-user refresh and nightly state, against a fixture AniList server. +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('fs'), os = require('os'), path = require('path'), net = require('net'), http = require('http'), crypto = require('crypto'); +const { spawn } = require('child_process'); + +const FIXTURES = path.join(__dirname, 'fixtures', 'anilist'); +const fixture = name => JSON.parse(fs.readFileSync(path.join(FIXTURES, `${name}.json`), 'utf8')); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + +// The anchor title carries no tags or staff, so a run stays at title + recs + fresh (the client allows 25 a minute). +function fixtureBodies() { + const title = fixture('title'); + Object.assign(title.data.Media, { genres: [], tags: [], staff: { edges: [] } }); + const recs = fixture('recs'); + recs.data.Media.recommendations.nodes[0].rating = 500; + return { title, recs, creator: fixture('creator'), tagtop: fixture('tagtop'), fresh: fixture('fresh') }; +} + +async function startAniList({ delayMs = 0 } = {}) { + const bodies = fixtureBodies(), seen = []; + const server = http.createServer((req, res) => { + let raw = ''; + req.on('data', chunk => { raw += chunk; }); + req.on('end', async () => { + const query = String(JSON.parse(raw || '{}').query || ''); + const kind = query.includes('recommendations') ? 'recs' : query.includes('Staff(') ? 'creator' + : /genre_in|tag_in/.test(query) ? 'tagtop' : query.includes('startDate_greater') ? 'fresh' : 'title'; + seen.push(kind); + if (delayMs) await sleep(delayMs); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify(bodies[kind])); + }); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + return { url: `http://127.0.0.1:${server.address().port}/`, seen, close: () => new Promise(resolve => server.close(resolve)) }; +} + +async function freePort() { + const listener = net.createServer(); + await new Promise(resolve => listener.listen(0, '127.0.0.1', resolve)); + const port = listener.address().port; + await new Promise(resolve => listener.close(resolve)); + return port; +} + +const hashPw = pw => { const salt = crypto.randomBytes(16).toString('hex'); return salt + ':' + crypto.scryptSync(pw, salt, 32).toString('hex'); }; +const sha = s => crypto.createHash('sha256').update(s).digest('hex'); +const USERS = { boss: { admin: true, discovery: false }, alice: { discovery: true }, bob: { discovery: true }, carol: { discovery: false } }; + +// A disposable reader with ready-made accounts and sessions; nightly harvests are off unless the store says otherwise. +async function startReader({ anilist, discovery = {} } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-server-')); + const library = path.join(root, 'library'), data = path.join(root, 'data'); + fs.mkdirSync(path.join(library, 'Example Manga'), { recursive: true }); + fs.writeFileSync(path.join(library, 'Example Manga', 'notes.txt'), 'not a chapter'); + fs.mkdirSync(path.join(data, 'scraper'), { recursive: true }); + fs.writeFileSync(path.join(data, 'scraper', 'queue.json'), JSON.stringify({ queue: [] })); + const users = {}, sessions = {}, cookies = {}; + for (const [name, spec] of Object.entries(USERS)) { + users[name] = { name, hash: hashPw(`${name}-password`), admin: !!spec.admin, created: Date.now(), + settings: { discoveryEnabled: spec.discovery } }; + const token = crypto.randomBytes(32).toString('base64url'); + sessions[sha(token)] = { u: name, t: Date.now() }; + cookies[name] = `sid=${token}`; + } + fs.writeFileSync(path.join(data, 'db.json'), JSON.stringify({ users, sessions, + discoveryFeedback: { alice: { 'anilist:101': 'interested' } } })); + fs.writeFileSync(path.join(data, 'discovery.json'), JSON.stringify({ schema: 1, revision: 0, candidates: {}, feeds: {}, + lastRun: { status: 'never' }, harvest: null, settings: { harvestEnabled: false, harvestAt: '03:30' }, + nightly: { lastDate: null }, ...discovery })); + + const port = await freePort(), output = []; + const child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true, + env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0', + SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '', + DISCOVERY_ANILIST_URL: anilist ? anilist.url : 'http://127.0.0.1:9/', DISCOVERY_HARVEST_AT: '' } }); + child.stdout.on('data', d => output.push(String(d))); + child.stderr.on('data', d => output.push(String(d))); + const base = `http://127.0.0.1:${port}`; + let ready = false; + for (let n = 0; n < 100 && !ready; n++) { + ready = await fetch(base + '/health').then(r => r.ok).catch(() => false); + if (!ready) await sleep(50); + } + assert.ok(ready, output.join('')); + const call = async (who, method, url, body) => { + const response = await fetch(base + url, { method, headers: { cookie: cookies[who], ...(body ? { 'content-type': 'application/json' } : {}) }, + body: body === undefined ? undefined : JSON.stringify(body) }); + return { status: response.status, body: await response.json().catch(() => null) }; + }; + return { + root, library, data, output, + get: (who, url) => call(who, 'GET', url), + post: (who, url, body = {}) => call(who, 'POST', url, body), + async stop() { + await new Promise(resolve => { if (child.exitCode !== null) return resolve(); child.once('exit', resolve); child.kill(); }); + fs.rmSync(root, { recursive: true, force: true }); + }, + }; +} + +async function until(check, what, timeoutMs = 20000) { + const deadline = Date.now() + timeoutMs; + let last; + while (Date.now() < deadline) { + last = await check(); + if (last) return last; + await sleep(100); + } + assert.fail(`Timed out waiting for ${what}`); +} +const idle = (reader, who) => until(async () => { + const { body } = await reader.get(who, '/api/discovery'); + return body.refresh.state === 'idle' ? body : null; +}, `${who}'s refresh to finish`); +const harvestDone = reader => until(async () => { + const { body } = await reader.get('boss', '/api/admin/discovery'); + return !body.currentHarvest && body.harvest && body.harvest.status !== 'running' ? body : null; +}, 'the harvest to finish'); + +function snapshot(dir) { + const out = {}; + const walk = rel => { + for (const entry of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) { + const child = path.join(rel, entry.name), stat = fs.statSync(path.join(dir, child)); + out[child] = entry.isDirectory() ? 'dir' : `${stat.size}:${stat.mtimeMs}`; + if (entry.isDirectory()) walk(child); + } + }; + walk(''); + return out; +} + +test('refresh requires opt-in, queues, runs and rescores only the caller', { timeout: 60000 }, async () => { + const anilist = await startAniList(); + const reader = await startReader({ anilist }); + try { + assert.equal((await reader.post('boss', '/api/admin/discovery/run')).status, 200); + await until(async () => (await reader.get('boss', '/api/admin/discovery')).body.currentRun?.status === 'success', 'the first run'); + const bobBefore = (await reader.get('bob', '/api/discovery')).body; + assert.ok(bobBefore.generatedAt); + assert.deepEqual(bobBefore.refresh, { state: 'idle' }); + + const refused = await reader.post('carol', '/api/discovery/refresh'); + assert.equal(refused.status, 403); + + const accepted = await reader.post('alice', '/api/discovery/refresh'); + assert.equal(accepted.status, 200, JSON.stringify(accepted.body)); + assert.match(accepted.body.harvestId, /^[0-9a-f]{16}$/); + assert.equal(accepted.body.position, 0); + + const alice = await idle(reader, 'alice'); + assert.ok(alice.items.some(item => item.candidateId === 'anilist:201'), JSON.stringify(alice.items.map(i => i.candidateId))); + const rec = alice.items.find(item => item.candidateId === 'anilist:201'); + assert.ok(rec.reasons.some(reason => /recommend/.test(reason))); + assert.ok(rec.candidate.fetchedAt); + assert.deepEqual(anilist.seen.sort(), ['fresh', 'recs', 'title']); + + const bobAfter = (await reader.get('bob', '/api/discovery')).body; + assert.equal(bobAfter.generatedAt, bobBefore.generatedAt); + assert.deepEqual(bobAfter.items.map(i => i.candidateId), bobBefore.items.map(i => i.candidateId)); + + const admin = (await reader.get('boss', '/api/admin/discovery')).body; + assert.equal(admin.harvest.kind, 'user'); + assert.equal(admin.harvest.user, 'alice'); + assert.equal(admin.harvest.status, 'success'); + assert.equal(admin.harvest.requests, 3); + assert.equal(admin.harvest.budget, 60); + assert.ok(admin.cache.files >= 3 && admin.cache.bytes > 0, JSON.stringify(admin.cache)); + assert.ok(fs.existsSync(path.join(reader.data, 'discovery-cache'))); + } finally { await reader.stop(); await anilist.close(); } +}); + +test('second refresh within an hour is 429 with nextAllowedAt', { timeout: 60000 }, async () => { + const anilist = await startAniList({ delayMs: 400 }); + const reader = await startReader({ anilist }); + try { + assert.equal((await reader.post('bob', '/api/discovery/refresh')).status, 200); + const duplicate = await reader.post('bob', '/api/discovery/refresh'); + assert.equal(duplicate.status, 409); + const state = await idle(reader, 'bob'); + assert.ok(Date.parse(state.refresh.nextAllowedAt) > Date.now() + 50 * 60000); + + const cooling = await reader.post('bob', '/api/discovery/refresh'); + assert.equal(cooling.status, 429); + assert.equal(cooling.body.nextAllowedAt, state.refresh.nextAllowedAt); + assert.equal(typeof cooling.body.error, 'string'); + } finally { await reader.stop(); await anilist.close(); } +}); + +test('admin harvest is admin-only and single-flight', { timeout: 60000 }, async () => { + const anilist = await startAniList({ delayMs: 500 }); + const reader = await startReader({ anilist }); + try { + assert.equal((await reader.post('alice', '/api/admin/discovery/harvest')).status, 403); + const first = await reader.post('boss', '/api/admin/discovery/harvest'); + assert.equal(first.status, 200, JSON.stringify(first.body)); + assert.equal(first.body.position, 0); + assert.equal((await reader.post('boss', '/api/admin/discovery/harvest')).status, 409); + const running = (await reader.get('boss', '/api/admin/discovery')).body; + assert.equal(running.currentHarvest.id, first.body.harvestId); + assert.equal(running.currentHarvest.kind, 'admin'); + const done = await harvestDone(reader); + assert.equal(done.harvest.kind, 'admin'); + assert.equal(done.harvest.budget, 300); + // Afterwards every opted-in user was rescored. + for (const who of ['alice', 'bob']) assert.ok((await reader.get(who, '/api/discovery')).body.generatedAt, who); + } finally { await reader.stop(); await anilist.close(); } +}); + +test('settings validation', { timeout: 30000 }, async () => { + const reader = await startReader(); + try { + const before = (await reader.get('boss', '/api/admin/discovery')).body; + assert.deepEqual(before.settings, { harvestEnabled: false, harvestAt: '03:30' }); + assert.equal(before.nextNightlyAt, null); + assert.equal((await reader.post('alice', '/api/admin/discovery/settings', { harvestEnabled: true, harvestAt: '04:00' })).status, 403); + for (const body of [{ harvestEnabled: true, harvestAt: '25:00' }, { harvestEnabled: true, harvestAt: '4:00' }, + { harvestEnabled: 'yes', harvestAt: '04:00' }, { harvestEnabled: true }, { harvestEnabled: true, harvestAt: '04:00', extra: 1 }]) { + const response = await reader.post('boss', '/api/admin/discovery/settings', body); + assert.equal(response.status, 400, JSON.stringify(body)); + } + const saved = await reader.post('boss', '/api/admin/discovery/settings', { harvestEnabled: true, harvestAt: '23:59' }); + assert.equal(saved.status, 200); + const after = (await reader.get('boss', '/api/admin/discovery')).body; + assert.deepEqual(after.settings, { harvestEnabled: true, harvestAt: '23:59' }); + const next = new Date(after.nextNightlyAt); + // The next slot today at 23:59, or "due now" when the test happens to run inside that minute. + const atSlot = next.getHours() === 23 && next.getMinutes() === 59 && next.getTime() > Date.now() - 120000; + assert.ok(atSlot || Math.abs(next.getTime() - Date.now()) < 120000, after.nextNightlyAt); + } finally { await reader.stop(); } +}); + +test('interrupted harvest is marked on startup', { timeout: 30000 }, async () => { + const reader = await startReader({ discovery: { harvest: { id: 'abc', kind: 'nightly', user: null, status: 'running', + startedAt: '2026-10-24T03:30:00.000Z', requests: 12, budget: 300 } } }); + try { + const { body } = await reader.get('boss', '/api/admin/discovery'); + assert.equal(body.harvest.status, 'interrupted'); + assert.equal(body.harvest.id, 'abc'); + assert.equal(body.harvest.requests, 12); + assert.ok(Date.parse(body.harvest.finishedAt)); + assert.equal(body.currentHarvest, null); + } finally { await reader.stop(); } +}); + +test('harvest never touches scraper queue or library', { timeout: 60000 }, async () => { + const anilist = await startAniList(); + const reader = await startReader({ anilist }); + try { + const libraryBefore = snapshot(reader.library), scraperBefore = snapshot(path.join(reader.data, 'scraper')); + assert.equal((await reader.post('alice', '/api/discovery/refresh')).status, 200); + await idle(reader, 'alice'); + assert.equal((await reader.post('boss', '/api/admin/discovery/harvest')).status, 200); + await harvestDone(reader); + assert.deepEqual(snapshot(reader.library), libraryBefore); + assert.deepEqual(snapshot(path.join(reader.data, 'scraper')), scraperBefore); + } finally { await reader.stop(); await anilist.close(); } +});