diff --git a/discovery/anchors.js b/discovery/anchors.js index 3e4bb03..236d3d0 100644 --- a/discovery/anchors.js +++ b/discovery/anchors.js @@ -47,8 +47,10 @@ function topTagsFor(anchors, recordOf, { mature = false } = {}) { const scores = new Map(); for (const anchor of anchors) { const record = recordOf(anchor.anilistId); - for (const tag of record?.tags || []) { - if (tag.rank < MIN_TAG_RANK || (tag.isAdult && !mature)) continue; + for (const tag of Array.isArray(record?.tags) ? record.tags : []) { + // A tag without a rank is no evidence of taste, so it must not pass the rank filter. + if (!tag || typeof tag !== 'object' || typeof tag.name !== 'string') continue; + if (!(tag.rank >= MIN_TAG_RANK) || (tag.isAdult && !mature)) continue; const entry = scores.get(tag.name) || { name: tag.name, kind: tag.kind, score: 0 }; entry.score += anchor.weight; scores.set(tag.name, entry); diff --git a/discovery/anilist.js b/discovery/anilist.js index e4ea4c7..588d605 100644 --- a/discovery/anilist.js +++ b/discovery/anilist.js @@ -64,7 +64,8 @@ function normalizeMedia(media) { ...(media.genres || []).map(name => ({ name, rank: 100, kind: 'genre', isAdult: false })), ...(media.tags || []).map(tag => ({ name: tag.name, rank: tag.rank, kind: 'tag', isAdult: !!tag.isAdult }))], authors: (media.staff?.edges || []) - .filter(edge => STAFF_ROLE.test(edge.role || '') && edge.node?.name?.full) + // An author without an integer id could never match a creator seed. + .filter(edge => STAFF_ROLE.test(edge?.role || '') && edge.node?.name?.full && Number.isInteger(edge.node.id)) .map(edge => ({ staffId: edge.node.id, name: edge.node.name.full, role: edge.role })), coverUrl: media.coverImage?.extraLarge || null, siteUrl: media.siteUrl || null, diff --git a/discovery/cache.js b/discovery/cache.js index 50238fc..02831e6 100644 --- a/discovery/cache.js +++ b/discovery/cache.js @@ -71,10 +71,14 @@ class DiscoveryCache { } return { files, bytes }; } - // Oldest entries go first so the cache never grows past maxFiles. + // Oldest entries go first so the cache never grows past maxFiles. Every put() calls this, so + // under the cap it only counts directory entries; files are read only when something must go. prune() { + const byKind = KINDS.map(kind => [kind, this.files(kind)]); + if (byKind.reduce((sum, [, files]) => sum + files.length, 0) <= this.maxFiles) return; const all = []; - for (const kind of KINDS) for (const file of this.files(kind)) { + for (const [kind, files] of byKind) for (const file of files) { + // readFile drops corrupt and schema-mismatched files itself. const entry = this.readFile(file, kind); if (entry) all.push({ file, at: Date.parse(entry.fetchedAt) }); } diff --git a/discovery/cli.js b/discovery/cli.js index 9498ea7..a6e4df7 100644 --- a/discovery/cli.js +++ b/discovery/cli.js @@ -22,7 +22,8 @@ const greatest = loadCatalog(arg('--catalog', path.join(__dirname, '..', 'greate const harvestDir = path.join(data, 'discovery-cache'); const harvestCache = fs.existsSync(harvestDir) ? new DiscoveryCache(harvestDir) : null; const result = generate({ db, library, greatest, cacheDir: path.join(data, 'scraper', 'metadata', 'anilist'), harvestCache }); -const summary = { examined: result.examined, candidates: Object.keys(result.candidates).length, +// candidates: the ones some feed shows (what a run would publish); examined/clipped: the whole pool. +const summary = { examined: result.examined, clipped: result.clipped, candidates: Object.keys(result.candidates).length, optedInUsers: Object.values(db.users || {}).filter(u => u.settings?.discoveryEnabled).length, feedSizes: Object.values(result.feeds).filter(f => !f.disabled).map(f => f.items.length) }; if (process.argv.includes('--show')) summary.titles = Object.values(result.feeds).find(f => !f.disabled)?.items diff --git a/discovery/engine.js b/discovery/engine.js index ee4a15a..60730ce 100644 --- a/discovery/engine.js +++ b/discovery/engine.js @@ -12,6 +12,9 @@ const MATURE_TAG_RANK = 60; const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']); const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null; const unique = values => [...new Set(values.filter(Boolean))]; +// manga.json and cached metadata are hand-editable files: a field that is not a list counts as empty. +const listOf = value => Array.isArray(value) ? value : []; +const stringsOf = value => unique(listOf(value).filter(item => typeof item === 'string')); function cachedMetadata(dir, id) { if (!Number.isSafeInteger(Number(id)) || Number(id) < 1) return null; @@ -29,7 +32,7 @@ function candidateFromGreatest(entry, cacheDir) { type: cached?.['x-anilist']?.format || 'MANGA', status: cached?.status || entry.status || 'unknown', cover: safeUrl(cached?.['x-anilist']?.coverUrl) || safeUrl(entry.cover), metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(entry.metadata?.url) || null, - tags: unique(cached?.tags || []), authors: unique(cached?.authors || []), + tags: stringsOf(cached?.tags), authors: stringsOf(cached?.authors), period: entry.period, editorialRank: entry.rank, greatestId: entry.id, evidence: [{ id: `greatest:${entry.id}`, kind: 'curation', label: 'Greatest of All Time selection', note: String(entry.note || '').slice(0, 600), url: safeUrl(entry.source?.url) || safeUrl(entry.metadata?.url) }], @@ -37,7 +40,8 @@ function candidateFromGreatest(entry, cacheDir) { }; } function addRelations(map, anchor, cacheDir) { - for (const relation of anchor.relations || []) { + for (const relation of listOf(anchor.relations)) { + if (!relation || typeof relation !== 'object') continue; if (!VALID_FORMATS.has(relation.format) || !Number.isSafeInteger(Number(relation.id))) continue; const id = `anilist:${Number(relation.id)}`; if (map.has(id)) continue; @@ -47,11 +51,11 @@ function addRelations(map, anchor, cacheDir) { year: cached?.year || null, type: relation.format, status: cached?.status || 'unknown', cover: safeUrl(cached?.['x-anilist']?.coverUrl), metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}`, - tags: unique(cached?.tags || []), authors: unique(cached?.authors || []), period: null, + tags: stringsOf(cached?.tags), authors: stringsOf(cached?.authors), period: null, editorialRank: null, greatestId: null, evidence: [{ id: `relation:${anchor.id}:${id}`, kind: 'relation', label: `${relation.type || 'RELATED'} to ${anchor.title}`, note: '', url: safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}` }], - relatedTo: { id: anchor.id, title: anchor.title, type: relation.type || 'RELATED' }, + relatedTo: { id: anchor.id, title: anchor.title, type: String(relation.type || 'RELATED') }, relations: cached?.['x-anilist']?.relations || [], }); } @@ -139,12 +143,12 @@ function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [ for (const tag of candidate?.tags || []) likedTags.set(tag.toLowerCase(), anchor); } const ownedTags = new Map(); - for (const manga of library) for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') ownedTags.set(tag.toLowerCase(), manga.title); + for (const manga of library) for (const tag of stringsOf(manga.meta?.tags)) ownedTags.set(tag.toLowerCase(), manga.title); const readTags = new Map(); if (settings.discoveryReadingActivity) for (const manga of library) { const progress = db.progress?.[user.name]?.[manga.id] || {}; if (!Object.values(progress).some(chapter => chapter?.done)) continue; - for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') readTags.set(tag.toLowerCase(), manga.title); + for (const tag of stringsOf(manga.meta?.tags)) readTags.set(tag.toLowerCase(), manga.title); } const seeds = seedIndexFor({ anchors: anchorsFor({ user, db, library, greatest }), harvestCache, mature: !!settings.discoveryMature }); const rows = []; @@ -217,8 +221,14 @@ function candidatePool({ greatest, library, cacheDir, harvestCache }) { } return { all, candidates }; } -const runResult = (all, candidates, feeds) => ({ candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds, - examined: all.length, clipped: Math.max(0, all.length - candidates.length) }); +// Only candidates a feed shows are published; the rest of the pool (up to 2000, with AniList +// records) would bloat discovery.json for nothing. examined/clipped still describe the whole pool. +function runResult(all, candidates, feeds) { + const shown = new Set(); + for (const feed of Object.values(feeds)) for (const item of feed?.items || []) shown.add(item.candidateId); + return { candidates: Object.fromEntries(candidates.filter(c => shown.has(c.id)).map(c => [c.id, c])), feeds, + examined: all.length, clipped: Math.max(0, all.length - candidates.length) }; +} function generate({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], now }) { const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache }); const feeds = {}; @@ -254,5 +264,5 @@ function mergeUserFeed(prior, result, userName) { return { candidates, feeds }; } -module.exports = { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, cachedMetadata, +module.exports = { generate, generateAsync, buildCandidates, candidatePool, feedForUser, mixSlots, mergeUserFeed, cachedMetadata, isMature, MAX_CANDIDATES, MAX_FEED, SLOTS }; \ No newline at end of file diff --git a/discovery/scoring.js b/discovery/scoring.js index deae4ad..c31cb82 100644 --- a/discovery/scoring.js +++ b/discovery/scoring.js @@ -17,6 +17,8 @@ const PART_ORDER = ['communityRec', 'sameCreator', 'tagTop', 'fresh', const round2 = value => Math.round(value * 100) / 100; const listOf = value => Array.isArray(value) ? value : []; +// Cached lists come from disk; anything that is not an object is skipped, never dereferenced. +const objectsOf = value => listOf(value).filter(item => item && typeof item === 'object'); const partRank = part => { const index = PART_ORDER.indexOf(part); return index < 0 ? PART_ORDER.length : index; }; function communityRecValue(rating, weight) { @@ -34,13 +36,13 @@ function seedIndexFor({ anchors, harvestCache, mature = false }) { for (const anchor of anchors) { const name = nameOf(anchor); const entry = harvestCache.get('recs', anchor.anilistId); - for (const item of listOf(entry?.data)) { - const id = `anilist:${Number(item?.anilistId)}`; + for (const item of objectsOf(entry?.data)) { + const id = `anilist:${Number(item.anilistId)}`; if (!index.recs.has(id)) index.recs.set(id, []); index.recs.get(id).push({ anchor, name, rating: Number(item.rating) || 0, record: item.record || null, fetchedAt: entry.fetchedAt }); } if (anchor.weight < CREATOR_MIN_WEIGHT) continue; - for (const author of listOf(recordOf(anchor.anilistId)?.authors)) { + for (const author of objectsOf(recordOf(anchor.anilistId)?.authors)) { const prior = index.creators.get(author.staffId); if (prior && prior.anchor.weight >= anchor.weight) continue; index.creators.set(author.staffId, { anchor, name, author, @@ -50,15 +52,15 @@ function seedIndexFor({ anchors, harvestCache, mature = false }) { index.topTags = topTagsFor(anchors, recordOf, { mature }); index.topTags.forEach((tag, place) => { const entry = harvestCache.get('tagTop', `${tag.kind}:${tag.name}`); - for (const item of listOf(entry?.data)) { - const id = `anilist:${Number(item?.anilistId)}`; + for (const item of objectsOf(entry?.data)) { + const id = `anilist:${Number(item.anilistId)}`; // Tags come best-first, so the first hit is the strongest. if (!index.tagTop.has(id)) index.tagTop.set(id, { tag, value: 3 - place, record: item.record || null, fetchedAt: entry.fetchedAt }); } }); const fresh = harvestCache.get('fresh', 'global'); - for (const item of listOf(fresh?.data)) { - index.fresh.set(`anilist:${Number(item?.anilistId)}`, { record: item.record || null, fetchedAt: fresh.fetchedAt }); + for (const item of objectsOf(fresh?.data)) { + index.fresh.set(`anilist:${Number(item.anilistId)}`, { record: item.record || null, fetchedAt: fresh.fetchedAt }); } return index; } @@ -80,8 +82,8 @@ function seedNotes(candidate, seeds, thisYear) { url: urlFor(bestRec.record), fetchedAt: bestRec.fetchedAt } }); let bestCreator = null; - for (const author of listOf(candidate.authorRecords)) { - const hit = seeds.creators.get(author?.staffId); + for (const author of objectsOf(candidate.authorRecords)) { + const hit = seeds.creators.get(author.staffId); // An anchor is never its own "also by" suggestion. if (!hit || `anilist:${hit.anchor.anilistId}` === candidate.id) continue; if (!bestCreator || hit.anchor.weight > bestCreator.anchor.weight) bestCreator = { ...hit, candidateAuthor: author }; diff --git a/docs/architecture.md b/docs/architecture.md index 1599ded..4639631 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -251,6 +251,8 @@ The reader alone talks to AniList for discovery ([discovery.md](discovery.md)). startedAt, finishedAt, requests, budget, stopReason }`. `stopReason` is `budget`, `rate-limited`, `forbidden`, `challenge`, `upstream-errors` or `error` (a local fault). A run cut off by a restart is marked `interrupted` at startup; queued refreshes are not persisted. +- **Published candidates:** only those at least one feed shows. `lastRun.candidateCount` (the admin + candidate count) counts that set; `lastRun.examined` and `clipped` still describe the whole pool. - **Published fields:** candidates found through AniList carry `tagRecords`, `authorRecords`, `seed` (`recs|creator|tagTop|fresh`) and `fetchedAt`; feed items carry `group` (seed kind or period) and `reasons` (all reasons, strongest first) next to `reason`; seed evidence carries `fetchedAt`. diff --git a/docs/discovery.md b/docs/discovery.md index 44b467e..17a3566 100644 --- a/docs/discovery.md +++ b/docs/discovery.md @@ -45,12 +45,13 @@ count. It then asks AniList for four seed kinds: | Community recommendations | "readers who liked X recommend Y" for each anchor | AniList readers who liked {anchor} recommend this ({rating}). | | Same creator | other works by the story/art authors of strong anchors | Also by {author} — you read/liked {anchor}. | | Top in your tags | well-rated titles in the user's three strongest tags | Top-rated in {tag}, one of your most-liked tags. | -| Fresh | recently started, releasing and trending titles | New: started {year}, ongoing and trending. | +| Fresh | recently started, releasing and trending titles | New: started {year}, ongoing and trending. — or, when the cached record does not show an ongoing title started in the last two years, Trending on AniList right now. | A title found by several seeds keeps every reason and evidence item; the card shows the strongest, the details list all of them, each with the date it was -fetched. A feed holds at most 20 suggestions, at most 8 from one seed kind, up to -4 fresh titles, and some from outside your strongest tags. +fetched. A feed holds at most 20 suggestions, at most 8 from one seed kind, at most +4 from one Greatest period (or from related works), up to 4 fresh titles, and up to 4 +from outside your strongest tags. - **Budgets:** a nightly or admin harvest makes at most 300 requests, a per-user refresh 60. Requests are sequential and limited to 25 a minute. When @@ -65,8 +66,8 @@ fetched. A feed holds at most 20 suggestions, at most 8 from one seed kind, up t the nightly harvest off. A night missed while the reader was down runs once after startup. - **Privacy:** for opted-in users only, the server sends AniList the IDs of - their anchor titles (and the names of their top tags), without login, cookies or - user names. Feeds and feedback stay private to this server; cache entries are + their anchor titles (liked, read or owned), the AniList staff IDs of those titles' + authors and the names of their top tags, without login, cookies or user names. Feeds and feedback stay private to this server; cache entries are not tied to a user. ### Stop reasons @@ -88,8 +89,9 @@ re-scoring failure after a harvest is shown as the admin panel's error line. ## Data and API -- `data/discovery.json`: atomic, reader-owned candidate and per-user feed - snapshots plus the last harvest record and the admin's harvest settings. A +- `data/discovery.json`: atomic, reader-owned per-user feed snapshots and the + candidates those feeds show (the rest of the pool is only counted: + `examined`, `clipped`), plus the last harvest record and the admin's harvest settings. A previous valid version is retained as `discovery.json.prev`. A corrupt file makes reads/runs/harvests fail closed rather than overwrite it. - `data/discovery-cache/`: harvested AniList responses; safe to delete. @@ -115,7 +117,9 @@ APP_IMAGE=mangacircus:discovery-test npm run test:discovery-ui ``` The dry run reads `db.json`, a saved library index if present, Greatest, and -cached AniList metadata. It writes nothing and does not contact providers. The +cached AniList metadata. It does not contact AniList or any other provider and does +not publish `discovery.json`; reading the harvest cache may still delete a corrupt +cache file, as the reader would. The Compose/browser suite uses a disposable data tree and queue, not the live one, and points `DISCOVERY_ANILIST_URL` at a closed local port so no harvest reaches AniList. diff --git a/docs/superpowers/plans/2026-10-25-discovery-wider-pool.md b/docs/superpowers/plans/2026-10-25-discovery-wider-pool.md index 29b49fe..08da8dd 100644 --- a/docs/superpowers/plans/2026-10-25-discovery-wider-pool.md +++ b/docs/superpowers/plans/2026-10-25-discovery-wider-pool.md @@ -334,4 +334,4 @@ - [ ] **Step 3: Implement** the UI and docs. - [ ] **Step 4: Run** the same two commands, then `npm test`. Expected: PASS. - [ ] **Step 5: Commit** `Discover page: refresh, harvest admin and seed reasons; docs (BL-0051)` and push. -- [ ] **Step 6: Update MangaOrg**: run `python D:\Projects\Manga\MangaOrg\tools\org.py` to set BL-0051 to `review`, release the `MangaCircus discovery` claim, then commit and push MangaOrg. Merging into `main` waits for the user. +- [ ] **Step 6: Update the project's coordination backlog item and release the claim.** Merging into `main` waits for the user. diff --git a/docs/superpowers/specs/2026-10-25-discovery-wider-pool-design.md b/docs/superpowers/specs/2026-10-25-discovery-wider-pool-design.md index ab104f6..64930c5 100644 --- a/docs/superpowers/specs/2026-10-25-discovery-wider-pool-design.md +++ b/docs/superpowers/specs/2026-10-25-discovery-wider-pool-design.md @@ -1,6 +1,6 @@ # Discovery: wider candidate pool — design -Status: approved design, not implemented. Date: 2026-10-25. Builds on the +Status: implemented on branch feature/discovery-wider-pool. Date: 2026-10-25. Builds on the deterministic Discovery Scout ([`docs/discovery.md`](../../discovery.md), [`vision/discovery-scout.md`](../../../vision/discovery-scout.md)). diff --git a/public/discover.js b/public/discover.js index 0321052..528a2da 100644 --- a/public/discover.js +++ b/public/discover.js @@ -49,7 +49,7 @@ const refreshState = state.refresh || { state: 'idle' }; const label = refreshState.state === 'queued' ? `Queued (position ${refreshState.position})` : refreshState.state === 'running' ? 'Fetching…' - : refreshState.blockedReason ? `Paused by AniList (${refreshState.blockedReason}) until the next nightly run` + : refreshState.blockedReason ? `Paused by AniList (${refreshState.blockedReason}) until the next nightly or admin harvest` : refreshState.nextAllowedAt ? `Next refresh possible at ${clock(refreshState.nextAllowedAt)}` : null; return h('button', { type: 'button', class: 'btn discovery-refresh', disabled: !!label, onclick: refreshMine }, label || 'Refresh my suggestions'); } @@ -60,7 +60,7 @@ } function preferences() { return h('section', { class: 'card discovery-settings' }, h('h2', {}, 'Your discovery preferences'), - toggle('Personalized discovery', 'discoveryEnabled', 'Off by default. When on, the server sends AniList the IDs of titles you liked, without login, to find related work. Your feed and feedback stay private to this server.'), + toggle('Personalized discovery', 'discoveryEnabled', 'Off by default. When on, the server sends AniList the IDs of titles you liked, read or own, their authors\' IDs and your top tags, without login, to find related work. Your feed and feedback stay private to this server.'), toggle('Use reading activity', 'discoveryReadingActivity', 'Separately opt in to using completed chapters as a small taste signal.'), toggle('Include mature titles', 'discoveryMature', 'Only affects titles AniList marks as adult or with a known mature tag.')); } diff --git a/server.js b/server.js index b35428f..242f203 100644 --- a/server.js +++ b/server.js @@ -317,8 +317,10 @@ try { const harvester = new Harvester({ cache: harvestCache, client: createAniListClient(), store: discoveryStore, targetsFor: harvestTargets, onDone: async job => { // A failed follow-up rescore must reach the admin panel (lastError), not only the log. - try { await rescore(job.kind === 'user' ? { onlyUser: job.user } : {}); } - catch (error) { discoveryError = `Re-scoring after harvest ${job.id} failed: ${String(error.message || error)}`.slice(0, 300); throw error; } + try { + await rescore(job.kind === 'user' ? { onlyUser: job.user } : {}); + discoveryError = null; // a later success must not leave an old failure on the admin panel + } catch (error) { discoveryError = `Re-scoring after harvest ${job.id} failed: ${String(error.message || error)}`.slice(0, 300); throw error; } } }); function requestHarvest(args) { readDiscovery(); // A corrupt store fails closed before any AniList call. diff --git a/test/discovery-anchors.test.js b/test/discovery-anchors.test.js index b9569e8..a545ca9 100644 --- a/test/discovery-anchors.test.js +++ b/test/discovery-anchors.test.js @@ -60,3 +60,14 @@ test('top tags weigh anchors, skip low rank and adult tags', () => { assert.deepEqual(topTagsFor(anchors, recordOf, { mature: true }), [ { name: 'Action', kind: 'genre' }, { name: 'Ecchi', kind: 'tag' }, { name: 'Alpha', kind: 'tag' }]); }); + +test('top tags drop tags without a rank and survive malformed records', () => { + const records = { + 1: { tags: [{ name: 'NoRank', kind: 'tag', isAdult: false }, { name: 'NullRank', rank: null, kind: 'tag', isAdult: false }, + null, 7, { name: 'Kept', rank: 60, kind: 'tag', isAdult: false }] }, + 2: { tags: 5 }, + 3: { tags: { name: 'Object', rank: 90 } }, + }; + const anchors = [{ anilistId: 1, weight: 5 }, { anilistId: 2, weight: 3 }, { anilistId: 3, weight: 3 }]; + assert.deepEqual(topTagsFor(anchors, id => records[id]), [{ name: 'Kept', kind: 'tag' }]); +}); diff --git a/test/discovery-anilist.test.js b/test/discovery-anilist.test.js index 57f3ce3..c30edce 100644 --- a/test/discovery-anilist.test.js +++ b/test/discovery-anilist.test.js @@ -34,6 +34,14 @@ test('normalizeMedia maps fields and drops novels', () => { assert.equal(normalizeMedia({ ...media, status: 'HIATUS' }).status, 'unknown'); }); +test('normalizeMedia drops authors without an integer staff id', () => { + const media = fixture('title').data.Media; + const edge = (id, full) => ({ role: 'Story', node: { id, name: { full } } }); + const record = normalizeMedia({ ...media, staff: { edges: [edge(5, 'Has Id'), edge(undefined, 'No Id'), + edge('6', 'String Id'), edge(1.5, 'Float Id'), edge(null, 'Null Id')] } }); + assert.deepEqual(record.authors, [{ staffId: 5, name: 'Has Id', role: 'Story' }]); +}); + test('title returns a record and sends the fixed headers', async () => { const { client, calls } = stub(() => json(fixture('title'))); const record = await client.title(101); diff --git a/test/discovery-cache.test.js b/test/discovery-cache.test.js index 710688f..b738557 100644 --- a/test/discovery-cache.test.js +++ b/test/discovery-cache.test.js @@ -49,6 +49,48 @@ test('prune keeps the newest maxFiles', (t) => { assert.notEqual(cache.get('title', 5), null); }); +test('prune removes the oldest entries across kinds', (t) => { + const { clock, cache } = setup(t, { maxFiles: 3 }); + const kinds = ['title', 'recs', 'creator', 'fresh', 'tagTop']; + kinds.forEach((kind, i) => { clock.t += 1000; cache.put(kind, i, { i }); }); + assert.equal(cache.stats().files, 3); + assert.equal(cache.get('title', 0), null); + assert.equal(cache.get('recs', 1), null); + for (const [i, kind] of kinds.entries()) if (i >= 2) assert.notEqual(cache.get(kind, i), null, kind); +}); + +test('a write under the cap does not read other entries', (t) => { + const { dir, cache } = setup(t, { maxFiles: 10 }); + cache.put('recs', 7, {}); + const file = path.join(dir, 'recs', fs.readdirSync(path.join(dir, 'recs'))[0]); + fs.writeFileSync(file, '{broken'); + cache.put('title', 1, {}); + assert.equal(fs.readFileSync(file, 'utf8'), '{broken'); +}); + +test('over the cap, corrupt and schema-mismatch entries go before valid ones', (t) => { + const { dir, clock, cache } = setup(t, { maxFiles: 2 }); + cache.put('recs', 1, {}); + cache.put('recs', 2, {}); + const [first, second] = fs.readdirSync(path.join(dir, 'recs')).map(name => path.join(dir, 'recs', name)); + fs.writeFileSync(first, '{broken'); + fs.writeFileSync(second, JSON.stringify({ schema: 2, kind: 'recs', key: 2, fetchedAt: new Date(clock.t).toISOString(), data: {} })); + clock.t += 1000; + cache.put('title', 3, {}); + assert.equal(fs.existsSync(first), false); + assert.equal(fs.existsSync(second), false); + assert.notEqual(cache.get('title', 3), null); +}); + +test('schema-mismatch entry is deleted like a corrupt one', (t) => { + const { dir, clock, cache } = setup(t); + cache.put('recs', 7, {}); + const file = path.join(dir, 'recs', fs.readdirSync(path.join(dir, 'recs'))[0]); + fs.writeFileSync(file, JSON.stringify({ schema: 2, kind: 'recs', key: 7, fetchedAt: new Date(clock.t).toISOString(), data: {} })); + assert.equal(cache.get('recs', 7), null); + assert.equal(fs.existsSync(file), false); +}); + test('corrupt entry is deleted', (t) => { const { dir, cache } = setup(t); cache.put('recs', 7, {}); diff --git a/test/discovery-server.test.js b/test/discovery-server.test.js index 3f8b161..e85b2ed 100644 --- a/test/discovery-server.test.js +++ b/test/discovery-server.test.js @@ -50,7 +50,7 @@ const sha = s => crypto.createHash('sha256').update(s).digest('hex'); const USERS = { boss: { admin: true, discovery: false }, alice: { discovery: true }, bob: { discovery: true }, carol: { discovery: false } }; // A disposable reader with ready-made accounts and sessions; nightly harvests are off unless the store says otherwise. -async function startReader({ anilist, discovery = {}, mangaMeta = null } = {}) { +async function startReader({ anilist, discovery = {}, mangaMeta = null, scraperUrl = 'http://127.0.0.1:9' } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-server-')); const library = path.join(root, 'library'), data = path.join(root, 'data'); fs.mkdirSync(path.join(library, 'Example Manga'), { recursive: true }); @@ -80,7 +80,7 @@ async function startReader({ anilist, discovery = {}, mangaMeta = null } = {}) { const port = await freePort(), output = []; const child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true, env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0', - SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '', + SCRAPER_URL: scraperUrl, SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '', DISCOVERY_ANILIST_URL: anilist ? anilist.url : 'http://127.0.0.1:9/', DISCOVERY_HARVEST_AT: '' } }); child.stdout.on('data', d => output.push(String(d))); child.stderr.on('data', d => output.push(String(d))); @@ -216,22 +216,55 @@ test('admin harvest is admin-only and single-flight', { timeout: 60000 }, async } finally { await reader.stop(); await anilist.close(); } }); +// A scraper stand-in. The rescore asks it for the queue after the harvest's final store patch and +// right before publishing, so onState can break discovery.json at exactly that point. +async function startScraper(onState) { + const server = http.createServer((req, res) => { + if (req.url.startsWith('/api/state')) onState(); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ queue: [] })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + return { url: `http://127.0.0.1:${server.address().port}`, close: () => new Promise(resolve => server.close(resolve)) }; +} + test('a failed rescore after a harvest reaches the admin as lastError', { timeout: 60000 }, async () => { const anilist = await startAniList(); - // A manga.json whose tags is not a list makes scoring throw; it stands in for any scoring fault. - const reader = await startReader({ anilist, mangaMeta: { title: 'Broken Tags', tags: 5 } }); + let reader, armed = false, saved = null; + const scraper = await startScraper(() => { + if (!armed) return; + armed = false; + const file = path.join(reader.data, 'discovery.json'); + saved = fs.readFileSync(file, 'utf8'); + fs.writeFileSync(file, '{broken'); // the publish now refuses to overwrite it + }); + // A malformed manga.json (tags not a list, relations an object) must not be what breaks scoring. + reader = await startReader({ anilist, scraperUrl: scraper.url, + mangaMeta: { title: 'Broken Tags', tags: 5, 'x-anilist': { id: 101, relations: {} } } }); try { await until(async () => (await reader.get('boss', '/api/library')).body?.mangas?.length === 1, 'the library scan'); - assert.equal((await reader.post('boss', '/api/admin/discovery/harvest')).status, 200); - const done = await harvestDone(reader); - assert.equal(done.harvest.kind, 'admin'); + armed = true; + const first = await reader.post('boss', '/api/admin/discovery/harvest'); + assert.equal(first.status, 200); + await until(async () => saved && reader.output.join('').includes(`${first.body.harvestId} follow-up failed`), 'the failed rescore'); + fs.writeFileSync(path.join(reader.data, 'discovery.json'), saved); const admin = await until(async () => { const { body } = await reader.get('boss', '/api/admin/discovery'); return body.lastError ? body : null; }, 'the rescore error'); - assert.match(admin.lastError, /^Re-scoring after harvest [0-9a-f]{16} failed: /); + assert.match(admin.lastError, /^Re-scoring after harvest [0-9a-f]{16} failed: Discovery store unavailable/); assert.equal(admin.currentRun, null); - } finally { await reader.stop(); await anilist.close(); } + assert.equal(admin.harvest.kind, 'admin'); + + // The next harvest's successful rescore clears the stale error. + const second = await reader.post('boss', '/api/admin/discovery/harvest'); + assert.equal(second.status, 200, JSON.stringify(second.body)); + const cleared = await until(async () => { + const { body } = await reader.get('boss', '/api/admin/discovery'); + return body.harvest?.id === second.body.harvestId && !body.currentHarvest && body.lastError === null ? body : null; + }, 'the error to clear'); + assert.ok(cleared.candidateCount > 0); + } finally { await reader.stop(); await anilist.close(); await scraper.close(); } }); test('settings validation', { timeout: 30000 }, async () => { diff --git a/test/discovery.test.js b/test/discovery.test.js index 09b8c07..458aaad 100644 --- a/test/discovery.test.js +++ b/test/discovery.test.js @@ -4,7 +4,7 @@ const assert = require('node:assert/strict'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); -const { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, MAX_CANDIDATES, SLOTS } = require('../discovery/engine'); +const { generate, generateAsync, buildCandidates, candidatePool, feedForUser, mixSlots, mergeUserFeed, MAX_CANDIDATES, SLOTS } = require('../discovery/engine'); const { DiscoveryStore } = require('../discovery/store'); const { DiscoveryCache } = require('../discovery/cache'); @@ -16,12 +16,17 @@ test('discovery is bounded, diversified, opt-in, evidence-linked and user-specif off: { name: 'off', settings: {} } }; const db = { users, greatestRatings: { alice: { [greatest.entries[0].id]: 'flame' }, bob: { [greatest.entries[20].id]: 'flame' } }, discoveryFeedback: { alice: {} }, progress: {} }; - const result = generate({ db, library: [], greatest, cacheDir: path.join(os.tmpdir(), 'nonexistent-discovery-cache') }); - assert.equal(Object.keys(result.candidates).length, Math.min(2000, result.examined)); + const cacheDir = path.join(os.tmpdir(), 'nonexistent-discovery-cache'); + const result = generate({ db, library: [], greatest, cacheDir }); + // Only candidates a feed shows are published; the pool itself stays bounded at 2000. + const shown = new Set(Object.values(result.feeds).flatMap(feed => (feed.items || []).map(item => item.candidateId))); + assert.deepEqual(Object.keys(result.candidates).sort(), [...shown].sort()); + const pool = candidatePool({ greatest, library: [], cacheDir }); + assert.equal(pool.candidates.length, Math.min(2000, result.examined)); assert.ok(result.feeds.alice.items.length > 0 && result.feeds.alice.items.length <= 20); assert.ok(result.feeds.bob.items.length > 0); assert.equal(result.feeds.off.disabled, true); - assert.ok(new Set(Object.values(result.candidates).map(c => c.period)).size >= 10, 'An early catalog page must not crowd out later eras'); + assert.ok(new Set(pool.candidates.map(c => c.period)).size >= 10, 'An early catalog page must not crowd out later eras'); assert.ok(result.feeds.alice.items.every(item => item.evidenceIds.every(id => item.evidence.some(e => e.id === id)))); assert.ok(result.feeds.alice.items.some(item => item.evidence.some(e => e.kind === 'personal'))); assert.ok(!result.feeds.bob.items.some(item => item.evidence.some(e => e.label?.includes(greatest.entries[0].title) && e.kind === 'personal'))); @@ -144,10 +149,48 @@ test('engine: empty harvest cache equals Phase A pool', t => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-empty-')); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); const db = { users: {}, greatestRatings: {}, discoveryFeedback: {}, progress: {} }; - const without = generate({ db, library: [], greatest, cacheDir: NO_SCRAPER_CACHE }); - const withEmpty = generate({ db, library: [], greatest, cacheDir: NO_SCRAPER_CACHE, harvestCache: new DiscoveryCache(dir) }); + const without = candidatePool({ greatest, library: [], cacheDir: NO_SCRAPER_CACHE }); + const withEmpty = candidatePool({ greatest, library: [], cacheDir: NO_SCRAPER_CACHE, harvestCache: new DiscoveryCache(dir) }); assert.equal(MAX_CANDIDATES, 2000); - assert.deepEqual(Object.keys(withEmpty.candidates).sort(), Object.keys(without.candidates).sort()); + assert.deepEqual(withEmpty.candidates.map(c => c.id).sort(), without.candidates.map(c => c.id).sort()); + const users = { alice: { name: 'alice', settings: { discoveryEnabled: true } } }; + const run = harvestCache => generate({ db: { ...db, users }, library: [], greatest, cacheDir: NO_SCRAPER_CACHE, harvestCache }); + assert.deepEqual(Object.keys(run(new DiscoveryCache(dir)).candidates).sort(), Object.keys(run(null).candidates).sort()); +}); + +test('published candidates are only those referenced by a feed', async t => { + const world = fixtureWorld(t); + world.db.users.bob = { name: 'bob', settings: { discoveryEnabled: true } }; + world.db.users.off = { name: 'off', settings: {} }; + const args = { db: world.db, library: world.library, greatest: world.greatest, cacheDir: NO_SCRAPER_CACHE, + harvestCache: world.harvestCache, queuedTitles: world.queuedTitles, now: NOW }; + const referenced = result => new Set(Object.values(result.feeds).flatMap(feed => (feed.items || []).map(item => item.candidateId))); + for (const result of [generate(args), await generateAsync(args), await generateAsync({ ...args, onlyUser: 'alice' })]) { + const shown = referenced(result); + assert.ok(shown.size > 0); + for (const id of Object.keys(result.candidates)) assert.ok(shown.has(id), `${id} is published but not shown`); + for (const id of shown) assert.ok(result.candidates[id], `${id} is shown but not published`); + assert.ok(result.examined > Object.keys(result.candidates).length, 'examined still counts the whole pool'); + assert.equal(result.clipped, 0); + } +}); + +test('malformed library metadata and cache records score without throwing', t => { + const world = fixtureWorld(t, { discoveryReadingActivity: true }); + world.library.push( + { id: 'm2', title: 'Bad Tags', meta: { tags: 5, 'x-anilist': { id: 220, relations: {} } } }, + { id: 'm3', title: 'Bad Relations', meta: { tags: [7, null, { name: 'Drama' }, 'Drama'], + 'x-anilist': { id: 221, relations: [null, 3, 'x', { id: 222, format: 'MANGA', type: 'SEQUEL', title: 'Sequel Title' }] } } }, + { id: 'm4', title: 'Bad Identity', meta: { tags: { a: 1 }, 'x-anilist': 5 } }); + world.db.progress = { alice: { m2: { c1: { done: true } }, m3: { c1: { done: true } } } }; + world.harvestCache.put('title', 220, seedRecord(220, { authors: [null, 5, AUTHOR], tags: [null, 'Drama', { name: 'Drama', kind: 'genre' }] })); + world.harvestCache.put('recs', 220, [null, 5, { ...seed(230), rating: 7 }]); + const args = { db: world.db, library: world.library, greatest: world.greatest, cacheDir: NO_SCRAPER_CACHE, + harvestCache: world.harvestCache, queuedTitles: world.queuedTitles, now: NOW }; + const result = generate(args); + const ids = result.feeds.alice.items.map(item => item.candidateId); + assert.ok(ids.includes('anilist:222'), JSON.stringify(ids)); + assert.ok(ids.includes('anilist:230'), JSON.stringify(ids)); }); test('harvested records join the pool and fill Greatest candidates', t => { @@ -262,6 +305,25 @@ test('slot mix', () => { assert.ok(full.filter(r => r.group === '1980s').length <= SLOTS.perPeriod); }); +test('slot mix reservations let low-scored fresh and outside rows past a full feed', () => { + const row = (id, score, group, extra = {}) => ({ candidateId: `anilist:${id}`, score, group, tags: ['Drama'], + scoreParts: { fresh: 0 }, ...extra }); + // 32 higher-scored rows that the caps alone would allow: 8 per seed kind, 4 per period. + const rows = []; + const groups = ['recs', 'creator', 'tagTop', '1980s', '1990s']; + const caps = { recs: 8, creator: 8, tagTop: 8, '1980s': 4, '1990s': 4 }; + let id = 5000; + for (const group of groups) for (let i = 0; i < caps[group]; i++) rows.push(row(id++, 100 - rows.length, group)); + assert.ok(rows.length > 20); + rows.push(row(2001, 1, 'fresh', { scoreParts: { fresh: 1 } }), row(2002, 0.9, 'fresh', { scoreParts: { fresh: 1 } })); + rows.push(row(3001, 0.2, '2000s', { tags: ['Sports'] }), row(3002, 0.1, '2000s', { tags: [] })); + const picked = mixSlots(rows, { topTags: [{ name: 'Drama', kind: 'genre' }] }); + assert.equal(picked.length, 20); + for (const low of ['anilist:2001', 'anilist:2002', 'anilist:3001', 'anilist:3002']) + assert.ok(picked.some(r => r.candidateId === low), `${low} needs its reserved slot`); + for (const group of groups) assert.ok(picked.filter(r => r.group === group).length <= caps[group], group); +}); + test('mergeUserFeed keeps other users\' referenced candidates', () => { const prior = { candidates: { a: { id: 'a' }, b: { id: 'b' }, c: { id: 'c' } }, feeds: { alice: { items: [{ candidateId: 'a' }] }, bob: { items: [{ candidateId: 'b' }] }, off: { items: [], disabled: true } } }; diff --git a/tools/test-discovery-ui.js b/tools/test-discovery-ui.js index 4078f73..389135f 100644 --- a/tools/test-discovery-ui.js +++ b/tools/test-discovery-ui.js @@ -15,19 +15,20 @@ module.exports = async function testDiscoveryUI(url, cookie, { xssTitle } = {}) await page.goto(url + '/discover.html'); await page.getByRole('heading', { name: 'Discover', exact: true }).waitFor(); await page.getByText('Turn on personalized discovery').waitFor(); - await page.getByText('the server sends AniList the IDs of titles you liked, without login').waitFor(); + await page.getByText('Off by default. When on, the server sends AniList the IDs of titles you liked, read or own, ' + + 'their authors\' IDs and your top tags, without login, to find related work. Your feed and feedback stay private to this server.').waitFor(); await page.getByLabel('Personalized discovery').check(); // Review focus 5: the seeded feed is the first one shown, before any run replaces it. - if (xssTitle) { - const seeded = page.locator('.discovery-card').filter({ hasText: 'AniList readers who liked Moon Garden' }); - await seeded.waitFor(); - assert.equal(await seeded.locator('h2').textContent(), xssTitle); - assert.equal(await page.locator('.discovery-card h2 img').count(), 0); - await seeded.getByText('Why this is here · evidence').click(); - const details = await seeded.locator('details li').allTextContents(); - assert.ok(details.some(text => text.includes('New: started 2025')), `All reasons listed: ${details}`); - assert.ok(details.some(text => /AniList recommendation.* · fetched \S+/.test(text)), `Evidence fetch date: ${details}`); - } + // The XSS check must never be skipped silently: the Compose driver always seeds the title. + assert.ok(xssTitle, 'The discovery UI test needs the seeded XSS title (run it through tools/test-compose.js --discovery-ui)'); + const seeded = page.locator('.discovery-card').filter({ hasText: 'AniList readers who liked Moon Garden' }); + await seeded.waitFor(); + assert.equal(await seeded.locator('h2').textContent(), xssTitle); + assert.equal(await page.locator('.discovery-card h2 img').count(), 0); + await seeded.getByText('Why this is here · evidence').click(); + const details = await seeded.locator('details li').allTextContents(); + assert.ok(details.some(text => text.includes('New: started 2025')), `All reasons listed: ${details}`); + assert.ok(details.some(text => /AniList recommendation.* · fetched \S+/.test(text)), `Evidence fetch date: ${details}`); await page.getByRole('button', { name: 'Run discovery now' }).click(); // The seeded store already says "success"; wait until the run's own feed replaced the seeded card. await page.waitForFunction(title => /Status: success/.test(document.querySelector('.discovery-admin')?.textContent || '')