Discovery final review fixes: lean publish, robust scoring, cheap prune (BL-0051)

- Cache prune counts directory entries first and only reads files when over
  the cap; tests for cross-kind prune, schema-mismatch and corrupt entries.
- Scoring tolerates malformed manga.json and cached records (non-list tags,
  object or null relations, non-object items); a tag without a rank no longer
  passes the top-tag filter; authors without an integer AniList id are dropped.
- discovery.json publishes only candidates some feed shows; examined/clipped
  still describe the pool. Tests that asserted the published count equals the
  pool size now check the pool via candidatePool() and the published set
  against the feeds.
- The post-harvest rescore failure test now breaks discovery.json right before
  the publish (via a scraper stand-in) instead of relying on tags: 5, and a
  following successful harvest clears the stale lastError.
- Consent text names everything sent to AniList; paused label mentions admin
  harvests; the UI test asserts the XSS seed instead of skipping it.
- Slot-mix test proves the fresh/outside reservations matter.
- Docs: privacy note, fresh fallback reason, per-period cap, dry-run wording,
  published-candidate rule; plan/spec hygiene and spec status.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

discovery/anchors.js

@@ -47,8 +47,10 @@ function topTagsFor(anchors, recordOf, { mature = false } = {}) {
   const scores = new Map();
   for (const anchor of anchors) {
     const record = recordOf(anchor.anilistId);
-    for (const tag of record?.tags || []) {
+    for (const tag of Array.isArray(record?.tags) ? record.tags : []) {
-      if (tag.rank < MIN_TAG_RANK || (tag.isAdult && !mature)) continue;
+      // A tag without a rank is no evidence of taste, so it must not pass the rank filter.
+      if (!tag || typeof tag !== 'object' || typeof tag.name !== 'string') continue;
+      if (!(tag.rank >= MIN_TAG_RANK) || (tag.isAdult && !mature)) continue;
       const entry = scores.get(tag.name) || { name: tag.name, kind: tag.kind, score: 0 };
       entry.score += anchor.weight;
       scores.set(tag.name, entry);

discovery/anilist.js

@@ -64,7 +64,8 @@ function normalizeMedia(media) {
       ...(media.genres || []).map(name => ({ name, rank: 100, kind: 'genre', isAdult: false })),
       ...(media.tags || []).map(tag => ({ name: tag.name, rank: tag.rank, kind: 'tag', isAdult: !!tag.isAdult }))],
     authors: (media.staff?.edges || [])
-      .filter(edge => STAFF_ROLE.test(edge.role || '') && edge.node?.name?.full)
+      // An author without an integer id could never match a creator seed.
+      .filter(edge => STAFF_ROLE.test(edge?.role || '') && edge.node?.name?.full && Number.isInteger(edge.node.id))
       .map(edge => ({ staffId: edge.node.id, name: edge.node.name.full, role: edge.role })),
     coverUrl: media.coverImage?.extraLarge || null,
     siteUrl: media.siteUrl || null,

discovery/cache.js

@@ -71,10 +71,14 @@ class DiscoveryCache {
     }
     return { files, bytes };
   }
-  // Oldest entries go first so the cache never grows past maxFiles.
+  // Oldest entries go first so the cache never grows past maxFiles. Every put() calls this, so
+  // under the cap it only counts directory entries; files are read only when something must go.
   prune() {
+    const byKind = KINDS.map(kind => [kind, this.files(kind)]);
+    if (byKind.reduce((sum, [, files]) => sum + files.length, 0) <= this.maxFiles) return;
     const all = [];
-    for (const kind of KINDS) for (const file of this.files(kind)) {
+    for (const [kind, files] of byKind) for (const file of files) {
+      // readFile drops corrupt and schema-mismatched files itself.
       const entry = this.readFile(file, kind);
       if (entry) all.push({ file, at: Date.parse(entry.fetchedAt) });
     }

discovery/cli.js

@@ -22,7 +22,8 @@ const greatest = loadCatalog(arg('--catalog', path.join(__dirname, '..', 'greate
 const harvestDir = path.join(data, 'discovery-cache');
 const harvestCache = fs.existsSync(harvestDir) ? new DiscoveryCache(harvestDir) : null;
 const result = generate({ db, library, greatest, cacheDir: path.join(data, 'scraper', 'metadata', 'anilist'), harvestCache });
-const summary = { examined: result.examined, candidates: Object.keys(result.candidates).length,
+// candidates: the ones some feed shows (what a run would publish); examined/clipped: the whole pool.
+const summary = { examined: result.examined, clipped: result.clipped, candidates: Object.keys(result.candidates).length,
   optedInUsers: Object.values(db.users || {}).filter(u => u.settings?.discoveryEnabled).length,
   feedSizes: Object.values(result.feeds).filter(f => !f.disabled).map(f => f.items.length) };
 if (process.argv.includes('--show')) summary.titles = Object.values(result.feeds).find(f => !f.disabled)?.items

discovery/engine.js

@@ -12,6 +12,9 @@ const MATURE_TAG_RANK = 60;
 const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']);
 const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null;
 const unique = values => [...new Set(values.filter(Boolean))];
+// manga.json and cached metadata are hand-editable files: a field that is not a list counts as empty.
+const listOf = value => Array.isArray(value) ? value : [];
+const stringsOf = value => unique(listOf(value).filter(item => typeof item === 'string'));
 
 function cachedMetadata(dir, id) {
   if (!Number.isSafeInteger(Number(id)) || Number(id) < 1) return null;
@@ -29,7 +32,7 @@ function candidateFromGreatest(entry, cacheDir) {
     type: cached?.['x-anilist']?.format || 'MANGA', status: cached?.status || entry.status || 'unknown',
     cover: safeUrl(cached?.['x-anilist']?.coverUrl) || safeUrl(entry.cover),
     metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(entry.metadata?.url) || null,
-    tags: unique(cached?.tags || []), authors: unique(cached?.authors || []),
+    tags: stringsOf(cached?.tags), authors: stringsOf(cached?.authors),
     period: entry.period, editorialRank: entry.rank, greatestId: entry.id,
     evidence: [{ id: `greatest:${entry.id}`, kind: 'curation', label: 'Greatest of All Time selection',
       note: String(entry.note || '').slice(0, 600), url: safeUrl(entry.source?.url) || safeUrl(entry.metadata?.url) }],
@@ -37,7 +40,8 @@ function candidateFromGreatest(entry, cacheDir) {
   };
 }
 function addRelations(map, anchor, cacheDir) {
-  for (const relation of anchor.relations || []) {
+  for (const relation of listOf(anchor.relations)) {
+    if (!relation || typeof relation !== 'object') continue;
     if (!VALID_FORMATS.has(relation.format) || !Number.isSafeInteger(Number(relation.id))) continue;
     const id = `anilist:${Number(relation.id)}`;
     if (map.has(id)) continue;
@@ -47,11 +51,11 @@ function addRelations(map, anchor, cacheDir) {
       year: cached?.year || null, type: relation.format, status: cached?.status || 'unknown',
       cover: safeUrl(cached?.['x-anilist']?.coverUrl),
       metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}`,
-      tags: unique(cached?.tags || []), authors: unique(cached?.authors || []), period: null,
+      tags: stringsOf(cached?.tags), authors: stringsOf(cached?.authors), period: null,
       editorialRank: null, greatestId: null,
       evidence: [{ id: `relation:${anchor.id}:${id}`, kind: 'relation', label: `${relation.type || 'RELATED'} to ${anchor.title}`,
         note: '', url: safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}` }],
-      relatedTo: { id: anchor.id, title: anchor.title, type: relation.type || 'RELATED' },
+      relatedTo: { id: anchor.id, title: anchor.title, type: String(relation.type || 'RELATED') },
       relations: cached?.['x-anilist']?.relations || [],
     });
   }
@@ -139,12 +143,12 @@ function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [
     for (const tag of candidate?.tags || []) likedTags.set(tag.toLowerCase(), anchor);
   }
   const ownedTags = new Map();
-  for (const manga of library) for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') ownedTags.set(tag.toLowerCase(), manga.title);
+  for (const manga of library) for (const tag of stringsOf(manga.meta?.tags)) ownedTags.set(tag.toLowerCase(), manga.title);
   const readTags = new Map();
   if (settings.discoveryReadingActivity) for (const manga of library) {
     const progress = db.progress?.[user.name]?.[manga.id] || {};
     if (!Object.values(progress).some(chapter => chapter?.done)) continue;
-    for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') readTags.set(tag.toLowerCase(), manga.title);
+    for (const tag of stringsOf(manga.meta?.tags)) readTags.set(tag.toLowerCase(), manga.title);
   }
   const seeds = seedIndexFor({ anchors: anchorsFor({ user, db, library, greatest }), harvestCache, mature: !!settings.discoveryMature });
   const rows = [];
@@ -217,8 +221,14 @@ function candidatePool({ greatest, library, cacheDir, harvestCache }) {
   }
   return { all, candidates };
 }
-const runResult = (all, candidates, feeds) => ({ candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds,
+// Only candidates a feed shows are published; the rest of the pool (up to 2000, with AniList
-  examined: all.length, clipped: Math.max(0, all.length - candidates.length) });
+// records) would bloat discovery.json for nothing. examined/clipped still describe the whole pool.
+function runResult(all, candidates, feeds) {
+  const shown = new Set();
+  for (const feed of Object.values(feeds)) for (const item of feed?.items || []) shown.add(item.candidateId);
+  return { candidates: Object.fromEntries(candidates.filter(c => shown.has(c.id)).map(c => [c.id, c])), feeds,
+    examined: all.length, clipped: Math.max(0, all.length - candidates.length) };
+}
 function generate({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], now }) {
   const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache });
   const feeds = {};
@@ -254,5 +264,5 @@ function mergeUserFeed(prior, result, userName) {
   return { candidates, feeds };
 }
 
-module.exports = { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, cachedMetadata,
+module.exports = { generate, generateAsync, buildCandidates, candidatePool, feedForUser, mixSlots, mergeUserFeed, cachedMetadata,
   isMature, MAX_CANDIDATES, MAX_FEED, SLOTS };

discovery/scoring.js

@@ -17,6 +17,8 @@ const PART_ORDER = ['communityRec', 'sameCreator', 'tagTop', 'fresh',
 
 const round2 = value => Math.round(value * 100) / 100;
 const listOf = value => Array.isArray(value) ? value : [];
+// Cached lists come from disk; anything that is not an object is skipped, never dereferenced.
+const objectsOf = value => listOf(value).filter(item => item && typeof item === 'object');
 const partRank = part => { const index = PART_ORDER.indexOf(part); return index < 0 ? PART_ORDER.length : index; };
 
 function communityRecValue(rating, weight) {
@@ -34,13 +36,13 @@ function seedIndexFor({ anchors, harvestCache, mature = false }) {
   for (const anchor of anchors) {
     const name = nameOf(anchor);
     const entry = harvestCache.get('recs', anchor.anilistId);
-    for (const item of listOf(entry?.data)) {
+    for (const item of objectsOf(entry?.data)) {
-      const id = `anilist:${Number(item?.anilistId)}`;
+      const id = `anilist:${Number(item.anilistId)}`;
       if (!index.recs.has(id)) index.recs.set(id, []);
       index.recs.get(id).push({ anchor, name, rating: Number(item.rating) || 0, record: item.record || null, fetchedAt: entry.fetchedAt });
     }
     if (anchor.weight < CREATOR_MIN_WEIGHT) continue;
-    for (const author of listOf(recordOf(anchor.anilistId)?.authors)) {
+    for (const author of objectsOf(recordOf(anchor.anilistId)?.authors)) {
       const prior = index.creators.get(author.staffId);
       if (prior && prior.anchor.weight >= anchor.weight) continue;
       index.creators.set(author.staffId, { anchor, name, author,
@@ -50,15 +52,15 @@ function seedIndexFor({ anchors, harvestCache, mature = false }) {
   index.topTags = topTagsFor(anchors, recordOf, { mature });
   index.topTags.forEach((tag, place) => {
     const entry = harvestCache.get('tagTop', `${tag.kind}:${tag.name}`);
-    for (const item of listOf(entry?.data)) {
+    for (const item of objectsOf(entry?.data)) {
-      const id = `anilist:${Number(item?.anilistId)}`;
+      const id = `anilist:${Number(item.anilistId)}`;
       // Tags come best-first, so the first hit is the strongest.
       if (!index.tagTop.has(id)) index.tagTop.set(id, { tag, value: 3 - place, record: item.record || null, fetchedAt: entry.fetchedAt });
     }
   });
   const fresh = harvestCache.get('fresh', 'global');
-  for (const item of listOf(fresh?.data)) {
+  for (const item of objectsOf(fresh?.data)) {
-    index.fresh.set(`anilist:${Number(item?.anilistId)}`, { record: item.record || null, fetchedAt: fresh.fetchedAt });
+    index.fresh.set(`anilist:${Number(item.anilistId)}`, { record: item.record || null, fetchedAt: fresh.fetchedAt });
   }
   return index;
 }
@@ -80,8 +82,8 @@ function seedNotes(candidate, seeds, thisYear) {
       url: urlFor(bestRec.record), fetchedAt: bestRec.fetchedAt } });
 
   let bestCreator = null;
-  for (const author of listOf(candidate.authorRecords)) {
+  for (const author of objectsOf(candidate.authorRecords)) {
-    const hit = seeds.creators.get(author?.staffId);
+    const hit = seeds.creators.get(author.staffId);
     // An anchor is never its own "also by" suggestion.
     if (!hit || `anilist:${hit.anchor.anilistId}` === candidate.id) continue;
     if (!bestCreator || hit.anchor.weight > bestCreator.anchor.weight) bestCreator = { ...hit, candidateAuthor: author };

docs/architecture.md

@@ -251,6 +251,8 @@ The reader alone talks to AniList for discovery ([discovery.md](discovery.md)).
   startedAt, finishedAt, requests, budget, stopReason }`. `stopReason` is `budget`, `rate-limited`, `forbidden`,
   `challenge`, `upstream-errors` or `error` (a local fault). A run cut off by a restart is marked `interrupted`
   at startup; queued refreshes are not persisted.
+- **Published candidates:** only those at least one feed shows. `lastRun.candidateCount` (the admin
+  candidate count) counts that set; `lastRun.examined` and `clipped` still describe the whole pool.
 - **Published fields:** candidates found through AniList carry `tagRecords`, `authorRecords`, `seed`
   (`recs|creator|tagTop|fresh`) and `fetchedAt`; feed items carry `group` (seed kind or period) and `reasons`
   (all reasons, strongest first) next to `reason`; seed evidence carries `fetchedAt`.

docs/discovery.md

@@ -45,12 +45,13 @@ count. It then asks AniList for four seed kinds:
 | Community recommendations | "readers who liked X recommend Y" for each anchor | AniList readers who liked {anchor} recommend this ({rating}). |
 | Same creator | other works by the story/art authors of strong anchors | Also by {author} — you read/liked {anchor}. |
 | Top in your tags | well-rated titles in the user's three strongest tags | Top-rated in {tag}, one of your most-liked tags. |
-| Fresh | recently started, releasing and trending titles | New: started {year}, ongoing and trending. |
+| Fresh | recently started, releasing and trending titles | New: started {year}, ongoing and trending. — or, when the cached record does not show an ongoing title started in the last two years, Trending on AniList right now. |
 
 A title found by several seeds keeps every reason and evidence item; the card
 shows the strongest, the details list all of them, each with the date it was
-fetched. A feed holds at most 20 suggestions, at most 8 from one seed kind, up to
+fetched. A feed holds at most 20 suggestions, at most 8 from one seed kind, at most
-4 fresh titles, and some from outside your strongest tags.
+4 from one Greatest period (or from related works), up to 4 fresh titles, and up to 4
+from outside your strongest tags.
 
 - **Budgets:** a nightly or admin harvest makes at most 300 requests, a
   per-user refresh 60. Requests are sequential and limited to 25 a minute. When
@@ -65,8 +66,8 @@ fetched. A feed holds at most 20 suggestions, at most 8 from one seed kind, up t
   the nightly harvest off. A night missed while the reader was down runs once
   after startup.
 - **Privacy:** for opted-in users only, the server sends AniList the IDs of
-  their anchor titles (and the names of their top tags), without login, cookies or
+  their anchor titles (liked, read or owned), the AniList staff IDs of those titles'
-  user names. Feeds and feedback stay private to this server; cache entries are
+  authors and the names of their top tags, without login, cookies or user names. Feeds and feedback stay private to this server; cache entries are
   not tied to a user.
 
 ### Stop reasons
@@ -88,8 +89,9 @@ re-scoring failure after a harvest is shown as the admin panel's error line.
 
 ## Data and API
 
-- `data/discovery.json`: atomic, reader-owned candidate and per-user feed
+- `data/discovery.json`: atomic, reader-owned per-user feed snapshots and the
-  snapshots plus the last harvest record and the admin's harvest settings. A
+  candidates those feeds show (the rest of the pool is only counted:
+  `examined`, `clipped`), plus the last harvest record and the admin's harvest settings. A
   previous valid version is retained as `discovery.json.prev`. A corrupt file
   makes reads/runs/harvests fail closed rather than overwrite it.
 - `data/discovery-cache/`: harvested AniList responses; safe to delete.
@@ -115,7 +117,9 @@ APP_IMAGE=mangacircus:discovery-test npm run test:discovery-ui
 ```
 
 The dry run reads `db.json`, a saved library index if present, Greatest, and
-cached AniList metadata. It writes nothing and does not contact providers. The
+cached AniList metadata. It does not contact AniList or any other provider and does
+not publish `discovery.json`; reading the harvest cache may still delete a corrupt
+cache file, as the reader would. The
 Compose/browser suite uses a disposable data tree and queue, not the live one,
 and points `DISCOVERY_ANILIST_URL` at a closed local port so no harvest reaches
 AniList.

docs/superpowers/plans/2026-10-25-discovery-wider-pool.md

@@ -334,4 +334,4 @@
 - [ ] **Step 3: Implement** the UI and docs.
 - [ ] **Step 4: Run** the same two commands, then `npm test`. Expected: PASS.
 - [ ] **Step 5: Commit** `Discover page: refresh, harvest admin and seed reasons; docs (BL-0051)` and push.
-- [ ] **Step 6: Update MangaOrg**: run `python D:\Projects\Manga\MangaOrg\tools\org.py` to set BL-0051 to `review`, release the `MangaCircus discovery` claim, then commit and push MangaOrg. Merging into `main` waits for the user.
+- [ ] **Step 6: Update the project's coordination backlog item and release the claim.** Merging into `main` waits for the user.

docs/superpowers/specs/2026-10-25-discovery-wider-pool-design.md

@@ -1,6 +1,6 @@
 # Discovery: wider candidate pool — design
 
-Status: approved design, not implemented. Date: 2026-10-25. Builds on the
+Status: implemented on branch feature/discovery-wider-pool. Date: 2026-10-25. Builds on the
 deterministic Discovery Scout ([`docs/discovery.md`](../../discovery.md),
 [`vision/discovery-scout.md`](../../../vision/discovery-scout.md)).
 

public/discover.js

@@ -49,7 +49,7 @@
     const refreshState = state.refresh || { state: 'idle' };
     const label = refreshState.state === 'queued' ? `Queued (position ${refreshState.position})`
       : refreshState.state === 'running' ? 'Fetching…'
-        : refreshState.blockedReason ? `Paused by AniList (${refreshState.blockedReason}) until the next nightly run`
+        : refreshState.blockedReason ? `Paused by AniList (${refreshState.blockedReason}) until the next nightly or admin harvest`
           : refreshState.nextAllowedAt ? `Next refresh possible at ${clock(refreshState.nextAllowedAt)}` : null;
     return h('button', { type: 'button', class: 'btn discovery-refresh', disabled: !!label, onclick: refreshMine }, label || 'Refresh my suggestions');
   }
@@ -60,7 +60,7 @@
   }
   function preferences() {
     return h('section', { class: 'card discovery-settings' }, h('h2', {}, 'Your discovery preferences'),
-      toggle('Personalized discovery', 'discoveryEnabled', 'Off by default. When on, the server sends AniList the IDs of titles you liked, without login, to find related work. Your feed and feedback stay private to this server.'),
+      toggle('Personalized discovery', 'discoveryEnabled', 'Off by default. When on, the server sends AniList the IDs of titles you liked, read or own, their authors\' IDs and your top tags, without login, to find related work. Your feed and feedback stay private to this server.'),
       toggle('Use reading activity', 'discoveryReadingActivity', 'Separately opt in to using completed chapters as a small taste signal.'),
       toggle('Include mature titles', 'discoveryMature', 'Only affects titles AniList marks as adult or with a known mature tag.'));
   }

server.js

@@ -317,8 +317,10 @@ try {
 const harvester = new Harvester({ cache: harvestCache, client: createAniListClient(), store: discoveryStore,
   targetsFor: harvestTargets, onDone: async job => {
     // A failed follow-up rescore must reach the admin panel (lastError), not only the log.
-    try { await rescore(job.kind === 'user' ? { onlyUser: job.user } : {}); }
+    try {
-    catch (error) { discoveryError = `Re-scoring after harvest ${job.id} failed: ${String(error.message || error)}`.slice(0, 300); throw error; }
+      await rescore(job.kind === 'user' ? { onlyUser: job.user } : {});
+      discoveryError = null; // a later success must not leave an old failure on the admin panel
+    } catch (error) { discoveryError = `Re-scoring after harvest ${job.id} failed: ${String(error.message || error)}`.slice(0, 300); throw error; }
   } });
 function requestHarvest(args) {
   readDiscovery(); // A corrupt store fails closed before any AniList call.

test/discovery-anchors.test.js

@@ -60,3 +60,14 @@ test('top tags weigh anchors, skip low rank and adult tags', () => {
   assert.deepEqual(topTagsFor(anchors, recordOf, { mature: true }), [
     { name: 'Action', kind: 'genre' }, { name: 'Ecchi', kind: 'tag' }, { name: 'Alpha', kind: 'tag' }]);
 });
+
+test('top tags drop tags without a rank and survive malformed records', () => {
+  const records = {
+    1: { tags: [{ name: 'NoRank', kind: 'tag', isAdult: false }, { name: 'NullRank', rank: null, kind: 'tag', isAdult: false },
+      null, 7, { name: 'Kept', rank: 60, kind: 'tag', isAdult: false }] },
+    2: { tags: 5 },
+    3: { tags: { name: 'Object', rank: 90 } },
+  };
+  const anchors = [{ anilistId: 1, weight: 5 }, { anilistId: 2, weight: 3 }, { anilistId: 3, weight: 3 }];
+  assert.deepEqual(topTagsFor(anchors, id => records[id]), [{ name: 'Kept', kind: 'tag' }]);
+});

test/discovery-anilist.test.js

@@ -34,6 +34,14 @@ test('normalizeMedia maps fields and drops novels', () => {
   assert.equal(normalizeMedia({ ...media, status: 'HIATUS' }).status, 'unknown');
 });
 
+test('normalizeMedia drops authors without an integer staff id', () => {
+  const media = fixture('title').data.Media;
+  const edge = (id, full) => ({ role: 'Story', node: { id, name: { full } } });
+  const record = normalizeMedia({ ...media, staff: { edges: [edge(5, 'Has Id'), edge(undefined, 'No Id'),
+    edge('6', 'String Id'), edge(1.5, 'Float Id'), edge(null, 'Null Id')] } });
+  assert.deepEqual(record.authors, [{ staffId: 5, name: 'Has Id', role: 'Story' }]);
+});
+
 test('title returns a record and sends the fixed headers', async () => {
   const { client, calls } = stub(() => json(fixture('title')));
   const record = await client.title(101);

test/discovery-cache.test.js

@@ -49,6 +49,48 @@ test('prune keeps the newest maxFiles', (t) => {
   assert.notEqual(cache.get('title', 5), null);
 });
 
+test('prune removes the oldest entries across kinds', (t) => {
+  const { clock, cache } = setup(t, { maxFiles: 3 });
+  const kinds = ['title', 'recs', 'creator', 'fresh', 'tagTop'];
+  kinds.forEach((kind, i) => { clock.t += 1000; cache.put(kind, i, { i }); });
+  assert.equal(cache.stats().files, 3);
+  assert.equal(cache.get('title', 0), null);
+  assert.equal(cache.get('recs', 1), null);
+  for (const [i, kind] of kinds.entries()) if (i >= 2) assert.notEqual(cache.get(kind, i), null, kind);
+});
+
+test('a write under the cap does not read other entries', (t) => {
+  const { dir, cache } = setup(t, { maxFiles: 10 });
+  cache.put('recs', 7, {});
+  const file = path.join(dir, 'recs', fs.readdirSync(path.join(dir, 'recs'))[0]);
+  fs.writeFileSync(file, '{broken');
+  cache.put('title', 1, {});
+  assert.equal(fs.readFileSync(file, 'utf8'), '{broken');
+});
+
+test('over the cap, corrupt and schema-mismatch entries go before valid ones', (t) => {
+  const { dir, clock, cache } = setup(t, { maxFiles: 2 });
+  cache.put('recs', 1, {});
+  cache.put('recs', 2, {});
+  const [first, second] = fs.readdirSync(path.join(dir, 'recs')).map(name => path.join(dir, 'recs', name));
+  fs.writeFileSync(first, '{broken');
+  fs.writeFileSync(second, JSON.stringify({ schema: 2, kind: 'recs', key: 2, fetchedAt: new Date(clock.t).toISOString(), data: {} }));
+  clock.t += 1000;
+  cache.put('title', 3, {});
+  assert.equal(fs.existsSync(first), false);
+  assert.equal(fs.existsSync(second), false);
+  assert.notEqual(cache.get('title', 3), null);
+});
+
+test('schema-mismatch entry is deleted like a corrupt one', (t) => {
+  const { dir, clock, cache } = setup(t);
+  cache.put('recs', 7, {});
+  const file = path.join(dir, 'recs', fs.readdirSync(path.join(dir, 'recs'))[0]);
+  fs.writeFileSync(file, JSON.stringify({ schema: 2, kind: 'recs', key: 7, fetchedAt: new Date(clock.t).toISOString(), data: {} }));
+  assert.equal(cache.get('recs', 7), null);
+  assert.equal(fs.existsSync(file), false);
+});
+
 test('corrupt entry is deleted', (t) => {
   const { dir, cache } = setup(t);
   cache.put('recs', 7, {});

test/discovery-server.test.js

@@ -50,7 +50,7 @@ const sha = s => crypto.createHash('sha256').update(s).digest('hex');
 const USERS = { boss: { admin: true, discovery: false }, alice: { discovery: true }, bob: { discovery: true }, carol: { discovery: false } };
 
 // A disposable reader with ready-made accounts and sessions; nightly harvests are off unless the store says otherwise.
-async function startReader({ anilist, discovery = {}, mangaMeta = null } = {}) {
+async function startReader({ anilist, discovery = {}, mangaMeta = null, scraperUrl = 'http://127.0.0.1:9' } = {}) {
   const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-server-'));
   const library = path.join(root, 'library'), data = path.join(root, 'data');
   fs.mkdirSync(path.join(library, 'Example Manga'), { recursive: true });
@@ -80,7 +80,7 @@ async function startReader({ anilist, discovery = {}, mangaMeta = null } = {}) {
   const port = await freePort(), output = [];
   const child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true,
     env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0',
-      SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '',
+      SCRAPER_URL: scraperUrl, SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '', BACKUP: '',
       DISCOVERY_ANILIST_URL: anilist ? anilist.url : 'http://127.0.0.1:9/', DISCOVERY_HARVEST_AT: '' } });
   child.stdout.on('data', d => output.push(String(d)));
   child.stderr.on('data', d => output.push(String(d)));
@@ -216,22 +216,55 @@ test('admin harvest is admin-only and single-flight', { timeout: 60000 }, async
   } finally { await reader.stop(); await anilist.close(); }
 });
 
+// A scraper stand-in. The rescore asks it for the queue after the harvest's final store patch and
+// right before publishing, so onState can break discovery.json at exactly that point.
+async function startScraper(onState) {
+  const server = http.createServer((req, res) => {
+    if (req.url.startsWith('/api/state')) onState();
+    res.writeHead(200, { 'content-type': 'application/json' });
+    res.end(JSON.stringify({ queue: [] }));
+  });
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  return { url: `http://127.0.0.1:${server.address().port}`, close: () => new Promise(resolve => server.close(resolve)) };
+}
+
 test('a failed rescore after a harvest reaches the admin as lastError', { timeout: 60000 }, async () => {
   const anilist = await startAniList();
-  // A manga.json whose tags is not a list makes scoring throw; it stands in for any scoring fault.
+  let reader, armed = false, saved = null;
-  const reader = await startReader({ anilist, mangaMeta: { title: 'Broken Tags', tags: 5 } });
+  const scraper = await startScraper(() => {
+    if (!armed) return;
+    armed = false;
+    const file = path.join(reader.data, 'discovery.json');
+    saved = fs.readFileSync(file, 'utf8');
+    fs.writeFileSync(file, '{broken'); // the publish now refuses to overwrite it
+  });
+  // A malformed manga.json (tags not a list, relations an object) must not be what breaks scoring.
+  reader = await startReader({ anilist, scraperUrl: scraper.url,
+    mangaMeta: { title: 'Broken Tags', tags: 5, 'x-anilist': { id: 101, relations: {} } } });
   try {
     await until(async () => (await reader.get('boss', '/api/library')).body?.mangas?.length === 1, 'the library scan');
-    assert.equal((await reader.post('boss', '/api/admin/discovery/harvest')).status, 200);
+    armed = true;
-    const done = await harvestDone(reader);
+    const first = await reader.post('boss', '/api/admin/discovery/harvest');
-    assert.equal(done.harvest.kind, 'admin');
+    assert.equal(first.status, 200);
+    await until(async () => saved && reader.output.join('').includes(`${first.body.harvestId} follow-up failed`), 'the failed rescore');
+    fs.writeFileSync(path.join(reader.data, 'discovery.json'), saved);
     const admin = await until(async () => {
       const { body } = await reader.get('boss', '/api/admin/discovery');
       return body.lastError ? body : null;
     }, 'the rescore error');
-    assert.match(admin.lastError, /^Re-scoring after harvest [0-9a-f]{16} failed: /);
+    assert.match(admin.lastError, /^Re-scoring after harvest [0-9a-f]{16} failed: Discovery store unavailable/);
     assert.equal(admin.currentRun, null);
-  } finally { await reader.stop(); await anilist.close(); }
+    assert.equal(admin.harvest.kind, 'admin');
+
+    // The next harvest's successful rescore clears the stale error.
+    const second = await reader.post('boss', '/api/admin/discovery/harvest');
+    assert.equal(second.status, 200, JSON.stringify(second.body));
+    const cleared = await until(async () => {
+      const { body } = await reader.get('boss', '/api/admin/discovery');
+      return body.harvest?.id === second.body.harvestId && !body.currentHarvest && body.lastError === null ? body : null;
+    }, 'the error to clear');
+    assert.ok(cleared.candidateCount > 0);
+  } finally { await reader.stop(); await anilist.close(); await scraper.close(); }
 });
 
 test('settings validation', { timeout: 30000 }, async () => {

test/discovery.test.js

@@ -4,7 +4,7 @@ const assert = require('node:assert/strict');
 const fs = require('node:fs');
 const os = require('node:os');
 const path = require('node:path');
-const { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, MAX_CANDIDATES, SLOTS } = require('../discovery/engine');
+const { generate, generateAsync, buildCandidates, candidatePool, feedForUser, mixSlots, mergeUserFeed, MAX_CANDIDATES, SLOTS } = require('../discovery/engine');
 const { DiscoveryStore } = require('../discovery/store');
 const { DiscoveryCache } = require('../discovery/cache');
 
@@ -16,12 +16,17 @@ test('discovery is bounded, diversified, opt-in, evidence-linked and user-specif
     off: { name: 'off', settings: {} } };
   const db = { users, greatestRatings: { alice: { [greatest.entries[0].id]: 'flame' },
     bob: { [greatest.entries[20].id]: 'flame' } }, discoveryFeedback: { alice: {} }, progress: {} };
-  const result = generate({ db, library: [], greatest, cacheDir: path.join(os.tmpdir(), 'nonexistent-discovery-cache') });
+  const cacheDir = path.join(os.tmpdir(), 'nonexistent-discovery-cache');
-  assert.equal(Object.keys(result.candidates).length, Math.min(2000, result.examined));
+  const result = generate({ db, library: [], greatest, cacheDir });
+  // Only candidates a feed shows are published; the pool itself stays bounded at 2000.
+  const shown = new Set(Object.values(result.feeds).flatMap(feed => (feed.items || []).map(item => item.candidateId)));
+  assert.deepEqual(Object.keys(result.candidates).sort(), [...shown].sort());
+  const pool = candidatePool({ greatest, library: [], cacheDir });
+  assert.equal(pool.candidates.length, Math.min(2000, result.examined));
   assert.ok(result.feeds.alice.items.length > 0 && result.feeds.alice.items.length <= 20);
   assert.ok(result.feeds.bob.items.length > 0);
   assert.equal(result.feeds.off.disabled, true);
-  assert.ok(new Set(Object.values(result.candidates).map(c => c.period)).size >= 10, 'An early catalog page must not crowd out later eras');
+  assert.ok(new Set(pool.candidates.map(c => c.period)).size >= 10, 'An early catalog page must not crowd out later eras');
   assert.ok(result.feeds.alice.items.every(item => item.evidenceIds.every(id => item.evidence.some(e => e.id === id))));
   assert.ok(result.feeds.alice.items.some(item => item.evidence.some(e => e.kind === 'personal')));
   assert.ok(!result.feeds.bob.items.some(item => item.evidence.some(e => e.label?.includes(greatest.entries[0].title) && e.kind === 'personal')));
@@ -144,10 +149,48 @@ test('engine: empty harvest cache equals Phase A pool', t => {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-empty-'));
   t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
   const db = { users: {}, greatestRatings: {}, discoveryFeedback: {}, progress: {} };
-  const without = generate({ db, library: [], greatest, cacheDir: NO_SCRAPER_CACHE });
+  const without = candidatePool({ greatest, library: [], cacheDir: NO_SCRAPER_CACHE });
-  const withEmpty = generate({ db, library: [], greatest, cacheDir: NO_SCRAPER_CACHE, harvestCache: new DiscoveryCache(dir) });
+  const withEmpty = candidatePool({ greatest, library: [], cacheDir: NO_SCRAPER_CACHE, harvestCache: new DiscoveryCache(dir) });
   assert.equal(MAX_CANDIDATES, 2000);
-  assert.deepEqual(Object.keys(withEmpty.candidates).sort(), Object.keys(without.candidates).sort());
+  assert.deepEqual(withEmpty.candidates.map(c => c.id).sort(), without.candidates.map(c => c.id).sort());
+  const users = { alice: { name: 'alice', settings: { discoveryEnabled: true } } };
+  const run = harvestCache => generate({ db: { ...db, users }, library: [], greatest, cacheDir: NO_SCRAPER_CACHE, harvestCache });
+  assert.deepEqual(Object.keys(run(new DiscoveryCache(dir)).candidates).sort(), Object.keys(run(null).candidates).sort());
+});
+
+test('published candidates are only those referenced by a feed', async t => {
+  const world = fixtureWorld(t);
+  world.db.users.bob = { name: 'bob', settings: { discoveryEnabled: true } };
+  world.db.users.off = { name: 'off', settings: {} };
+  const args = { db: world.db, library: world.library, greatest: world.greatest, cacheDir: NO_SCRAPER_CACHE,
+    harvestCache: world.harvestCache, queuedTitles: world.queuedTitles, now: NOW };
+  const referenced = result => new Set(Object.values(result.feeds).flatMap(feed => (feed.items || []).map(item => item.candidateId)));
+  for (const result of [generate(args), await generateAsync(args), await generateAsync({ ...args, onlyUser: 'alice' })]) {
+    const shown = referenced(result);
+    assert.ok(shown.size > 0);
+    for (const id of Object.keys(result.candidates)) assert.ok(shown.has(id), `${id} is published but not shown`);
+    for (const id of shown) assert.ok(result.candidates[id], `${id} is shown but not published`);
+    assert.ok(result.examined > Object.keys(result.candidates).length, 'examined still counts the whole pool');
+    assert.equal(result.clipped, 0);
+  }
+});
+
+test('malformed library metadata and cache records score without throwing', t => {
+  const world = fixtureWorld(t, { discoveryReadingActivity: true });
+  world.library.push(
+    { id: 'm2', title: 'Bad Tags', meta: { tags: 5, 'x-anilist': { id: 220, relations: {} } } },
+    { id: 'm3', title: 'Bad Relations', meta: { tags: [7, null, { name: 'Drama' }, 'Drama'],
+      'x-anilist': { id: 221, relations: [null, 3, 'x', { id: 222, format: 'MANGA', type: 'SEQUEL', title: 'Sequel Title' }] } } },
+    { id: 'm4', title: 'Bad Identity', meta: { tags: { a: 1 }, 'x-anilist': 5 } });
+  world.db.progress = { alice: { m2: { c1: { done: true } }, m3: { c1: { done: true } } } };
+  world.harvestCache.put('title', 220, seedRecord(220, { authors: [null, 5, AUTHOR], tags: [null, 'Drama', { name: 'Drama', kind: 'genre' }] }));
+  world.harvestCache.put('recs', 220, [null, 5, { ...seed(230), rating: 7 }]);
+  const args = { db: world.db, library: world.library, greatest: world.greatest, cacheDir: NO_SCRAPER_CACHE,
+    harvestCache: world.harvestCache, queuedTitles: world.queuedTitles, now: NOW };
+  const result = generate(args);
+  const ids = result.feeds.alice.items.map(item => item.candidateId);
+  assert.ok(ids.includes('anilist:222'), JSON.stringify(ids));
+  assert.ok(ids.includes('anilist:230'), JSON.stringify(ids));
 });
 
 test('harvested records join the pool and fill Greatest candidates', t => {
@@ -262,6 +305,25 @@ test('slot mix', () => {
   assert.ok(full.filter(r => r.group === '1980s').length <= SLOTS.perPeriod);
 });
 
+test('slot mix reservations let low-scored fresh and outside rows past a full feed', () => {
+  const row = (id, score, group, extra = {}) => ({ candidateId: `anilist:${id}`, score, group, tags: ['Drama'],
+    scoreParts: { fresh: 0 }, ...extra });
+  // 32 higher-scored rows that the caps alone would allow: 8 per seed kind, 4 per period.
+  const rows = [];
+  const groups = ['recs', 'creator', 'tagTop', '1980s', '1990s'];
+  const caps = { recs: 8, creator: 8, tagTop: 8, '1980s': 4, '1990s': 4 };
+  let id = 5000;
+  for (const group of groups) for (let i = 0; i < caps[group]; i++) rows.push(row(id++, 100 - rows.length, group));
+  assert.ok(rows.length > 20);
+  rows.push(row(2001, 1, 'fresh', { scoreParts: { fresh: 1 } }), row(2002, 0.9, 'fresh', { scoreParts: { fresh: 1 } }));
+  rows.push(row(3001, 0.2, '2000s', { tags: ['Sports'] }), row(3002, 0.1, '2000s', { tags: [] }));
+  const picked = mixSlots(rows, { topTags: [{ name: 'Drama', kind: 'genre' }] });
+  assert.equal(picked.length, 20);
+  for (const low of ['anilist:2001', 'anilist:2002', 'anilist:3001', 'anilist:3002'])
+    assert.ok(picked.some(r => r.candidateId === low), `${low} needs its reserved slot`);
+  for (const group of groups) assert.ok(picked.filter(r => r.group === group).length <= caps[group], group);
+});
+
 test('mergeUserFeed keeps other users\' referenced candidates', () => {
   const prior = { candidates: { a: { id: 'a' }, b: { id: 'b' }, c: { id: 'c' } },
     feeds: { alice: { items: [{ candidateId: 'a' }] }, bob: { items: [{ candidateId: 'b' }] }, off: { items: [], disabled: true } } };

tools/test-discovery-ui.js

@@ -15,19 +15,20 @@ module.exports = async function testDiscoveryUI(url, cookie, { xssTitle } = {})
     await page.goto(url + '/discover.html');
     await page.getByRole('heading', { name: 'Discover', exact: true }).waitFor();
     await page.getByText('Turn on personalized discovery').waitFor();
-    await page.getByText('the server sends AniList the IDs of titles you liked, without login').waitFor();
+    await page.getByText('Off by default. When on, the server sends AniList the IDs of titles you liked, read or own, '
+      + 'their authors\' IDs and your top tags, without login, to find related work. Your feed and feedback stay private to this server.').waitFor();
     await page.getByLabel('Personalized discovery').check();
     // Review focus 5: the seeded feed is the first one shown, before any run replaces it.
-    if (xssTitle) {
+    // The XSS check must never be skipped silently: the Compose driver always seeds the title.
-      const seeded = page.locator('.discovery-card').filter({ hasText: 'AniList readers who liked Moon Garden' });
+    assert.ok(xssTitle, 'The discovery UI test needs the seeded XSS title (run it through tools/test-compose.js --discovery-ui)');
-      await seeded.waitFor();
+    const seeded = page.locator('.discovery-card').filter({ hasText: 'AniList readers who liked Moon Garden' });
-      assert.equal(await seeded.locator('h2').textContent(), xssTitle);
+    await seeded.waitFor();
-      assert.equal(await page.locator('.discovery-card h2 img').count(), 0);
+    assert.equal(await seeded.locator('h2').textContent(), xssTitle);
-      await seeded.getByText('Why this is here · evidence').click();
+    assert.equal(await page.locator('.discovery-card h2 img').count(), 0);
-      const details = await seeded.locator('details li').allTextContents();
+    await seeded.getByText('Why this is here · evidence').click();
-      assert.ok(details.some(text => text.includes('New: started 2025')), `All reasons listed: ${details}`);
+    const details = await seeded.locator('details li').allTextContents();
-      assert.ok(details.some(text => /AniList recommendation.* · fetched \S+/.test(text)), `Evidence fetch date: ${details}`);
+    assert.ok(details.some(text => text.includes('New: started 2025')), `All reasons listed: ${details}`);
-    }
+    assert.ok(details.some(text => /AniList recommendation.* · fetched \S+/.test(text)), `Evidence fetch date: ${details}`);
     await page.getByRole('button', { name: 'Run discovery now' }).click();
     // The seeded store already says "success"; wait until the run's own feed replaced the seeded card.
     await page.waitForFunction(title => /Status: success/.test(document.querySelector('.discovery-admin')?.textContent || '')