Discovery final review fixes: lean publish, robust scoring, cheap prune (BL-0051)

- Cache prune counts directory entries first and only reads files when over
  the cap; tests for cross-kind prune, schema-mismatch and corrupt entries.
- Scoring tolerates malformed manga.json and cached records (non-list tags,
  object or null relations, non-object items); a tag without a rank no longer
  passes the top-tag filter; authors without an integer AniList id are dropped.
- discovery.json publishes only candidates some feed shows; examined/clipped
  still describe the pool. Tests that asserted the published count equals the
  pool size now check the pool via candidatePool() and the published set
  against the feeds.
- The post-harvest rescore failure test now breaks discovery.json right before
  the publish (via a scraper stand-in) instead of relying on tags: 5, and a
  following successful harvest clears the stale lastError.
- Consent text names everything sent to AniList; paused label mentions admin
  harvests; the UI test asserts the XSS seed instead of skipping it.
- Slot-mix test proves the fresh/outside reservations matter.
- Docs: privacy note, fresh fallback reason, per-period cap, dry-run wording,
  published-candidate rule; plan/spec hygiene and spec status.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

discovery/anchors.js

@@ -47,8 +47,10 @@ function topTagsFor(anchors, recordOf, { mature = false } = {}) {
   const scores = new Map();
   for (const anchor of anchors) {
     const record = recordOf(anchor.anilistId);
-    for (const tag of record?.tags || []) {
-      if (tag.rank < MIN_TAG_RANK || (tag.isAdult && !mature)) continue;
+    for (const tag of Array.isArray(record?.tags) ? record.tags : []) {
+      // A tag without a rank is no evidence of taste, so it must not pass the rank filter.
+      if (!tag || typeof tag !== 'object' || typeof tag.name !== 'string') continue;
+      if (!(tag.rank >= MIN_TAG_RANK) || (tag.isAdult && !mature)) continue;
       const entry = scores.get(tag.name) || { name: tag.name, kind: tag.kind, score: 0 };
       entry.score += anchor.weight;
       scores.set(tag.name, entry);

discovery/anilist.js

@@ -64,7 +64,8 @@ function normalizeMedia(media) {
       ...(media.genres || []).map(name => ({ name, rank: 100, kind: 'genre', isAdult: false })),
       ...(media.tags || []).map(tag => ({ name: tag.name, rank: tag.rank, kind: 'tag', isAdult: !!tag.isAdult }))],
     authors: (media.staff?.edges || [])
-      .filter(edge => STAFF_ROLE.test(edge.role || '') && edge.node?.name?.full)
+      // An author without an integer id could never match a creator seed.
+      .filter(edge => STAFF_ROLE.test(edge?.role || '') && edge.node?.name?.full && Number.isInteger(edge.node.id))
       .map(edge => ({ staffId: edge.node.id, name: edge.node.name.full, role: edge.role })),
     coverUrl: media.coverImage?.extraLarge || null,
     siteUrl: media.siteUrl || null,

discovery/cache.js

@@ -71,10 +71,14 @@ class DiscoveryCache {
     }
     return { files, bytes };
   }
-  // Oldest entries go first so the cache never grows past maxFiles.
+  // Oldest entries go first so the cache never grows past maxFiles. Every put() calls this, so
+  // under the cap it only counts directory entries; files are read only when something must go.
   prune() {
+    const byKind = KINDS.map(kind => [kind, this.files(kind)]);
+    if (byKind.reduce((sum, [, files]) => sum + files.length, 0) <= this.maxFiles) return;
     const all = [];
-    for (const kind of KINDS) for (const file of this.files(kind)) {
+    for (const [kind, files] of byKind) for (const file of files) {
+      // readFile drops corrupt and schema-mismatched files itself.
       const entry = this.readFile(file, kind);
       if (entry) all.push({ file, at: Date.parse(entry.fetchedAt) });
     }

discovery/cli.js

@@ -22,7 +22,8 @@ const greatest = loadCatalog(arg('--catalog', path.join(__dirname, '..', 'greate
 const harvestDir = path.join(data, 'discovery-cache');
 const harvestCache = fs.existsSync(harvestDir) ? new DiscoveryCache(harvestDir) : null;
 const result = generate({ db, library, greatest, cacheDir: path.join(data, 'scraper', 'metadata', 'anilist'), harvestCache });
-const summary = { examined: result.examined, candidates: Object.keys(result.candidates).length,
+// candidates: the ones some feed shows (what a run would publish); examined/clipped: the whole pool.
+const summary = { examined: result.examined, clipped: result.clipped, candidates: Object.keys(result.candidates).length,
   optedInUsers: Object.values(db.users || {}).filter(u => u.settings?.discoveryEnabled).length,
   feedSizes: Object.values(result.feeds).filter(f => !f.disabled).map(f => f.items.length) };
 if (process.argv.includes('--show')) summary.titles = Object.values(result.feeds).find(f => !f.disabled)?.items

discovery/engine.js

@@ -12,6 +12,9 @@ const MATURE_TAG_RANK = 60;
 const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']);
 const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null;
 const unique = values => [...new Set(values.filter(Boolean))];
+// manga.json and cached metadata are hand-editable files: a field that is not a list counts as empty.
+const listOf = value => Array.isArray(value) ? value : [];
+const stringsOf = value => unique(listOf(value).filter(item => typeof item === 'string'));
 
 function cachedMetadata(dir, id) {
   if (!Number.isSafeInteger(Number(id)) || Number(id) < 1) return null;
@@ -29,7 +32,7 @@ function candidateFromGreatest(entry, cacheDir) {
     type: cached?.['x-anilist']?.format || 'MANGA', status: cached?.status || entry.status || 'unknown',
     cover: safeUrl(cached?.['x-anilist']?.coverUrl) || safeUrl(entry.cover),
     metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(entry.metadata?.url) || null,
-    tags: unique(cached?.tags || []), authors: unique(cached?.authors || []),
+    tags: stringsOf(cached?.tags), authors: stringsOf(cached?.authors),
     period: entry.period, editorialRank: entry.rank, greatestId: entry.id,
     evidence: [{ id: `greatest:${entry.id}`, kind: 'curation', label: 'Greatest of All Time selection',
       note: String(entry.note || '').slice(0, 600), url: safeUrl(entry.source?.url) || safeUrl(entry.metadata?.url) }],
@@ -37,7 +40,8 @@ function candidateFromGreatest(entry, cacheDir) {
   };
 }
 function addRelations(map, anchor, cacheDir) {
-  for (const relation of anchor.relations || []) {
+  for (const relation of listOf(anchor.relations)) {
+    if (!relation || typeof relation !== 'object') continue;
     if (!VALID_FORMATS.has(relation.format) || !Number.isSafeInteger(Number(relation.id))) continue;
     const id = `anilist:${Number(relation.id)}`;
     if (map.has(id)) continue;
@@ -47,11 +51,11 @@ function addRelations(map, anchor, cacheDir) {
       year: cached?.year || null, type: relation.format, status: cached?.status || 'unknown',
       cover: safeUrl(cached?.['x-anilist']?.coverUrl),
       metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}`,
-      tags: unique(cached?.tags || []), authors: unique(cached?.authors || []), period: null,
+      tags: stringsOf(cached?.tags), authors: stringsOf(cached?.authors), period: null,
       editorialRank: null, greatestId: null,
       evidence: [{ id: `relation:${anchor.id}:${id}`, kind: 'relation', label: `${relation.type || 'RELATED'} to ${anchor.title}`,
         note: '', url: safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}` }],
-      relatedTo: { id: anchor.id, title: anchor.title, type: relation.type || 'RELATED' },
+      relatedTo: { id: anchor.id, title: anchor.title, type: String(relation.type || 'RELATED') },
       relations: cached?.['x-anilist']?.relations || [],
     });
   }
@@ -139,12 +143,12 @@ function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [
     for (const tag of candidate?.tags || []) likedTags.set(tag.toLowerCase(), anchor);
   }
   const ownedTags = new Map();
-  for (const manga of library) for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') ownedTags.set(tag.toLowerCase(), manga.title);
+  for (const manga of library) for (const tag of stringsOf(manga.meta?.tags)) ownedTags.set(tag.toLowerCase(), manga.title);
   const readTags = new Map();
   if (settings.discoveryReadingActivity) for (const manga of library) {
     const progress = db.progress?.[user.name]?.[manga.id] || {};
     if (!Object.values(progress).some(chapter => chapter?.done)) continue;
-    for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') readTags.set(tag.toLowerCase(), manga.title);
+    for (const tag of stringsOf(manga.meta?.tags)) readTags.set(tag.toLowerCase(), manga.title);
   }
   const seeds = seedIndexFor({ anchors: anchorsFor({ user, db, library, greatest }), harvestCache, mature: !!settings.discoveryMature });
   const rows = [];
@@ -217,8 +221,14 @@ function candidatePool({ greatest, library, cacheDir, harvestCache }) {
   }
   return { all, candidates };
 }
-const runResult = (all, candidates, feeds) => ({ candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds,
-  examined: all.length, clipped: Math.max(0, all.length - candidates.length) });
+// Only candidates a feed shows are published; the rest of the pool (up to 2000, with AniList
+// records) would bloat discovery.json for nothing. examined/clipped still describe the whole pool.
+function runResult(all, candidates, feeds) {
+  const shown = new Set();
+  for (const feed of Object.values(feeds)) for (const item of feed?.items || []) shown.add(item.candidateId);
+  return { candidates: Object.fromEntries(candidates.filter(c => shown.has(c.id)).map(c => [c.id, c])), feeds,
+    examined: all.length, clipped: Math.max(0, all.length - candidates.length) };
+}
 function generate({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], now }) {
   const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache });
   const feeds = {};
@@ -254,5 +264,5 @@ function mergeUserFeed(prior, result, userName) {
   return { candidates, feeds };
 }
 
-module.exports = { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, cachedMetadata,
+module.exports = { generate, generateAsync, buildCandidates, candidatePool, feedForUser, mixSlots, mergeUserFeed, cachedMetadata,
   isMature, MAX_CANDIDATES, MAX_FEED, SLOTS };

discovery/scoring.js

@@ -17,6 +17,8 @@ const PART_ORDER = ['communityRec', 'sameCreator', 'tagTop', 'fresh',
 
 const round2 = value => Math.round(value * 100) / 100;
 const listOf = value => Array.isArray(value) ? value : [];
+// Cached lists come from disk; anything that is not an object is skipped, never dereferenced.
+const objectsOf = value => listOf(value).filter(item => item && typeof item === 'object');
 const partRank = part => { const index = PART_ORDER.indexOf(part); return index < 0 ? PART_ORDER.length : index; };
 
 function communityRecValue(rating, weight) {
@@ -34,13 +36,13 @@ function seedIndexFor({ anchors, harvestCache, mature = false }) {
   for (const anchor of anchors) {
     const name = nameOf(anchor);
     const entry = harvestCache.get('recs', anchor.anilistId);
-    for (const item of listOf(entry?.data)) {
-      const id = `anilist:${Number(item?.anilistId)}`;
+    for (const item of objectsOf(entry?.data)) {
+      const id = `anilist:${Number(item.anilistId)}`;
       if (!index.recs.has(id)) index.recs.set(id, []);
       index.recs.get(id).push({ anchor, name, rating: Number(item.rating) || 0, record: item.record || null, fetchedAt: entry.fetchedAt });
     }
     if (anchor.weight < CREATOR_MIN_WEIGHT) continue;
-    for (const author of listOf(recordOf(anchor.anilistId)?.authors)) {
+    for (const author of objectsOf(recordOf(anchor.anilistId)?.authors)) {
       const prior = index.creators.get(author.staffId);
       if (prior && prior.anchor.weight >= anchor.weight) continue;
       index.creators.set(author.staffId, { anchor, name, author,
@@ -50,15 +52,15 @@ function seedIndexFor({ anchors, harvestCache, mature = false }) {
   index.topTags = topTagsFor(anchors, recordOf, { mature });
   index.topTags.forEach((tag, place) => {
     const entry = harvestCache.get('tagTop', `${tag.kind}:${tag.name}`);
-    for (const item of listOf(entry?.data)) {
-      const id = `anilist:${Number(item?.anilistId)}`;
+    for (const item of objectsOf(entry?.data)) {
+      const id = `anilist:${Number(item.anilistId)}`;
       // Tags come best-first, so the first hit is the strongest.
       if (!index.tagTop.has(id)) index.tagTop.set(id, { tag, value: 3 - place, record: item.record || null, fetchedAt: entry.fetchedAt });
     }
   });
   const fresh = harvestCache.get('fresh', 'global');
-  for (const item of listOf(fresh?.data)) {
-    index.fresh.set(`anilist:${Number(item?.anilistId)}`, { record: item.record || null, fetchedAt: fresh.fetchedAt });
+  for (const item of objectsOf(fresh?.data)) {
+    index.fresh.set(`anilist:${Number(item.anilistId)}`, { record: item.record || null, fetchedAt: fresh.fetchedAt });
   }
   return index;
 }
@@ -80,8 +82,8 @@ function seedNotes(candidate, seeds, thisYear) {
       url: urlFor(bestRec.record), fetchedAt: bestRec.fetchedAt } });
 
   let bestCreator = null;
-  for (const author of listOf(candidate.authorRecords)) {
-    const hit = seeds.creators.get(author?.staffId);
+  for (const author of objectsOf(candidate.authorRecords)) {
+    const hit = seeds.creators.get(author.staffId);
     // An anchor is never its own "also by" suggestion.
     if (!hit || `anilist:${hit.anchor.anilistId}` === candidate.id) continue;
     if (!bestCreator || hit.anchor.weight > bestCreator.anchor.weight) bestCreator = { ...hit, candidateAuthor: author };