Maashkinoozhe/MangaCircus
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Discovery scores AniList seeds with slot mix and isAdult filter (BL-0051)

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
MaashkinoozheandClaude Opus 5.5 committed 2026-10-25 09:07:53 +01:00
1 parent 552a599475
commit 5de5eb6a08
4 files changed
+513 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

discovery/cli.js
+5 -1
View File
@@ -5,6 +5,7 @@ const fs = require('node:fs');
const path = require('node:path'); const path = require('node:path');
const { loadCatalog } = require('../greatest'); const { loadCatalog } = require('../greatest');
const { generate } = require('./engine'); const { generate } = require('./engine');
const { DiscoveryCache } = require('./cache');
if (!process.argv.includes('--dry-run')) { if (!process.argv.includes('--dry-run')) {
console.error('Usage: node discovery/cli.js run --dry-run [--data DIR] [--catalog FILE] [--show]'); console.error('Usage: node discovery/cli.js run --dry-run [--data DIR] [--catalog FILE] [--show]');
@@ -17,7 +18,10 @@ let library = [];
try { library = JSON.parse(fs.readFileSync(path.join(data, 'library-index.json'), 'utf8')).mangas || []; } try { library = JSON.parse(fs.readFileSync(path.join(data, 'library-index.json'), 'utf8')).mangas || []; }
catch (error) { if (error.code !== 'ENOENT') throw error; } catch (error) { if (error.code !== 'ENOENT') throw error; }
const greatest = loadCatalog(arg('--catalog', path.join(__dirname, '..', 'greatest', 'catalog.json'))); const greatest = loadCatalog(arg('--catalog', path.join(__dirname, '..', 'greatest', 'catalog.json')));
const result = generate({ db, library, greatest, cacheDir: path.join(data, 'scraper', 'metadata', 'anilist') }); // The harvest cache is only read here (get/list, never put); note get/list still drop a corrupt entry file.
const harvestDir = path.join(data, 'discovery-cache');
const harvestCache = fs.existsSync(harvestDir) ? new DiscoveryCache(harvestDir) : null;
const result = generate({ db, library, greatest, cacheDir: path.join(data, 'scraper', 'metadata', 'anilist'), harvestCache });
const summary = { examined: result.examined, candidates: Object.keys(result.candidates).length, const summary = { examined: result.examined, candidates: Object.keys(result.candidates).length,
optedInUsers: Object.values(db.users || {}).filter(u => u.settings?.discoveryEnabled).length, optedInUsers: Object.values(db.users || {}).filter(u => u.settings?.discoveryEnabled).length,
feedSizes: Object.values(result.feeds).filter(f => !f.disabled).map(f => f.items.length) }; feedSizes: Object.values(result.feeds).filter(f => !f.disabled).map(f => f.items.length) };
discovery/engine.js
+127 -43
View File
@@ -3,9 +3,12 @@
const fs = require('node:fs'); const fs = require('node:fs');
const path = require('node:path'); const path = require('node:path');
const { normalize } = require('../greatest'); const { normalize } = require('../greatest');
const { anchorsFor } = require('./anchors');
const { MAX_FEED, SLOTS, round2, seedIndexFor, seedNotes, groupFor, partRank, mixSlots } = require('./scoring');
const MAX_CANDIDATES = 50; const MAX_CANDIDATES = 2000;
const MAX_FEED = 20; const SEED_KINDS = ['recs', 'creator', 'tagTop', 'fresh'];
const MATURE_TAG_RANK = 60;
const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']); const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']);
const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null; const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null;
const unique = values => [...new Set(values.filter(Boolean))]; const unique = values => [...new Set(values.filter(Boolean))];
@@ -53,7 +56,42 @@ function addRelations(map, anchor, cacheDir) {
}); });
} }
} }
function buildCandidates({ greatest, library, cacheDir }) { const textList = values => unique(Array.isArray(values) ? values.map(value => value?.name).filter(name => typeof name === 'string') : []);
const recordList = values => Array.isArray(values) ? values.filter(value => value && typeof value === 'object') : [];
function candidateFromRecord(record, id, seed, fetchedAt) {
return {
id: `anilist:${id}`, title: String(record.title || `AniList ${id}`), year: Number(record.year) || null,
type: record.format || 'MANGA', status: record.status || 'unknown', cover: safeUrl(record.coverUrl),
metadataUrl: safeUrl(record.siteUrl) || `https://anilist.co/manga/${id}`,
tags: textList(record.tags), authors: textList(record.authors), period: null, editorialRank: null, greatestId: null,
evidence: [], relations: [], isAdult: record.isAdult === true,
averageScore: Number.isFinite(record.averageScore) ? record.averageScore : null,
tagRecords: recordList(record.tags), authorRecords: recordList(record.authors), fetchedAt, seed,
};
}
// Greatest and relation candidates keep their own fields; AniList only fills what they lack.
function fillFromRecord(candidate, record, fetchedAt) {
if (candidate.isAdult === undefined) candidate.isAdult = record.isAdult === true;
if (!candidate.tagRecords) candidate.tagRecords = recordList(record.tags);
if (!candidate.authorRecords) candidate.authorRecords = recordList(record.authors);
if (candidate.averageScore == null && Number.isFinite(record.averageScore)) candidate.averageScore = record.averageScore;
if (!candidate.fetchedAt) candidate.fetchedAt = fetchedAt;
}
function addHarvested(map, harvestCache) {
for (const kind of SEED_KINDS) {
const entries = harvestCache.list(kind).sort((a, b) => String(a.key).localeCompare(String(b.key)));
for (const entry of entries) for (const item of Array.isArray(entry.data) ? entry.data : []) {
const record = item?.record;
const id = Number(item?.anilistId);
if (!record || typeof record !== 'object' || !Number.isSafeInteger(id) || id < 1) continue;
if (record.format && !VALID_FORMATS.has(record.format)) continue;
const existing = map.get(`anilist:${id}`);
if (existing) fillFromRecord(existing, record, entry.fetchedAt);
else map.set(`anilist:${id}`, candidateFromRecord(record, id, kind, entry.fetchedAt));
}
}
}
function buildCandidates({ greatest, library, cacheDir, harvestCache = null }) {
const map = new Map(); const map = new Map();
for (const entry of greatest.entries) { for (const entry of greatest.entries) {
const candidate = candidateFromGreatest(entry, cacheDir); const candidate = candidateFromGreatest(entry, cacheDir);
@@ -65,8 +103,15 @@ function buildCandidates({ greatest, library, cacheDir }) {
addRelations(map, { id: `anilist:${identity.id}`, title: manga.title, relations: identity.relations }, cacheDir); addRelations(map, { id: `anilist:${identity.id}`, title: manga.title, relations: identity.relations }, cacheDir);
} }
for (const candidate of [...map.values()]) if (candidate.greatestId) addRelations(map, candidate, cacheDir); for (const candidate of [...map.values()]) if (candidate.greatestId) addRelations(map, candidate, cacheDir);
if (harvestCache) addHarvested(map, harvestCache);
return [...map.values()]; return [...map.values()];
} }
// Title-level isAdult or a strong adult tag; the Phase A regex only where AniList gave no isAdult.
function isMature(candidate) {
if (candidate.isAdult === undefined) return candidate.tags.some(tag => /adult|hentai|erotica/i.test(tag));
return candidate.isAdult === true
|| (candidate.tagRecords || []).some(tag => tag?.isAdult && Number(tag.rank) >= MATURE_TAG_RANK);
}
function ownedKeySet(library) { function ownedKeySet(library) {
const ids = new Set(), names = new Set(); const ids = new Set(), names = new Set();
for (const manga of library) { for (const manga of library) {
@@ -77,9 +122,10 @@ function ownedKeySet(library) {
return { ids, names }; return { ids, names };
} }
function labelForVote(greatest, id) { return greatest.entries.find(entry => entry.id === id)?.title || null; } function labelForVote(greatest, id) { return greatest.entries.find(entry => entry.id === id)?.title || null; }
function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [] }) { function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [], harvestCache = null, now = () => new Date() }) {
const settings = user.settings || {}; const settings = user.settings || {};
if (!settings.discoveryEnabled) return { generatedAt: new Date().toISOString(), items: [], disabled: true }; const today = now();
if (!settings.discoveryEnabled) return { generatedAt: today.toISOString(), items: [], disabled: true };
const votes = db.greatestRatings?.[user.name] || {}; const votes = db.greatestRatings?.[user.name] || {};
const feedback = db.discoveryFeedback?.[user.name] || {}; const feedback = db.discoveryFeedback?.[user.name] || {};
const owned = ownedKeySet(library); const owned = ownedKeySet(library);
@@ -100,13 +146,14 @@ function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [
if (!Object.values(progress).some(chapter => chapter?.done)) continue; if (!Object.values(progress).some(chapter => chapter?.done)) continue;
for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') readTags.set(tag.toLowerCase(), manga.title); for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') readTags.set(tag.toLowerCase(), manga.title);
} }
const seeds = seedIndexFor({ anchors: anchorsFor({ user, db, library, greatest }), harvestCache, mature: !!settings.discoveryMature });
const rows = []; const rows = [];
for (const candidate of candidates) { for (const candidate of candidates) {
if (candidate.greatestId && votes[candidate.greatestId]) continue; if (candidate.greatestId && votes[candidate.greatestId]) continue;
if (['dismissed', 'hidden'].includes(feedback[candidate.id])) continue; if (['dismissed', 'hidden'].includes(feedback[candidate.id])) continue;
if (owned.ids.has(candidate.id) || owned.names.has(`${normalize(candidate.title)}:${candidate.year || '?'}`)) continue; if (owned.ids.has(candidate.id) || owned.names.has(`${normalize(candidate.title)}:${candidate.year || '?'}`)) continue;
if (queued.has(normalize(candidate.title))) continue; if (queued.has(normalize(candidate.title))) continue;
if (!settings.discoveryMature && candidate.tags.some(tag => /adult|hentai|erotica/i.test(tag))) continue; if (!settings.discoveryMature && isMature(candidate)) continue;
const matchingLikedTag = candidate.tags.find(tag => likedTags.has(tag.toLowerCase())); const matchingLikedTag = candidate.tags.find(tag => likedTags.has(tag.toLowerCase()));
const matchingOwnedTag = candidate.tags.find(tag => ownedTags.has(tag.toLowerCase())); const matchingOwnedTag = candidate.tags.find(tag => ownedTags.has(tag.toLowerCase()));
const matchingReadTag = candidate.tags.find(tag => readTags.has(tag.toLowerCase())); const matchingReadTag = candidate.tags.find(tag => readTags.has(tag.toLowerCase()));
@@ -115,47 +162,50 @@ function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [
const scoreParts = { editorial: candidate.greatestId ? Math.max(0, 6 - Math.min(6, Number(candidate.editorialRank) || 6)) : 0, const scoreParts = { editorial: candidate.greatestId ? Math.max(0, 6 - Math.min(6, Number(candidate.editorialRank) || 6)) : 0,
relation: relation ? 3 : 0, sharedTaste: matchingLikedTag ? 4 : 0, sharedLibrary: matchingOwnedTag ? 1 : 0, relation: relation ? 3 : 0, sharedTaste: matchingLikedTag ? 4 : 0, sharedLibrary: matchingOwnedTag ? 1 : 0,
era: era ? 2 : 0, reading: matchingReadTag ? 2 : 0, era: era ? 2 : 0, reading: matchingReadTag ? 2 : 0,
interest: feedback[candidate.id] === 'interested' ? 5 : feedback[candidate.id] === 'maybe' ? 2 : 0 }; interest: feedback[candidate.id] === 'interested' ? 5 : feedback[candidate.id] === 'maybe' ? 2 : 0,
let reason, personalEvidence = null; communityRec: 0, sameCreator: 0, tagTop: 0, fresh: 0 };
if (relation) reason = `${candidate.title} is listed as ${relation.type.toLowerCase().replaceAll('_', ' ')} to ${relation.title}. Check the edition and reading order first.`; // Every applicable part brings its reason and, for taste parts, its personal evidence.
else if (matchingLikedTag) { const notes = [];
if (scoreParts.editorial && candidate.evidence[0]?.note) notes.push({ part: 'editorial', text: candidate.evidence[0].note });
if (relation) notes.push({ part: 'relation', text: `${candidate.title} is listed as ${relation.type.toLowerCase().replaceAll('_', ' ')} to ${relation.title}. Check the edition and reading order first.` });
if (matchingLikedTag) {
const anchor = likedTags.get(matchingLikedTag.toLowerCase()); const anchor = likedTags.get(matchingLikedTag.toLowerCase());
reason = `You liked ${anchor.title}; this also carries the ${matchingLikedTag} tag.`; notes.push({ part: 'sharedTaste', text: `You liked ${anchor.title}; this also carries the ${matchingLikedTag} tag.`,
personalEvidence = { id: `taste:greatest:${anchor.id}`, kind: 'personal', label: `Your Greatest vote for ${anchor.title}`, note: `Shared tag: ${matchingLikedTag}`, url: null }; evidence: { id: `taste:greatest:${anchor.id}`, kind: 'personal', label: `Your Greatest vote for ${anchor.title}`, note: `Shared tag: ${matchingLikedTag}`, url: null } });
} else if (era) { }
reason = `You liked ${era.title}; this is another curated work from ${candidate.period}.`; if (era) notes.push({ part: 'era', text: `You liked ${era.title}; this is another curated work from ${candidate.period}.`,
personalEvidence = { id: `taste:greatest:${era.id}`, kind: 'personal', label: `Your Greatest vote for ${era.title}`, note: `Shared period: ${candidate.period}`, url: null }; evidence: { id: `taste:greatest:${era.id}`, kind: 'personal', label: `Your Greatest vote for ${era.title}`, note: `Shared period: ${candidate.period}`, url: null } });
} else if (matchingReadTag) { if (matchingReadTag) {
const title = readTags.get(matchingReadTag.toLowerCase()); const title = readTags.get(matchingReadTag.toLowerCase());
reason = `You have read ${title}; this also carries the ${matchingReadTag} tag.`; notes.push({ part: 'reading', text: `You have read ${title}; this also carries the ${matchingReadTag} tag.`,
personalEvidence = { id: `taste:read:${normalize(title)}`, kind: 'personal', label: `Your completed reading in ${title}`, note: `Shared tag: ${matchingReadTag}`, url: null }; evidence: { id: `taste:read:${normalize(title)}`, kind: 'personal', label: `Your completed reading in ${title}`, note: `Shared tag: ${matchingReadTag}`, url: null } });
} else if (matchingOwnedTag) { }
if (matchingOwnedTag) {
const title = ownedTags.get(matchingOwnedTag.toLowerCase()); const title = ownedTags.get(matchingOwnedTag.toLowerCase());
reason = `This shares the ${matchingOwnedTag} tag with ${title} in your library.`; notes.push({ part: 'sharedLibrary', text: `This shares the ${matchingOwnedTag} tag with ${title} in your library.`,
personalEvidence = { id: `taste:library:${normalize(title)}`, kind: 'personal', label: `${title} in your library`, note: `Shared tag: ${matchingOwnedTag}`, url: null }; evidence: { id: `taste:library:${normalize(title)}`, kind: 'personal', label: `${title} in your library`, note: `Shared tag: ${matchingOwnedTag}`, url: null } });
} }
else reason = candidate.evidence[0]?.note || 'A curated manga worth a closer look.'; for (const note of seedNotes(candidate, seeds, today.getFullYear())) { scoreParts[note.part] = note.value; notes.push(note); }
reason = reason.slice(0, 280); notes.sort((a, b) => scoreParts[b.part] - scoreParts[a.part] || partRank(a.part) - partRank(b.part));
const evidence = [...candidate.evidence, ...(personalEvidence ? [personalEvidence] : [])]; const reasons = notes.map(note => String(note.text).slice(0, 280));
rows.push({ candidateId: candidate.id, reason, evidence, evidenceIds: evidence.map(e => e.id), scoreParts, const fallback = candidate.evidence[0]?.note || (candidate.greatestId || relation ? 'A curated manga worth a closer look.' : 'A manga from AniList worth a closer look.');
score: Object.values(scoreParts).reduce((a, b) => a + b, 0), period: candidate.period || 'Related works' }); const evidence = [...candidate.evidence];
for (const note of notes) if (note.evidence && !evidence.some(e => e.id === note.evidence.id)) evidence.push(note.evidence);
rows.push({ candidateId: candidate.id, reason: (reasons[0] || fallback).slice(0, 280), reasons, evidence,
evidenceIds: evidence.map(e => e.id), scoreParts, score: round2(Object.values(scoreParts).reduce((a, b) => a + b, 0)),
period: candidate.period || 'Related works', group: groupFor(scoreParts, candidate.period), tags: candidate.tags });
} }
rows.sort((a, b) => b.score - a.score || a.candidateId.localeCompare(b.candidateId)); // Tags only steer the slot mix; the published row does not repeat them.
const picked = [], periodCounts = new Map(); const items = mixSlots(rows, { topTags: seeds.topTags }).map(({ tags, ...row }) => row);
for (const row of rows) { return { generatedAt: today.toISOString(), items };
if (picked.length >= MAX_FEED) break;
if ((periodCounts.get(row.period) || 0) >= 4) continue;
picked.push(row); periodCounts.set(row.period, (periodCounts.get(row.period) || 0) + 1);
} }
return { generatedAt: new Date().toISOString(), items: picked.map((row, index) => ({ ...row, rank: index + 1 })) }; // Sample across eras (and seed kinds) before the work cap; insertion order in the curated
} // catalog must not make recent manga invisible.
function generate({ db, library, greatest, cacheDir, queuedTitles = [] }) { function candidatePool({ greatest, library, cacheDir, harvestCache }) {
const all = buildCandidates({ greatest, library, cacheDir }); const all = buildCandidates({ greatest, library, cacheDir, harvestCache });
// Sample across eras before the work cap; insertion order in the curated catalog
// must not make recent manga invisible.
const buckets = new Map(); const buckets = new Map();
for (const candidate of all) { for (const candidate of all) {
const key = candidate.period || 'Related works'; const key = candidate.period || candidate.seed || 'Related works';
if (!buckets.has(key)) buckets.set(key, []); if (!buckets.has(key)) buckets.set(key, []);
buckets.get(key).push(candidate); buckets.get(key).push(candidate);
} }
@@ -165,10 +215,44 @@ function generate({ db, library, greatest, cacheDir, queuedTitles = [] }) {
if (values.length && candidates.length < MAX_CANDIDATES) candidates.push(values.shift()); if (values.length && candidates.length < MAX_CANDIDATES) candidates.push(values.shift());
} }
} }
return { all, candidates };
}
const runResult = (all, candidates, feeds) => ({ candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds,
examined: all.length, clipped: Math.max(0, all.length - candidates.length) });
function generate({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], now }) {
const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache });
const feeds = {}; const feeds = {};
for (const user of Object.values(db.users || {})) feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles }); for (const user of Object.values(db.users || {}))
return { candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds, feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now });
examined: all.length, clipped: Math.max(0, all.length - candidates.length) }; return runResult(all, candidates, feeds);
}
// Same result as generate(), but yields to the event loop between users so the reader stays responsive.
async function generateAsync({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], onlyUser, now }) {
const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache });
const feeds = {};
for (const user of Object.values(db.users || {})) {
if (onlyUser !== undefined && user.name !== onlyUser) continue;
await new Promise(setImmediate);
feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now });
}
return runResult(all, candidates, feeds);
}
// A per-user rescore replaces one feed; the others and the candidates they show are carried over.
function mergeUserFeed(prior, result, userName) {
const feeds = { ...(prior?.feeds || {}) };
if (result.feeds?.[userName]) feeds[userName] = result.feeds[userName];
else delete feeds[userName];
const candidates = {};
for (const [name, feed] of Object.entries(feeds)) {
if (name === userName) continue;
for (const item of feed?.items || []) {
const candidate = prior?.candidates?.[item.candidateId];
if (candidate) candidates[item.candidateId] = candidate;
}
}
Object.assign(candidates, result.candidates || {});
return { candidates, feeds };
} }
module.exports = { generate, buildCandidates, feedForUser, cachedMetadata, MAX_CANDIDATES, MAX_FEED }; module.exports = { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, cachedMetadata,
isMature, MAX_CANDIDATES, MAX_FEED, SLOTS };
discovery/scoring.js
+156
View File
@@ -0,0 +1,156 @@
'use strict';
// Pure scoring helpers for the AniList seed parts and the feed slot mix (BL-0051).
const { topTagsFor } = require('./anchors');
const MAX_FEED = 20;
const SLOTS = { perKind: 8, perPeriod: 4, fresh: 4, outside: 4 };
const CREATOR_MIN_WEIGHT = 3;
const TAG_TOP_MIN_SCORE = 75;
const FRESH_YEARS = 2;
// Part name -> group (seed kind) for the four AniList seed parts.
const SEED_PART_GROUP = { communityRec: 'recs', sameCreator: 'creator', tagTop: 'tagTop', fresh: 'fresh' };
const SEED_GROUPS = new Set(Object.values(SEED_PART_GROUP));
// Tie-break when two parts are equal: seed parts first, then Phase A order.
const PART_ORDER = ['communityRec', 'sameCreator', 'tagTop', 'fresh',
'interest', 'editorial', 'sharedTaste', 'relation', 'era', 'reading', 'sharedLibrary'];
const round2 = value => Math.round(value * 100) / 100;
const listOf = value => Array.isArray(value) ? value : [];
const partRank = part => { const index = PART_ORDER.indexOf(part); return index < 0 ? PART_ORDER.length : index; };
function communityRecValue(rating, weight) {
const safe = Math.max(0, Number(rating) || 0);
return Math.min(5, 1 + Math.log2(1 + safe) / 2) * weight / 5;
}
const sameCreatorValue = weight => 4 * weight / 5;
// What the harvest cache says about this user's anchors: one lookup map per seed kind.
function seedIndexFor({ anchors, harvestCache, mature = false }) {
const index = { recs: new Map(), creators: new Map(), tagTop: new Map(), fresh: new Map(), topTags: [] };
if (!harvestCache) return index;
const recordOf = id => harvestCache.get('title', id)?.data || null;
const nameOf = anchor => anchor.title || recordOf(anchor.anilistId)?.title || `AniList ${anchor.anilistId}`;
for (const anchor of anchors) {
const name = nameOf(anchor);
const entry = harvestCache.get('recs', anchor.anilistId);
for (const item of listOf(entry?.data)) {
const id = `anilist:${Number(item?.anilistId)}`;
if (!index.recs.has(id)) index.recs.set(id, []);
index.recs.get(id).push({ anchor, name, rating: Number(item.rating) || 0, record: item.record || null, fetchedAt: entry.fetchedAt });
}
if (anchor.weight < CREATOR_MIN_WEIGHT) continue;
for (const author of listOf(recordOf(anchor.anilistId)?.authors)) {
const prior = index.creators.get(author.staffId);
if (prior && prior.anchor.weight >= anchor.weight) continue;
index.creators.set(author.staffId, { anchor, name, author,
fetchedAt: harvestCache.get('creator', author.staffId)?.fetchedAt || null });
}
}
index.topTags = topTagsFor(anchors, recordOf, { mature });
index.topTags.forEach((tag, place) => {
const entry = harvestCache.get('tagTop', `${tag.kind}:${tag.name}`);
for (const item of listOf(entry?.data)) {
const id = `anilist:${Number(item?.anilistId)}`;
// Tags come best-first, so the first hit is the strongest.
if (!index.tagTop.has(id)) index.tagTop.set(id, { tag, value: 3 - place, record: item.record || null, fetchedAt: entry.fetchedAt });
}
});
const fresh = harvestCache.get('fresh', 'global');
for (const item of listOf(fresh?.data)) {
index.fresh.set(`anilist:${Number(item?.anilistId)}`, { record: item.record || null, fetchedAt: fresh.fetchedAt });
}
return index;
}
// Seed parts for one candidate with the reason and evidence each one brings.
function seedNotes(candidate, seeds, thisYear) {
const notes = [];
const urlFor = record => record?.siteUrl && /^https:\/\/[^\s]+$/i.test(String(record.siteUrl)) ? String(record.siteUrl) : candidate.metadataUrl || null;
let bestRec = null;
for (const hit of seeds.recs.get(candidate.id) || []) {
const value = communityRecValue(hit.rating, hit.anchor.weight);
if (!bestRec || value > bestRec.value) bestRec = { ...hit, value };
}
if (bestRec) notes.push({ part: 'communityRec', value: round2(bestRec.value),
text: `AniList readers who liked ${bestRec.name} recommend this (${bestRec.rating}).`,
evidence: { id: `rec:${bestRec.anchor.anilistId}:${candidate.id}`, kind: 'recommendation',
label: `AniList recommendation from readers of ${bestRec.name}`, note: `Rating ${bestRec.rating}`,
url: urlFor(bestRec.record), fetchedAt: bestRec.fetchedAt } });
let bestCreator = null;
for (const author of listOf(candidate.authorRecords)) {
const hit = seeds.creators.get(author?.staffId);
// An anchor is never its own "also by" suggestion.
if (!hit || `anilist:${hit.anchor.anilistId}` === candidate.id) continue;
if (!bestCreator || hit.anchor.weight > bestCreator.anchor.weight) bestCreator = { ...hit, candidateAuthor: author };
}
if (bestCreator) {
const author = bestCreator.candidateAuthor.name || bestCreator.author.name;
const verb = bestCreator.anchor.source === 'read' ? 'read' : 'liked';
notes.push({ part: 'sameCreator', value: round2(sameCreatorValue(bestCreator.anchor.weight)),
text: `Also by ${author} — you ${verb} ${bestCreator.name}.`,
evidence: { id: `creator:${bestCreator.author.staffId}:${candidate.id}`, kind: 'creator',
label: `Also by ${author}`, note: `Shared creator with ${bestCreator.name}`,
url: urlFor(null), fetchedAt: bestCreator.fetchedAt || candidate.fetchedAt || null } });
}
const top = seeds.tagTop.get(candidate.id);
const averageScore = Number(candidate.averageScore ?? top?.record?.averageScore);
if (top && averageScore >= TAG_TOP_MIN_SCORE) notes.push({ part: 'tagTop', value: top.value,
text: `Top-rated in ${top.tag.name}, one of your most-liked tags.`,
evidence: { id: `tagtop:${top.tag.kind}:${top.tag.name}:${candidate.id}`, kind: 'tag-top',
label: `Top-rated in ${top.tag.name} on AniList`, note: `Average score ${averageScore}`,
url: urlFor(top.record), fetchedAt: top.fetchedAt } });
const fresh = seeds.fresh.get(candidate.id);
if (fresh) {
const status = fresh.record?.status ?? candidate.status;
const year = Number(fresh.record?.year ?? candidate.year);
const isNew = status === 'ongoing' && year >= thisYear - FRESH_YEARS;
notes.push({ part: 'fresh', value: isNew ? 2 : 1,
// The exact template claims a start year and "ongoing", which only the value-2 case proves.
text: isNew ? `New: started ${year}, ongoing and trending.` : 'Trending on AniList right now.',
evidence: { id: `fresh:${candidate.id}`, kind: 'fresh', label: 'Trending new manga on AniList', note: '',
url: urlFor(fresh.record), fetchedAt: fresh.fetchedAt } });
}
return notes;
}
function largestPart(scoreParts) {
let best = null;
for (const [part, value] of Object.entries(scoreParts)) {
if (!best || value > best.value || (value === best.value && partRank(part) < partRank(best.part))) best = { part, value };
}
return best;
}
function groupFor(scoreParts, period) {
const top = largestPart(scoreParts);
return top && top.value > 0 && SEED_PART_GROUP[top.part] ? SEED_PART_GROUP[top.part] : period || 'Related works';
}
const byScore = (a, b) => b.score - a.score || (a.candidateId < b.candidateId ? -1 : a.candidateId > b.candidateId ? 1 : 0);
// Reserve slots for fresh and outside-the-top-tags rows, then fill greedily; every step respects the group caps.
function mixSlots(rows, { topTags = [], max = MAX_FEED } = {}) {
const sorted = [...rows].sort(byScore);
const picked = new Set(), counts = new Map();
const capOf = group => SEED_GROUPS.has(group) ? SLOTS.perKind : SLOTS.perPeriod;
const take = row => {
if (picked.has(row) || picked.size >= max || (counts.get(row.group) || 0) >= capOf(row.group)) return false;
picked.add(row); counts.set(row.group, (counts.get(row.group) || 0) + 1);
return true;
};
const reserve = (limit, wanted) => {
let taken = 0;
for (const row of sorted) if (taken < limit && wanted(row) && take(row)) taken++;
};
const top = new Set(topTags.map(tag => String(tag.name).toLowerCase()));
reserve(SLOTS.fresh, row => row.scoreParts?.fresh > 0);
reserve(SLOTS.outside, row => !listOf(row.tags).some(tag => top.has(String(tag).toLowerCase())));
for (const row of sorted) take(row);
return [...picked].sort(byScore).map((row, index) => ({ ...row, rank: index + 1 }));
}
module.exports = { MAX_FEED, SLOTS, PART_ORDER, round2, communityRecValue, sameCreatorValue,
seedIndexFor, seedNotes, largestPart, groupFor, partRank, mixSlots };
test/discovery.test.js
+225 -2
View File
@@ -4,8 +4,9 @@ const assert = require('node:assert/strict');
const fs = require('node:fs'); const fs = require('node:fs');
const os = require('node:os'); const os = require('node:os');
const path = require('node:path'); const path = require('node:path');
const { generate } = require('../discovery/engine'); const { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, MAX_CANDIDATES, SLOTS } = require('../discovery/engine');
const { DiscoveryStore } = require('../discovery/store'); const { DiscoveryStore } = require('../discovery/store');
const { DiscoveryCache } = require('../discovery/cache');
test('discovery is bounded, diversified, opt-in, evidence-linked and user-specific', () => { test('discovery is bounded, diversified, opt-in, evidence-linked and user-specific', () => {
const { loadCatalog } = require('../greatest'); const { loadCatalog } = require('../greatest');
@@ -16,7 +17,7 @@ test('discovery is bounded, diversified, opt-in, evidence-linked and user-specif
const db = { users, greatestRatings: { alice: { [greatest.entries[0].id]: 'flame' }, const db = { users, greatestRatings: { alice: { [greatest.entries[0].id]: 'flame' },
bob: { [greatest.entries[20].id]: 'flame' } }, discoveryFeedback: { alice: {} }, progress: {} }; bob: { [greatest.entries[20].id]: 'flame' } }, discoveryFeedback: { alice: {} }, progress: {} };
const result = generate({ db, library: [], greatest, cacheDir: path.join(os.tmpdir(), 'nonexistent-discovery-cache') }); const result = generate({ db, library: [], greatest, cacheDir: path.join(os.tmpdir(), 'nonexistent-discovery-cache') });
assert.equal(Object.keys(result.candidates).length, 50); assert.equal(Object.keys(result.candidates).length, Math.min(2000, result.examined));
assert.ok(result.feeds.alice.items.length > 0 && result.feeds.alice.items.length <= 20); assert.ok(result.feeds.alice.items.length > 0 && result.feeds.alice.items.length <= 20);
assert.ok(result.feeds.bob.items.length > 0); assert.ok(result.feeds.bob.items.length > 0);
assert.equal(result.feeds.off.disabled, true); assert.equal(result.feeds.off.disabled, true);
@@ -85,3 +86,225 @@ test('patch refuses a corrupt store', t => {
assert.throws(() => store.patch({ harvest: {} }), /refusing to overwrite/); assert.throws(() => store.patch({ harvest: {} }), /refusing to overwrite/);
assert.equal(fs.readFileSync(store.file, 'utf8'), '{broken'); assert.equal(fs.readFileSync(store.file, 'utf8'), '{broken');
}); });
// ---- Wider pool (BL-0051): AniList seeds, new score parts, slot mix ----
const NOW = () => new Date('2026-10-25T12:00:00Z');
const FETCHED_AT = '2026-10-20T00:00:00.000Z';
const NO_SCRAPER_CACHE = path.join(os.tmpdir(), 'nonexistent-discovery-cache');
const AUTHOR = { staffId: 7, name: 'Author Seven', role: 'Story' };
function seedRecord(id, extra = {}) {
return { anilistId: id, title: `Seed ${id}`, altTitles: [], format: 'MANGA', isAdult: false, status: 'completed',
year: 2010, averageScore: 70, popularity: 1000, trending: 0,
tags: [{ name: 'Action', rank: 90, kind: 'genre', isAdult: false }],
authors: [{ staffId: 900 + id, name: `Writer ${id}`, role: 'Story' }],
coverUrl: `https://img.example/${id}.jpg`, siteUrl: `https://anilist.co/manga/${id}`, ...extra };
}
const seed = (id, extra) => ({ anilistId: id, record: seedRecord(id, extra) });
function harvestFixture(t) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-harvest-'));
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
const cache = new DiscoveryCache(dir, { now: () => Date.parse(FETCHED_AT) });
cache.put('title', 100, seedRecord(100, { title: 'Anchor One', authors: [AUTHOR],
tags: [{ name: 'Drama', rank: 100, kind: 'genre', isAdult: false }, { name: 'Time Skip', rank: 80, kind: 'tag', isAdult: false }] }));
cache.put('recs', 100, [
{ ...seed(201), rating: 42 },
{ ...seed(209, { authors: [AUTHOR] }), rating: 10 },
{ ...seed(207, { isAdult: true }), rating: 5 },
{ ...seed(208, { tags: [{ name: 'Nudity', rank: 65, kind: 'tag', isAdult: true }] }), rating: 5 },
{ ...seed(212), rating: 3 },
]);
cache.put('creator', 7, [{ ...seed(202, { authors: [AUTHOR] }), role: 'Story' }, { ...seed(209, { authors: [AUTHOR] }), role: 'Story' }]);
cache.put('tagTop', 'genre:Drama', [seed(203, { averageScore: 80, tags: [{ name: 'Drama', rank: 90, kind: 'genre', isAdult: false }] })]);
cache.put('tagTop', 'tag:Time Skip', [seed(204, { averageScore: 70 })]);
cache.put('fresh', 'global', [seed(205, { status: 'ongoing', year: 2025 }), seed(206, { status: 'ongoing', year: 2020 }),
seed(210, { title: 'Owned Title', year: 2020 }), seed(211, { title: 'Queued Title' }), seed(213, { status: 'ongoing', year: 2026 })]);
return cache;
}
const fixtureGreatest = { entries: [
{ id: 'g1', title: 'Anchor One', period: '2000s', rank: 1, note: '', metadata: { id: 100 } },
{ id: 'g2', title: 'Voted Two', period: '1990s', rank: 2, note: '', metadata: { id: 212 } },
] };
function fixtureWorld(t, settings = {}) {
const user = { name: 'alice', settings: { discoveryEnabled: true, ...settings } };
const db = { users: { alice: user }, greatestRatings: { alice: { g1: 'flame', g2: 'good' } },
discoveryFeedback: { alice: { 'anilist:213': 'dismissed' } }, progress: {} };
const library = [{ id: 'm1', title: 'Owned Title', meta: { year: 2020, 'x-anilist': { id: 210 } } }];
return { user, db, library, greatest: fixtureGreatest, harvestCache: harvestFixture(t), queuedTitles: ['Queued Title'] };
}
function fixtureFeed(t, settings) {
const world = fixtureWorld(t, settings);
const candidates = buildCandidates({ greatest: world.greatest, library: world.library, cacheDir: NO_SCRAPER_CACHE, harvestCache: world.harvestCache });
const feed = feedForUser({ ...world, candidates, now: NOW });
return { world, candidates, feed, item: id => feed.items.find(item => item.candidateId === `anilist:${id}`) };
}
test('engine: empty harvest cache equals Phase A pool', t => {
const { loadCatalog } = require('../greatest');
const greatest = loadCatalog();
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-empty-'));
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
const db = { users: {}, greatestRatings: {}, discoveryFeedback: {}, progress: {} };
const without = generate({ db, library: [], greatest, cacheDir: NO_SCRAPER_CACHE });
const withEmpty = generate({ db, library: [], greatest, cacheDir: NO_SCRAPER_CACHE, harvestCache: new DiscoveryCache(dir) });
assert.equal(MAX_CANDIDATES, 2000);
assert.deepEqual(Object.keys(withEmpty.candidates).sort(), Object.keys(without.candidates).sort());
});
test('harvested records join the pool and fill Greatest candidates', t => {
const { candidates } = fixtureFeed(t);
const byId = new Map(candidates.map(c => [c.id, c]));
for (const id of [201, 202, 203, 204, 205, 206, 207, 208, 209]) assert.ok(byId.has(`anilist:${id}`), `anilist:${id}`);
const fromRecs = byId.get('anilist:201');
assert.equal(fromRecs.isAdult, false);
assert.equal(fromRecs.averageScore, 70);
assert.deepEqual(fromRecs.tags, ['Action']);
assert.deepEqual(fromRecs.authors, ['Writer 201']);
assert.equal(fromRecs.tagRecords[0].rank, 90);
assert.equal(fromRecs.authorRecords[0].staffId, 1101);
assert.equal(fromRecs.fetchedAt, FETCHED_AT);
const greatestOne = byId.get('anilist:212');
assert.equal(greatestOne.greatestId, 'g2');
assert.equal(greatestOne.title, 'Voted Two');
assert.equal(greatestOne.isAdult, false);
assert.equal(greatestOne.tagRecords.length, 1);
});
test('community rec, creator, tagTop and fresh parts and reasons', t => {
const { item } = fixtureFeed(t);
const rec = item(201);
assert.equal(rec.scoreParts.communityRec, 3.71);
assert.equal(rec.group, 'recs');
assert.equal(rec.reason, 'AniList readers who liked Anchor One recommend this (42).');
const recEvidence = rec.evidence.find(e => e.id === 'rec:100:anilist:201');
assert.equal(recEvidence.kind, 'recommendation');
assert.equal(recEvidence.fetchedAt, FETCHED_AT);
assert.equal(recEvidence.url, 'https://anilist.co/manga/201');
assert.ok(rec.evidenceIds.includes('rec:100:anilist:201'));
const creator = item(202);
assert.equal(creator.scoreParts.sameCreator, 4);
assert.equal(creator.group, 'creator');
assert.equal(creator.reason, 'Also by Author Seven — you liked Anchor One.');
const creatorEvidence = creator.evidence.find(e => e.id === 'creator:7:anilist:202');
assert.equal(creatorEvidence.kind, 'creator');
assert.equal(creatorEvidence.fetchedAt, FETCHED_AT);
const top = item(203);
assert.equal(top.scoreParts.tagTop, 3);
assert.equal(top.group, 'tagTop');
assert.equal(top.reason, 'Top-rated in Drama, one of your most-liked tags.');
assert.equal(top.evidence.find(e => e.id === 'tagtop:genre:Drama:anilist:203').kind, 'tag-top');
assert.equal(item(204).scoreParts.tagTop, 0, 'averageScore below 75 earns no tagTop part');
const fresh = item(205);
assert.equal(fresh.scoreParts.fresh, 2);
assert.equal(fresh.group, 'fresh');
assert.equal(fresh.reason, 'New: started 2025, ongoing and trending.');
const freshEvidence = fresh.evidence.find(e => e.id === 'fresh:anilist:205');
assert.equal(freshEvidence.kind, 'fresh');
assert.equal(freshEvidence.fetchedAt, FETCHED_AT);
assert.equal(item(206).scoreParts.fresh, 1);
for (const row of [rec, creator, top, fresh]) assert.ok(row.reason.length <= 280);
});
test('isAdult filter', t => {
const plain = fixtureFeed(t);
assert.equal(plain.item(207), undefined);
assert.equal(plain.item(208), undefined, 'an adult tag with rank >= 60 counts as mature');
const mature = fixtureFeed(t, { discoveryMature: true });
assert.ok(mature.item(207));
assert.ok(mature.item(208));
});
test('multiple seeds merge reasons', t => {
const { item } = fixtureFeed(t);
const both = item(209);
assert.equal(both.scoreParts.sameCreator, 4);
assert.equal(both.scoreParts.communityRec, 2.73);
assert.equal(both.score, 6.73);
assert.equal(both.group, 'creator');
assert.deepEqual(both.reasons, ['Also by Author Seven — you liked Anchor One.',
'AniList readers who liked Anchor One recommend this (10).']);
assert.equal(both.reason, both.reasons[0]);
assert.ok(both.evidenceIds.includes('rec:100:anilist:209'));
assert.ok(both.evidenceIds.includes('creator:7:anilist:209'));
});
test('owned, queued, voted, dismissed still excluded', t => {
const { item, candidates } = fixtureFeed(t);
for (const id of [210, 211, 212, 213]) {
assert.ok(candidates.some(c => c.id === `anilist:${id}`), `anilist:${id} is in the pool`);
assert.equal(item(id), undefined, `anilist:${id} must not be suggested`);
}
});
test('slot mix', () => {
const row = (id, score, group, extra = {}) => ({ candidateId: `anilist:${id}`, score, group, tags: ['Drama'],
scoreParts: { fresh: 0 }, ...extra });
const rows = [];
for (let i = 0; i < 30; i++) rows.push(row(1000 + i, 30 - i, 'recs'));
rows.push(row(2001, 0.5, 'fresh', { scoreParts: { fresh: 1 } }), row(2002, 0.4, 'fresh', { scoreParts: { fresh: 1 } }));
rows.push(row(3001, 0.2, '1990s', { tags: ['Sports'] }), row(3002, 0.1, '1990s', { tags: [] }));
const picked = mixSlots(rows, { topTags: [{ name: 'Drama', kind: 'genre' }] });
assert.equal(SLOTS.perKind, 8);
assert.equal(picked.filter(r => r.group === 'recs').length, 8);
assert.ok(picked.some(r => r.candidateId === 'anilist:2001') && picked.some(r => r.candidateId === 'anilist:2002'));
assert.ok(picked.some(r => r.candidateId === 'anilist:3001') && picked.some(r => r.candidateId === 'anilist:3002'));
assert.equal(picked.length, 12, 'never pads up to 20');
assert.deepEqual(picked.map(r => r.rank), picked.map((_, i) => i + 1));
for (let i = 1; i < picked.length; i++) assert.ok(picked[i - 1].score >= picked[i].score);
const many = [];
for (let i = 0; i < 40; i++) many.push(row(4000 + i, 100 - i, ['recs', 'creator', 'tagTop', 'fresh', '1980s'][i % 5],
{ scoreParts: { fresh: i % 5 === 3 ? 1 : 0 } }));
const full = mixSlots(many, { topTags: [] });
assert.equal(full.length, 20);
assert.ok(full.filter(r => r.group === '1980s').length <= SLOTS.perPeriod);
});
test('mergeUserFeed keeps other users\' referenced candidates', () => {
const prior = { candidates: { a: { id: 'a' }, b: { id: 'b' }, c: { id: 'c' } },
feeds: { alice: { items: [{ candidateId: 'a' }] }, bob: { items: [{ candidateId: 'b' }] }, off: { items: [], disabled: true } } };
const result = { candidates: { d: { id: 'd' }, b: { id: 'b', fresh: true } }, feeds: { alice: { items: [{ candidateId: 'd' }] } } };
const merged = mergeUserFeed(prior, result, 'alice');
assert.deepEqual(Object.keys(merged.feeds).sort(), ['alice', 'bob', 'off']);
assert.deepEqual(merged.feeds.alice.items, [{ candidateId: 'd' }]);
assert.deepEqual(merged.feeds.bob, prior.feeds.bob);
assert.deepEqual(Object.keys(merged.candidates).sort(), ['b', 'd']);
assert.equal(merged.candidates.b.fresh, true);
});
test('generateAsync matches generate and can score one user', async t => {
const world = fixtureWorld(t);
world.db.users.bob = { name: 'bob', settings: { discoveryEnabled: true } };
const args = { db: world.db, library: world.library, greatest: world.greatest, cacheDir: NO_SCRAPER_CACHE,
harvestCache: world.harvestCache, queuedTitles: world.queuedTitles, now: NOW };
const sync = generate(args);
const all = await generateAsync(args);
assert.deepEqual(all.feeds.alice.items, sync.feeds.alice.items);
assert.deepEqual(Object.keys(all.candidates), Object.keys(sync.candidates));
const one = await generateAsync({ ...args, onlyUser: 'alice' });
assert.deepEqual(Object.keys(one.feeds), ['alice']);
assert.deepEqual(one.feeds.alice.items, sync.feeds.alice.items);
});
test('scoring one user stays under 200 ms on 2000 candidates', t => {
const world = fixtureWorld(t);
const tagNames = ['Drama', 'Action', 'Romance', 'Time Skip', 'Comedy', 'Horror'];
const candidates = [];
for (let i = 0; i < 2000; i++) {
const id = 10000 + i;
candidates.push({ id: `anilist:${id}`, title: `Synthetic ${id}`, year: 2000 + (i % 27), type: 'MANGA',
status: i % 3 ? 'completed' : 'ongoing', cover: null, metadataUrl: `https://anilist.co/manga/${id}`,
tags: [tagNames[i % 6], tagNames[(i + 1) % 6]], authors: [`Writer ${i % 50}`], period: null, editorialRank: null,
greatestId: null, evidence: [], relations: [], isAdult: i % 97 === 0, averageScore: 60 + (i % 40),
tagRecords: [{ name: tagNames[i % 6], rank: 80, kind: 'tag', isAdult: false }],
authorRecords: [{ staffId: i % 50, name: `Writer ${i % 50}`, role: 'Story' }], fetchedAt: FETCHED_AT, seed: 'recs' });
}
feedForUser({ ...world, candidates, now: NOW });
const started = performance.now();
const feed = feedForUser({ ...world, candidates, now: NOW });
const elapsed = performance.now() - started;
assert.ok(feed.items.length > 0);
assert.ok(elapsed < 200, `took ${elapsed.toFixed(1)} ms`);
});