diff --git a/discovery/cli.js b/discovery/cli.js index 76f176d..9498ea7 100644 --- a/discovery/cli.js +++ b/discovery/cli.js @@ -5,6 +5,7 @@ const fs = require('node:fs'); const path = require('node:path'); const { loadCatalog } = require('../greatest'); const { generate } = require('./engine'); +const { DiscoveryCache } = require('./cache'); if (!process.argv.includes('--dry-run')) { console.error('Usage: node discovery/cli.js run --dry-run [--data DIR] [--catalog FILE] [--show]'); @@ -17,7 +18,10 @@ let library = []; try { library = JSON.parse(fs.readFileSync(path.join(data, 'library-index.json'), 'utf8')).mangas || []; } catch (error) { if (error.code !== 'ENOENT') throw error; } const greatest = loadCatalog(arg('--catalog', path.join(__dirname, '..', 'greatest', 'catalog.json'))); -const result = generate({ db, library, greatest, cacheDir: path.join(data, 'scraper', 'metadata', 'anilist') }); +// The harvest cache is only read here (get/list, never put); note get/list still drop a corrupt entry file. +const harvestDir = path.join(data, 'discovery-cache'); +const harvestCache = fs.existsSync(harvestDir) ? new DiscoveryCache(harvestDir) : null; +const result = generate({ db, library, greatest, cacheDir: path.join(data, 'scraper', 'metadata', 'anilist'), harvestCache }); const summary = { examined: result.examined, candidates: Object.keys(result.candidates).length, optedInUsers: Object.values(db.users || {}).filter(u => u.settings?.discoveryEnabled).length, feedSizes: Object.values(result.feeds).filter(f => !f.disabled).map(f => f.items.length) }; diff --git a/discovery/engine.js b/discovery/engine.js index 26d8853..ee4a15a 100644 --- a/discovery/engine.js +++ b/discovery/engine.js @@ -3,9 +3,12 @@ const fs = require('node:fs'); const path = require('node:path'); const { normalize } = require('../greatest'); +const { anchorsFor } = require('./anchors'); +const { MAX_FEED, SLOTS, round2, seedIndexFor, seedNotes, groupFor, partRank, mixSlots } = require('./scoring'); -const MAX_CANDIDATES = 50; -const MAX_FEED = 20; +const MAX_CANDIDATES = 2000; +const SEED_KINDS = ['recs', 'creator', 'tagTop', 'fresh']; +const MATURE_TAG_RANK = 60; const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']); const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null; const unique = values => [...new Set(values.filter(Boolean))]; @@ -53,7 +56,42 @@ function addRelations(map, anchor, cacheDir) { }); } } -function buildCandidates({ greatest, library, cacheDir }) { +const textList = values => unique(Array.isArray(values) ? values.map(value => value?.name).filter(name => typeof name === 'string') : []); +const recordList = values => Array.isArray(values) ? values.filter(value => value && typeof value === 'object') : []; +function candidateFromRecord(record, id, seed, fetchedAt) { + return { + id: `anilist:${id}`, title: String(record.title || `AniList ${id}`), year: Number(record.year) || null, + type: record.format || 'MANGA', status: record.status || 'unknown', cover: safeUrl(record.coverUrl), + metadataUrl: safeUrl(record.siteUrl) || `https://anilist.co/manga/${id}`, + tags: textList(record.tags), authors: textList(record.authors), period: null, editorialRank: null, greatestId: null, + evidence: [], relations: [], isAdult: record.isAdult === true, + averageScore: Number.isFinite(record.averageScore) ? record.averageScore : null, + tagRecords: recordList(record.tags), authorRecords: recordList(record.authors), fetchedAt, seed, + }; +} +// Greatest and relation candidates keep their own fields; AniList only fills what they lack. +function fillFromRecord(candidate, record, fetchedAt) { + if (candidate.isAdult === undefined) candidate.isAdult = record.isAdult === true; + if (!candidate.tagRecords) candidate.tagRecords = recordList(record.tags); + if (!candidate.authorRecords) candidate.authorRecords = recordList(record.authors); + if (candidate.averageScore == null && Number.isFinite(record.averageScore)) candidate.averageScore = record.averageScore; + if (!candidate.fetchedAt) candidate.fetchedAt = fetchedAt; +} +function addHarvested(map, harvestCache) { + for (const kind of SEED_KINDS) { + const entries = harvestCache.list(kind).sort((a, b) => String(a.key).localeCompare(String(b.key))); + for (const entry of entries) for (const item of Array.isArray(entry.data) ? entry.data : []) { + const record = item?.record; + const id = Number(item?.anilistId); + if (!record || typeof record !== 'object' || !Number.isSafeInteger(id) || id < 1) continue; + if (record.format && !VALID_FORMATS.has(record.format)) continue; + const existing = map.get(`anilist:${id}`); + if (existing) fillFromRecord(existing, record, entry.fetchedAt); + else map.set(`anilist:${id}`, candidateFromRecord(record, id, kind, entry.fetchedAt)); + } + } +} +function buildCandidates({ greatest, library, cacheDir, harvestCache = null }) { const map = new Map(); for (const entry of greatest.entries) { const candidate = candidateFromGreatest(entry, cacheDir); @@ -65,8 +103,15 @@ function buildCandidates({ greatest, library, cacheDir }) { addRelations(map, { id: `anilist:${identity.id}`, title: manga.title, relations: identity.relations }, cacheDir); } for (const candidate of [...map.values()]) if (candidate.greatestId) addRelations(map, candidate, cacheDir); + if (harvestCache) addHarvested(map, harvestCache); return [...map.values()]; } +// Title-level isAdult or a strong adult tag; the Phase A regex only where AniList gave no isAdult. +function isMature(candidate) { + if (candidate.isAdult === undefined) return candidate.tags.some(tag => /adult|hentai|erotica/i.test(tag)); + return candidate.isAdult === true + || (candidate.tagRecords || []).some(tag => tag?.isAdult && Number(tag.rank) >= MATURE_TAG_RANK); +} function ownedKeySet(library) { const ids = new Set(), names = new Set(); for (const manga of library) { @@ -77,9 +122,10 @@ function ownedKeySet(library) { return { ids, names }; } function labelForVote(greatest, id) { return greatest.entries.find(entry => entry.id === id)?.title || null; } -function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [] }) { +function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [], harvestCache = null, now = () => new Date() }) { const settings = user.settings || {}; - if (!settings.discoveryEnabled) return { generatedAt: new Date().toISOString(), items: [], disabled: true }; + const today = now(); + if (!settings.discoveryEnabled) return { generatedAt: today.toISOString(), items: [], disabled: true }; const votes = db.greatestRatings?.[user.name] || {}; const feedback = db.discoveryFeedback?.[user.name] || {}; const owned = ownedKeySet(library); @@ -100,13 +146,14 @@ function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [ if (!Object.values(progress).some(chapter => chapter?.done)) continue; for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') readTags.set(tag.toLowerCase(), manga.title); } + const seeds = seedIndexFor({ anchors: anchorsFor({ user, db, library, greatest }), harvestCache, mature: !!settings.discoveryMature }); const rows = []; for (const candidate of candidates) { if (candidate.greatestId && votes[candidate.greatestId]) continue; if (['dismissed', 'hidden'].includes(feedback[candidate.id])) continue; if (owned.ids.has(candidate.id) || owned.names.has(`${normalize(candidate.title)}:${candidate.year || '?'}`)) continue; if (queued.has(normalize(candidate.title))) continue; - if (!settings.discoveryMature && candidate.tags.some(tag => /adult|hentai|erotica/i.test(tag))) continue; + if (!settings.discoveryMature && isMature(candidate)) continue; const matchingLikedTag = candidate.tags.find(tag => likedTags.has(tag.toLowerCase())); const matchingOwnedTag = candidate.tags.find(tag => ownedTags.has(tag.toLowerCase())); const matchingReadTag = candidate.tags.find(tag => readTags.has(tag.toLowerCase())); @@ -115,47 +162,50 @@ function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [ const scoreParts = { editorial: candidate.greatestId ? Math.max(0, 6 - Math.min(6, Number(candidate.editorialRank) || 6)) : 0, relation: relation ? 3 : 0, sharedTaste: matchingLikedTag ? 4 : 0, sharedLibrary: matchingOwnedTag ? 1 : 0, era: era ? 2 : 0, reading: matchingReadTag ? 2 : 0, - interest: feedback[candidate.id] === 'interested' ? 5 : feedback[candidate.id] === 'maybe' ? 2 : 0 }; - let reason, personalEvidence = null; - if (relation) reason = `${candidate.title} is listed as ${relation.type.toLowerCase().replaceAll('_', ' ')} to ${relation.title}. Check the edition and reading order first.`; - else if (matchingLikedTag) { + interest: feedback[candidate.id] === 'interested' ? 5 : feedback[candidate.id] === 'maybe' ? 2 : 0, + communityRec: 0, sameCreator: 0, tagTop: 0, fresh: 0 }; + // Every applicable part brings its reason and, for taste parts, its personal evidence. + const notes = []; + if (scoreParts.editorial && candidate.evidence[0]?.note) notes.push({ part: 'editorial', text: candidate.evidence[0].note }); + if (relation) notes.push({ part: 'relation', text: `${candidate.title} is listed as ${relation.type.toLowerCase().replaceAll('_', ' ')} to ${relation.title}. Check the edition and reading order first.` }); + if (matchingLikedTag) { const anchor = likedTags.get(matchingLikedTag.toLowerCase()); - reason = `You liked ${anchor.title}; this also carries the ${matchingLikedTag} tag.`; - personalEvidence = { id: `taste:greatest:${anchor.id}`, kind: 'personal', label: `Your Greatest vote for ${anchor.title}`, note: `Shared tag: ${matchingLikedTag}`, url: null }; - } else if (era) { - reason = `You liked ${era.title}; this is another curated work from ${candidate.period}.`; - personalEvidence = { id: `taste:greatest:${era.id}`, kind: 'personal', label: `Your Greatest vote for ${era.title}`, note: `Shared period: ${candidate.period}`, url: null }; - } else if (matchingReadTag) { - const title = readTags.get(matchingReadTag.toLowerCase()); - reason = `You have read ${title}; this also carries the ${matchingReadTag} tag.`; - personalEvidence = { id: `taste:read:${normalize(title)}`, kind: 'personal', label: `Your completed reading in ${title}`, note: `Shared tag: ${matchingReadTag}`, url: null }; - } else if (matchingOwnedTag) { - const title = ownedTags.get(matchingOwnedTag.toLowerCase()); - reason = `This shares the ${matchingOwnedTag} tag with ${title} in your library.`; - personalEvidence = { id: `taste:library:${normalize(title)}`, kind: 'personal', label: `${title} in your library`, note: `Shared tag: ${matchingOwnedTag}`, url: null }; + notes.push({ part: 'sharedTaste', text: `You liked ${anchor.title}; this also carries the ${matchingLikedTag} tag.`, + evidence: { id: `taste:greatest:${anchor.id}`, kind: 'personal', label: `Your Greatest vote for ${anchor.title}`, note: `Shared tag: ${matchingLikedTag}`, url: null } }); } - else reason = candidate.evidence[0]?.note || 'A curated manga worth a closer look.'; - reason = reason.slice(0, 280); - const evidence = [...candidate.evidence, ...(personalEvidence ? [personalEvidence] : [])]; - rows.push({ candidateId: candidate.id, reason, evidence, evidenceIds: evidence.map(e => e.id), scoreParts, - score: Object.values(scoreParts).reduce((a, b) => a + b, 0), period: candidate.period || 'Related works' }); + if (era) notes.push({ part: 'era', text: `You liked ${era.title}; this is another curated work from ${candidate.period}.`, + evidence: { id: `taste:greatest:${era.id}`, kind: 'personal', label: `Your Greatest vote for ${era.title}`, note: `Shared period: ${candidate.period}`, url: null } }); + if (matchingReadTag) { + const title = readTags.get(matchingReadTag.toLowerCase()); + notes.push({ part: 'reading', text: `You have read ${title}; this also carries the ${matchingReadTag} tag.`, + evidence: { id: `taste:read:${normalize(title)}`, kind: 'personal', label: `Your completed reading in ${title}`, note: `Shared tag: ${matchingReadTag}`, url: null } }); + } + if (matchingOwnedTag) { + const title = ownedTags.get(matchingOwnedTag.toLowerCase()); + notes.push({ part: 'sharedLibrary', text: `This shares the ${matchingOwnedTag} tag with ${title} in your library.`, + evidence: { id: `taste:library:${normalize(title)}`, kind: 'personal', label: `${title} in your library`, note: `Shared tag: ${matchingOwnedTag}`, url: null } }); + } + for (const note of seedNotes(candidate, seeds, today.getFullYear())) { scoreParts[note.part] = note.value; notes.push(note); } + notes.sort((a, b) => scoreParts[b.part] - scoreParts[a.part] || partRank(a.part) - partRank(b.part)); + const reasons = notes.map(note => String(note.text).slice(0, 280)); + const fallback = candidate.evidence[0]?.note || (candidate.greatestId || relation ? 'A curated manga worth a closer look.' : 'A manga from AniList worth a closer look.'); + const evidence = [...candidate.evidence]; + for (const note of notes) if (note.evidence && !evidence.some(e => e.id === note.evidence.id)) evidence.push(note.evidence); + rows.push({ candidateId: candidate.id, reason: (reasons[0] || fallback).slice(0, 280), reasons, evidence, + evidenceIds: evidence.map(e => e.id), scoreParts, score: round2(Object.values(scoreParts).reduce((a, b) => a + b, 0)), + period: candidate.period || 'Related works', group: groupFor(scoreParts, candidate.period), tags: candidate.tags }); } - rows.sort((a, b) => b.score - a.score || a.candidateId.localeCompare(b.candidateId)); - const picked = [], periodCounts = new Map(); - for (const row of rows) { - if (picked.length >= MAX_FEED) break; - if ((periodCounts.get(row.period) || 0) >= 4) continue; - picked.push(row); periodCounts.set(row.period, (periodCounts.get(row.period) || 0) + 1); - } - return { generatedAt: new Date().toISOString(), items: picked.map((row, index) => ({ ...row, rank: index + 1 })) }; + // Tags only steer the slot mix; the published row does not repeat them. + const items = mixSlots(rows, { topTags: seeds.topTags }).map(({ tags, ...row }) => row); + return { generatedAt: today.toISOString(), items }; } -function generate({ db, library, greatest, cacheDir, queuedTitles = [] }) { - const all = buildCandidates({ greatest, library, cacheDir }); - // Sample across eras before the work cap; insertion order in the curated catalog - // must not make recent manga invisible. +// Sample across eras (and seed kinds) before the work cap; insertion order in the curated +// catalog must not make recent manga invisible. +function candidatePool({ greatest, library, cacheDir, harvestCache }) { + const all = buildCandidates({ greatest, library, cacheDir, harvestCache }); const buckets = new Map(); for (const candidate of all) { - const key = candidate.period || 'Related works'; + const key = candidate.period || candidate.seed || 'Related works'; if (!buckets.has(key)) buckets.set(key, []); buckets.get(key).push(candidate); } @@ -165,10 +215,44 @@ function generate({ db, library, greatest, cacheDir, queuedTitles = [] }) { if (values.length && candidates.length < MAX_CANDIDATES) candidates.push(values.shift()); } } + return { all, candidates }; +} +const runResult = (all, candidates, feeds) => ({ candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds, + examined: all.length, clipped: Math.max(0, all.length - candidates.length) }); +function generate({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], now }) { + const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache }); const feeds = {}; - for (const user of Object.values(db.users || {})) feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles }); - return { candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds, - examined: all.length, clipped: Math.max(0, all.length - candidates.length) }; + for (const user of Object.values(db.users || {})) + feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now }); + return runResult(all, candidates, feeds); +} +// Same result as generate(), but yields to the event loop between users so the reader stays responsive. +async function generateAsync({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], onlyUser, now }) { + const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache }); + const feeds = {}; + for (const user of Object.values(db.users || {})) { + if (onlyUser !== undefined && user.name !== onlyUser) continue; + await new Promise(setImmediate); + feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now }); + } + return runResult(all, candidates, feeds); +} +// A per-user rescore replaces one feed; the others and the candidates they show are carried over. +function mergeUserFeed(prior, result, userName) { + const feeds = { ...(prior?.feeds || {}) }; + if (result.feeds?.[userName]) feeds[userName] = result.feeds[userName]; + else delete feeds[userName]; + const candidates = {}; + for (const [name, feed] of Object.entries(feeds)) { + if (name === userName) continue; + for (const item of feed?.items || []) { + const candidate = prior?.candidates?.[item.candidateId]; + if (candidate) candidates[item.candidateId] = candidate; + } + } + Object.assign(candidates, result.candidates || {}); + return { candidates, feeds }; } -module.exports = { generate, buildCandidates, feedForUser, cachedMetadata, MAX_CANDIDATES, MAX_FEED }; +module.exports = { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, cachedMetadata, + isMature, MAX_CANDIDATES, MAX_FEED, SLOTS }; \ No newline at end of file diff --git a/discovery/scoring.js b/discovery/scoring.js new file mode 100644 index 0000000..deae4ad --- /dev/null +++ b/discovery/scoring.js @@ -0,0 +1,156 @@ +'use strict'; + +// Pure scoring helpers for the AniList seed parts and the feed slot mix (BL-0051). +const { topTagsFor } = require('./anchors'); + +const MAX_FEED = 20; +const SLOTS = { perKind: 8, perPeriod: 4, fresh: 4, outside: 4 }; +const CREATOR_MIN_WEIGHT = 3; +const TAG_TOP_MIN_SCORE = 75; +const FRESH_YEARS = 2; +// Part name -> group (seed kind) for the four AniList seed parts. +const SEED_PART_GROUP = { communityRec: 'recs', sameCreator: 'creator', tagTop: 'tagTop', fresh: 'fresh' }; +const SEED_GROUPS = new Set(Object.values(SEED_PART_GROUP)); +// Tie-break when two parts are equal: seed parts first, then Phase A order. +const PART_ORDER = ['communityRec', 'sameCreator', 'tagTop', 'fresh', + 'interest', 'editorial', 'sharedTaste', 'relation', 'era', 'reading', 'sharedLibrary']; + +const round2 = value => Math.round(value * 100) / 100; +const listOf = value => Array.isArray(value) ? value : []; +const partRank = part => { const index = PART_ORDER.indexOf(part); return index < 0 ? PART_ORDER.length : index; }; + +function communityRecValue(rating, weight) { + const safe = Math.max(0, Number(rating) || 0); + return Math.min(5, 1 + Math.log2(1 + safe) / 2) * weight / 5; +} +const sameCreatorValue = weight => 4 * weight / 5; + +// What the harvest cache says about this user's anchors: one lookup map per seed kind. +function seedIndexFor({ anchors, harvestCache, mature = false }) { + const index = { recs: new Map(), creators: new Map(), tagTop: new Map(), fresh: new Map(), topTags: [] }; + if (!harvestCache) return index; + const recordOf = id => harvestCache.get('title', id)?.data || null; + const nameOf = anchor => anchor.title || recordOf(anchor.anilistId)?.title || `AniList ${anchor.anilistId}`; + for (const anchor of anchors) { + const name = nameOf(anchor); + const entry = harvestCache.get('recs', anchor.anilistId); + for (const item of listOf(entry?.data)) { + const id = `anilist:${Number(item?.anilistId)}`; + if (!index.recs.has(id)) index.recs.set(id, []); + index.recs.get(id).push({ anchor, name, rating: Number(item.rating) || 0, record: item.record || null, fetchedAt: entry.fetchedAt }); + } + if (anchor.weight < CREATOR_MIN_WEIGHT) continue; + for (const author of listOf(recordOf(anchor.anilistId)?.authors)) { + const prior = index.creators.get(author.staffId); + if (prior && prior.anchor.weight >= anchor.weight) continue; + index.creators.set(author.staffId, { anchor, name, author, + fetchedAt: harvestCache.get('creator', author.staffId)?.fetchedAt || null }); + } + } + index.topTags = topTagsFor(anchors, recordOf, { mature }); + index.topTags.forEach((tag, place) => { + const entry = harvestCache.get('tagTop', `${tag.kind}:${tag.name}`); + for (const item of listOf(entry?.data)) { + const id = `anilist:${Number(item?.anilistId)}`; + // Tags come best-first, so the first hit is the strongest. + if (!index.tagTop.has(id)) index.tagTop.set(id, { tag, value: 3 - place, record: item.record || null, fetchedAt: entry.fetchedAt }); + } + }); + const fresh = harvestCache.get('fresh', 'global'); + for (const item of listOf(fresh?.data)) { + index.fresh.set(`anilist:${Number(item?.anilistId)}`, { record: item.record || null, fetchedAt: fresh.fetchedAt }); + } + return index; +} + +// Seed parts for one candidate with the reason and evidence each one brings. +function seedNotes(candidate, seeds, thisYear) { + const notes = []; + const urlFor = record => record?.siteUrl && /^https:\/\/[^\s]+$/i.test(String(record.siteUrl)) ? String(record.siteUrl) : candidate.metadataUrl || null; + + let bestRec = null; + for (const hit of seeds.recs.get(candidate.id) || []) { + const value = communityRecValue(hit.rating, hit.anchor.weight); + if (!bestRec || value > bestRec.value) bestRec = { ...hit, value }; + } + if (bestRec) notes.push({ part: 'communityRec', value: round2(bestRec.value), + text: `AniList readers who liked ${bestRec.name} recommend this (${bestRec.rating}).`, + evidence: { id: `rec:${bestRec.anchor.anilistId}:${candidate.id}`, kind: 'recommendation', + label: `AniList recommendation from readers of ${bestRec.name}`, note: `Rating ${bestRec.rating}`, + url: urlFor(bestRec.record), fetchedAt: bestRec.fetchedAt } }); + + let bestCreator = null; + for (const author of listOf(candidate.authorRecords)) { + const hit = seeds.creators.get(author?.staffId); + // An anchor is never its own "also by" suggestion. + if (!hit || `anilist:${hit.anchor.anilistId}` === candidate.id) continue; + if (!bestCreator || hit.anchor.weight > bestCreator.anchor.weight) bestCreator = { ...hit, candidateAuthor: author }; + } + if (bestCreator) { + const author = bestCreator.candidateAuthor.name || bestCreator.author.name; + const verb = bestCreator.anchor.source === 'read' ? 'read' : 'liked'; + notes.push({ part: 'sameCreator', value: round2(sameCreatorValue(bestCreator.anchor.weight)), + text: `Also by ${author} — you ${verb} ${bestCreator.name}.`, + evidence: { id: `creator:${bestCreator.author.staffId}:${candidate.id}`, kind: 'creator', + label: `Also by ${author}`, note: `Shared creator with ${bestCreator.name}`, + url: urlFor(null), fetchedAt: bestCreator.fetchedAt || candidate.fetchedAt || null } }); + } + + const top = seeds.tagTop.get(candidate.id); + const averageScore = Number(candidate.averageScore ?? top?.record?.averageScore); + if (top && averageScore >= TAG_TOP_MIN_SCORE) notes.push({ part: 'tagTop', value: top.value, + text: `Top-rated in ${top.tag.name}, one of your most-liked tags.`, + evidence: { id: `tagtop:${top.tag.kind}:${top.tag.name}:${candidate.id}`, kind: 'tag-top', + label: `Top-rated in ${top.tag.name} on AniList`, note: `Average score ${averageScore}`, + url: urlFor(top.record), fetchedAt: top.fetchedAt } }); + + const fresh = seeds.fresh.get(candidate.id); + if (fresh) { + const status = fresh.record?.status ?? candidate.status; + const year = Number(fresh.record?.year ?? candidate.year); + const isNew = status === 'ongoing' && year >= thisYear - FRESH_YEARS; + notes.push({ part: 'fresh', value: isNew ? 2 : 1, + // The exact template claims a start year and "ongoing", which only the value-2 case proves. + text: isNew ? `New: started ${year}, ongoing and trending.` : 'Trending on AniList right now.', + evidence: { id: `fresh:${candidate.id}`, kind: 'fresh', label: 'Trending new manga on AniList', note: '', + url: urlFor(fresh.record), fetchedAt: fresh.fetchedAt } }); + } + return notes; +} + +function largestPart(scoreParts) { + let best = null; + for (const [part, value] of Object.entries(scoreParts)) { + if (!best || value > best.value || (value === best.value && partRank(part) < partRank(best.part))) best = { part, value }; + } + return best; +} +function groupFor(scoreParts, period) { + const top = largestPart(scoreParts); + return top && top.value > 0 && SEED_PART_GROUP[top.part] ? SEED_PART_GROUP[top.part] : period || 'Related works'; +} +const byScore = (a, b) => b.score - a.score || (a.candidateId < b.candidateId ? -1 : a.candidateId > b.candidateId ? 1 : 0); + +// Reserve slots for fresh and outside-the-top-tags rows, then fill greedily; every step respects the group caps. +function mixSlots(rows, { topTags = [], max = MAX_FEED } = {}) { + const sorted = [...rows].sort(byScore); + const picked = new Set(), counts = new Map(); + const capOf = group => SEED_GROUPS.has(group) ? SLOTS.perKind : SLOTS.perPeriod; + const take = row => { + if (picked.has(row) || picked.size >= max || (counts.get(row.group) || 0) >= capOf(row.group)) return false; + picked.add(row); counts.set(row.group, (counts.get(row.group) || 0) + 1); + return true; + }; + const reserve = (limit, wanted) => { + let taken = 0; + for (const row of sorted) if (taken < limit && wanted(row) && take(row)) taken++; + }; + const top = new Set(topTags.map(tag => String(tag.name).toLowerCase())); + reserve(SLOTS.fresh, row => row.scoreParts?.fresh > 0); + reserve(SLOTS.outside, row => !listOf(row.tags).some(tag => top.has(String(tag).toLowerCase()))); + for (const row of sorted) take(row); + return [...picked].sort(byScore).map((row, index) => ({ ...row, rank: index + 1 })); +} + +module.exports = { MAX_FEED, SLOTS, PART_ORDER, round2, communityRecValue, sameCreatorValue, + seedIndexFor, seedNotes, largestPart, groupFor, partRank, mixSlots }; diff --git a/test/discovery.test.js b/test/discovery.test.js index c8a43fe..09b8c07 100644 --- a/test/discovery.test.js +++ b/test/discovery.test.js @@ -4,8 +4,9 @@ const assert = require('node:assert/strict'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); -const { generate } = require('../discovery/engine'); +const { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, MAX_CANDIDATES, SLOTS } = require('../discovery/engine'); const { DiscoveryStore } = require('../discovery/store'); +const { DiscoveryCache } = require('../discovery/cache'); test('discovery is bounded, diversified, opt-in, evidence-linked and user-specific', () => { const { loadCatalog } = require('../greatest'); @@ -16,7 +17,7 @@ test('discovery is bounded, diversified, opt-in, evidence-linked and user-specif const db = { users, greatestRatings: { alice: { [greatest.entries[0].id]: 'flame' }, bob: { [greatest.entries[20].id]: 'flame' } }, discoveryFeedback: { alice: {} }, progress: {} }; const result = generate({ db, library: [], greatest, cacheDir: path.join(os.tmpdir(), 'nonexistent-discovery-cache') }); - assert.equal(Object.keys(result.candidates).length, 50); + assert.equal(Object.keys(result.candidates).length, Math.min(2000, result.examined)); assert.ok(result.feeds.alice.items.length > 0 && result.feeds.alice.items.length <= 20); assert.ok(result.feeds.bob.items.length > 0); assert.equal(result.feeds.off.disabled, true); @@ -85,3 +86,225 @@ test('patch refuses a corrupt store', t => { assert.throws(() => store.patch({ harvest: {} }), /refusing to overwrite/); assert.equal(fs.readFileSync(store.file, 'utf8'), '{broken'); }); + +// ---- Wider pool (BL-0051): AniList seeds, new score parts, slot mix ---- +const NOW = () => new Date('2026-10-25T12:00:00Z'); +const FETCHED_AT = '2026-10-20T00:00:00.000Z'; +const NO_SCRAPER_CACHE = path.join(os.tmpdir(), 'nonexistent-discovery-cache'); +const AUTHOR = { staffId: 7, name: 'Author Seven', role: 'Story' }; +function seedRecord(id, extra = {}) { + return { anilistId: id, title: `Seed ${id}`, altTitles: [], format: 'MANGA', isAdult: false, status: 'completed', + year: 2010, averageScore: 70, popularity: 1000, trending: 0, + tags: [{ name: 'Action', rank: 90, kind: 'genre', isAdult: false }], + authors: [{ staffId: 900 + id, name: `Writer ${id}`, role: 'Story' }], + coverUrl: `https://img.example/${id}.jpg`, siteUrl: `https://anilist.co/manga/${id}`, ...extra }; +} +const seed = (id, extra) => ({ anilistId: id, record: seedRecord(id, extra) }); +function harvestFixture(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-harvest-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const cache = new DiscoveryCache(dir, { now: () => Date.parse(FETCHED_AT) }); + cache.put('title', 100, seedRecord(100, { title: 'Anchor One', authors: [AUTHOR], + tags: [{ name: 'Drama', rank: 100, kind: 'genre', isAdult: false }, { name: 'Time Skip', rank: 80, kind: 'tag', isAdult: false }] })); + cache.put('recs', 100, [ + { ...seed(201), rating: 42 }, + { ...seed(209, { authors: [AUTHOR] }), rating: 10 }, + { ...seed(207, { isAdult: true }), rating: 5 }, + { ...seed(208, { tags: [{ name: 'Nudity', rank: 65, kind: 'tag', isAdult: true }] }), rating: 5 }, + { ...seed(212), rating: 3 }, + ]); + cache.put('creator', 7, [{ ...seed(202, { authors: [AUTHOR] }), role: 'Story' }, { ...seed(209, { authors: [AUTHOR] }), role: 'Story' }]); + cache.put('tagTop', 'genre:Drama', [seed(203, { averageScore: 80, tags: [{ name: 'Drama', rank: 90, kind: 'genre', isAdult: false }] })]); + cache.put('tagTop', 'tag:Time Skip', [seed(204, { averageScore: 70 })]); + cache.put('fresh', 'global', [seed(205, { status: 'ongoing', year: 2025 }), seed(206, { status: 'ongoing', year: 2020 }), + seed(210, { title: 'Owned Title', year: 2020 }), seed(211, { title: 'Queued Title' }), seed(213, { status: 'ongoing', year: 2026 })]); + return cache; +} +const fixtureGreatest = { entries: [ + { id: 'g1', title: 'Anchor One', period: '2000s', rank: 1, note: '', metadata: { id: 100 } }, + { id: 'g2', title: 'Voted Two', period: '1990s', rank: 2, note: '', metadata: { id: 212 } }, +] }; +function fixtureWorld(t, settings = {}) { + const user = { name: 'alice', settings: { discoveryEnabled: true, ...settings } }; + const db = { users: { alice: user }, greatestRatings: { alice: { g1: 'flame', g2: 'good' } }, + discoveryFeedback: { alice: { 'anilist:213': 'dismissed' } }, progress: {} }; + const library = [{ id: 'm1', title: 'Owned Title', meta: { year: 2020, 'x-anilist': { id: 210 } } }]; + return { user, db, library, greatest: fixtureGreatest, harvestCache: harvestFixture(t), queuedTitles: ['Queued Title'] }; +} +function fixtureFeed(t, settings) { + const world = fixtureWorld(t, settings); + const candidates = buildCandidates({ greatest: world.greatest, library: world.library, cacheDir: NO_SCRAPER_CACHE, harvestCache: world.harvestCache }); + const feed = feedForUser({ ...world, candidates, now: NOW }); + return { world, candidates, feed, item: id => feed.items.find(item => item.candidateId === `anilist:${id}`) }; +} + +test('engine: empty harvest cache equals Phase A pool', t => { + const { loadCatalog } = require('../greatest'); + const greatest = loadCatalog(); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-discovery-empty-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const db = { users: {}, greatestRatings: {}, discoveryFeedback: {}, progress: {} }; + const without = generate({ db, library: [], greatest, cacheDir: NO_SCRAPER_CACHE }); + const withEmpty = generate({ db, library: [], greatest, cacheDir: NO_SCRAPER_CACHE, harvestCache: new DiscoveryCache(dir) }); + assert.equal(MAX_CANDIDATES, 2000); + assert.deepEqual(Object.keys(withEmpty.candidates).sort(), Object.keys(without.candidates).sort()); +}); + +test('harvested records join the pool and fill Greatest candidates', t => { + const { candidates } = fixtureFeed(t); + const byId = new Map(candidates.map(c => [c.id, c])); + for (const id of [201, 202, 203, 204, 205, 206, 207, 208, 209]) assert.ok(byId.has(`anilist:${id}`), `anilist:${id}`); + const fromRecs = byId.get('anilist:201'); + assert.equal(fromRecs.isAdult, false); + assert.equal(fromRecs.averageScore, 70); + assert.deepEqual(fromRecs.tags, ['Action']); + assert.deepEqual(fromRecs.authors, ['Writer 201']); + assert.equal(fromRecs.tagRecords[0].rank, 90); + assert.equal(fromRecs.authorRecords[0].staffId, 1101); + assert.equal(fromRecs.fetchedAt, FETCHED_AT); + const greatestOne = byId.get('anilist:212'); + assert.equal(greatestOne.greatestId, 'g2'); + assert.equal(greatestOne.title, 'Voted Two'); + assert.equal(greatestOne.isAdult, false); + assert.equal(greatestOne.tagRecords.length, 1); +}); + +test('community rec, creator, tagTop and fresh parts and reasons', t => { + const { item } = fixtureFeed(t); + const rec = item(201); + assert.equal(rec.scoreParts.communityRec, 3.71); + assert.equal(rec.group, 'recs'); + assert.equal(rec.reason, 'AniList readers who liked Anchor One recommend this (42).'); + const recEvidence = rec.evidence.find(e => e.id === 'rec:100:anilist:201'); + assert.equal(recEvidence.kind, 'recommendation'); + assert.equal(recEvidence.fetchedAt, FETCHED_AT); + assert.equal(recEvidence.url, 'https://anilist.co/manga/201'); + assert.ok(rec.evidenceIds.includes('rec:100:anilist:201')); + + const creator = item(202); + assert.equal(creator.scoreParts.sameCreator, 4); + assert.equal(creator.group, 'creator'); + assert.equal(creator.reason, 'Also by Author Seven — you liked Anchor One.'); + const creatorEvidence = creator.evidence.find(e => e.id === 'creator:7:anilist:202'); + assert.equal(creatorEvidence.kind, 'creator'); + assert.equal(creatorEvidence.fetchedAt, FETCHED_AT); + + const top = item(203); + assert.equal(top.scoreParts.tagTop, 3); + assert.equal(top.group, 'tagTop'); + assert.equal(top.reason, 'Top-rated in Drama, one of your most-liked tags.'); + assert.equal(top.evidence.find(e => e.id === 'tagtop:genre:Drama:anilist:203').kind, 'tag-top'); + assert.equal(item(204).scoreParts.tagTop, 0, 'averageScore below 75 earns no tagTop part'); + + const fresh = item(205); + assert.equal(fresh.scoreParts.fresh, 2); + assert.equal(fresh.group, 'fresh'); + assert.equal(fresh.reason, 'New: started 2025, ongoing and trending.'); + const freshEvidence = fresh.evidence.find(e => e.id === 'fresh:anilist:205'); + assert.equal(freshEvidence.kind, 'fresh'); + assert.equal(freshEvidence.fetchedAt, FETCHED_AT); + assert.equal(item(206).scoreParts.fresh, 1); + for (const row of [rec, creator, top, fresh]) assert.ok(row.reason.length <= 280); +}); + +test('isAdult filter', t => { + const plain = fixtureFeed(t); + assert.equal(plain.item(207), undefined); + assert.equal(plain.item(208), undefined, 'an adult tag with rank >= 60 counts as mature'); + const mature = fixtureFeed(t, { discoveryMature: true }); + assert.ok(mature.item(207)); + assert.ok(mature.item(208)); +}); + +test('multiple seeds merge reasons', t => { + const { item } = fixtureFeed(t); + const both = item(209); + assert.equal(both.scoreParts.sameCreator, 4); + assert.equal(both.scoreParts.communityRec, 2.73); + assert.equal(both.score, 6.73); + assert.equal(both.group, 'creator'); + assert.deepEqual(both.reasons, ['Also by Author Seven — you liked Anchor One.', + 'AniList readers who liked Anchor One recommend this (10).']); + assert.equal(both.reason, both.reasons[0]); + assert.ok(both.evidenceIds.includes('rec:100:anilist:209')); + assert.ok(both.evidenceIds.includes('creator:7:anilist:209')); +}); + +test('owned, queued, voted, dismissed still excluded', t => { + const { item, candidates } = fixtureFeed(t); + for (const id of [210, 211, 212, 213]) { + assert.ok(candidates.some(c => c.id === `anilist:${id}`), `anilist:${id} is in the pool`); + assert.equal(item(id), undefined, `anilist:${id} must not be suggested`); + } +}); + +test('slot mix', () => { + const row = (id, score, group, extra = {}) => ({ candidateId: `anilist:${id}`, score, group, tags: ['Drama'], + scoreParts: { fresh: 0 }, ...extra }); + const rows = []; + for (let i = 0; i < 30; i++) rows.push(row(1000 + i, 30 - i, 'recs')); + rows.push(row(2001, 0.5, 'fresh', { scoreParts: { fresh: 1 } }), row(2002, 0.4, 'fresh', { scoreParts: { fresh: 1 } })); + rows.push(row(3001, 0.2, '1990s', { tags: ['Sports'] }), row(3002, 0.1, '1990s', { tags: [] })); + const picked = mixSlots(rows, { topTags: [{ name: 'Drama', kind: 'genre' }] }); + assert.equal(SLOTS.perKind, 8); + assert.equal(picked.filter(r => r.group === 'recs').length, 8); + assert.ok(picked.some(r => r.candidateId === 'anilist:2001') && picked.some(r => r.candidateId === 'anilist:2002')); + assert.ok(picked.some(r => r.candidateId === 'anilist:3001') && picked.some(r => r.candidateId === 'anilist:3002')); + assert.equal(picked.length, 12, 'never pads up to 20'); + assert.deepEqual(picked.map(r => r.rank), picked.map((_, i) => i + 1)); + for (let i = 1; i < picked.length; i++) assert.ok(picked[i - 1].score >= picked[i].score); + + const many = []; + for (let i = 0; i < 40; i++) many.push(row(4000 + i, 100 - i, ['recs', 'creator', 'tagTop', 'fresh', '1980s'][i % 5], + { scoreParts: { fresh: i % 5 === 3 ? 1 : 0 } })); + const full = mixSlots(many, { topTags: [] }); + assert.equal(full.length, 20); + assert.ok(full.filter(r => r.group === '1980s').length <= SLOTS.perPeriod); +}); + +test('mergeUserFeed keeps other users\' referenced candidates', () => { + const prior = { candidates: { a: { id: 'a' }, b: { id: 'b' }, c: { id: 'c' } }, + feeds: { alice: { items: [{ candidateId: 'a' }] }, bob: { items: [{ candidateId: 'b' }] }, off: { items: [], disabled: true } } }; + const result = { candidates: { d: { id: 'd' }, b: { id: 'b', fresh: true } }, feeds: { alice: { items: [{ candidateId: 'd' }] } } }; + const merged = mergeUserFeed(prior, result, 'alice'); + assert.deepEqual(Object.keys(merged.feeds).sort(), ['alice', 'bob', 'off']); + assert.deepEqual(merged.feeds.alice.items, [{ candidateId: 'd' }]); + assert.deepEqual(merged.feeds.bob, prior.feeds.bob); + assert.deepEqual(Object.keys(merged.candidates).sort(), ['b', 'd']); + assert.equal(merged.candidates.b.fresh, true); +}); + +test('generateAsync matches generate and can score one user', async t => { + const world = fixtureWorld(t); + world.db.users.bob = { name: 'bob', settings: { discoveryEnabled: true } }; + const args = { db: world.db, library: world.library, greatest: world.greatest, cacheDir: NO_SCRAPER_CACHE, + harvestCache: world.harvestCache, queuedTitles: world.queuedTitles, now: NOW }; + const sync = generate(args); + const all = await generateAsync(args); + assert.deepEqual(all.feeds.alice.items, sync.feeds.alice.items); + assert.deepEqual(Object.keys(all.candidates), Object.keys(sync.candidates)); + const one = await generateAsync({ ...args, onlyUser: 'alice' }); + assert.deepEqual(Object.keys(one.feeds), ['alice']); + assert.deepEqual(one.feeds.alice.items, sync.feeds.alice.items); +}); + +test('scoring one user stays under 200 ms on 2000 candidates', t => { + const world = fixtureWorld(t); + const tagNames = ['Drama', 'Action', 'Romance', 'Time Skip', 'Comedy', 'Horror']; + const candidates = []; + for (let i = 0; i < 2000; i++) { + const id = 10000 + i; + candidates.push({ id: `anilist:${id}`, title: `Synthetic ${id}`, year: 2000 + (i % 27), type: 'MANGA', + status: i % 3 ? 'completed' : 'ongoing', cover: null, metadataUrl: `https://anilist.co/manga/${id}`, + tags: [tagNames[i % 6], tagNames[(i + 1) % 6]], authors: [`Writer ${i % 50}`], period: null, editorialRank: null, + greatestId: null, evidence: [], relations: [], isAdult: i % 97 === 0, averageScore: 60 + (i % 40), + tagRecords: [{ name: tagNames[i % 6], rank: 80, kind: 'tag', isAdult: false }], + authorRecords: [{ staffId: i % 50, name: `Writer ${i % 50}`, role: 'Story' }], fetchedAt: FETCHED_AT, seed: 'recs' }); + } + feedForUser({ ...world, candidates, now: NOW }); + const started = performance.now(); + const feed = feedForUser({ ...world, candidates, now: NOW }); + const elapsed = performance.now() - started; + assert.ok(feed.items.length > 0); + assert.ok(elapsed < 200, `took ${elapsed.toFixed(1)} ms`); +});