'use strict'; const fs = require('node:fs'); const path = require('node:path'); const { normalize } = require('../greatest'); const { anchorsFor } = require('./anchors'); const { MAX_FEED, SLOTS, round2, seedIndexFor, seedNotes, groupFor, partRank, mixSlots } = require('./scoring'); const MAX_CANDIDATES = 2000; const SEED_KINDS = ['recs', 'creator', 'tagTop', 'fresh']; const MATURE_TAG_RANK = 60; const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']); const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null; const unique = values => [...new Set(values.filter(Boolean))]; // manga.json and cached metadata are hand-editable files: a field that is not a list counts as empty. const listOf = value => Array.isArray(value) ? value : []; const stringsOf = value => unique(listOf(value).filter(item => typeof item === 'string')); function cachedMetadata(dir, id) { if (!Number.isSafeInteger(Number(id)) || Number(id) < 1) return null; try { const value = JSON.parse(fs.readFileSync(path.join(dir, `id-${Number(id)}.json`), 'utf8')); return value._cacheVersion === 4 && value.metadata?.['x-anilist']?.id === Number(id) ? value.metadata : null; } catch { return null; } } function candidateFromGreatest(entry, cacheDir) { const id = Number(entry.metadata?.id); const cached = id ? cachedMetadata(cacheDir, id) : null; return { id: id ? `anilist:${id}` : `greatest:${entry.id}`, title: cached?.title || entry.title, year: cached?.year || entry.year || null, type: cached?.['x-anilist']?.format || 'MANGA', status: cached?.status || entry.status || 'unknown', cover: safeUrl(cached?.['x-anilist']?.coverUrl) || safeUrl(entry.cover), metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(entry.metadata?.url) || null, tags: stringsOf(cached?.tags), authors: stringsOf(cached?.authors), period: entry.period, editorialRank: entry.rank, greatestId: entry.id, evidence: [{ id: `greatest:${entry.id}`, kind: 'curation', label: 'Greatest of All Time selection', note: String(entry.note || '').slice(0, 600), url: safeUrl(entry.source?.url) || safeUrl(entry.metadata?.url) }], relations: cached?.['x-anilist']?.relations || [], }; } function addRelations(map, anchor, cacheDir) { for (const relation of listOf(anchor.relations)) { if (!relation || typeof relation !== 'object') continue; if (!VALID_FORMATS.has(relation.format) || !Number.isSafeInteger(Number(relation.id))) continue; const id = `anilist:${Number(relation.id)}`; if (map.has(id)) continue; const cached = cachedMetadata(cacheDir, relation.id); map.set(id, { id, title: cached?.title || relation.title || `AniList ${relation.id}`, year: cached?.year || null, type: relation.format, status: cached?.status || 'unknown', cover: safeUrl(cached?.['x-anilist']?.coverUrl), metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}`, tags: stringsOf(cached?.tags), authors: stringsOf(cached?.authors), period: null, editorialRank: null, greatestId: null, evidence: [{ id: `relation:${anchor.id}:${id}`, kind: 'relation', label: `${relation.type || 'RELATED'} to ${anchor.title}`, note: '', url: safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}` }], relatedTo: { id: anchor.id, title: anchor.title, type: String(relation.type || 'RELATED') }, relations: cached?.['x-anilist']?.relations || [], }); } } const textList = values => unique(Array.isArray(values) ? values.map(value => value?.name).filter(name => typeof name === 'string') : []); const recordList = values => Array.isArray(values) ? values.filter(value => value && typeof value === 'object') : []; function candidateFromRecord(record, id, seed, fetchedAt) { return { id: `anilist:${id}`, title: String(record.title || `AniList ${id}`), year: Number(record.year) || null, type: record.format || 'MANGA', status: record.status || 'unknown', cover: safeUrl(record.coverUrl), metadataUrl: safeUrl(record.siteUrl) || `https://anilist.co/manga/${id}`, tags: textList(record.tags), authors: textList(record.authors), period: null, editorialRank: null, greatestId: null, evidence: [], relations: [], isAdult: record.isAdult === true, averageScore: Number.isFinite(record.averageScore) ? record.averageScore : null, tagRecords: recordList(record.tags), authorRecords: recordList(record.authors), fetchedAt, seed, }; } // Greatest and relation candidates keep their own fields; AniList only fills what they lack. function fillFromRecord(candidate, record, fetchedAt) { if (candidate.isAdult === undefined) candidate.isAdult = record.isAdult === true; if (!candidate.tagRecords) candidate.tagRecords = recordList(record.tags); if (!candidate.authorRecords) candidate.authorRecords = recordList(record.authors); if (candidate.averageScore == null && Number.isFinite(record.averageScore)) candidate.averageScore = record.averageScore; if (!candidate.fetchedAt) candidate.fetchedAt = fetchedAt; } function addHarvested(map, harvestCache) { for (const kind of SEED_KINDS) { const entries = harvestCache.list(kind).sort((a, b) => String(a.key).localeCompare(String(b.key))); for (const entry of entries) for (const item of Array.isArray(entry.data) ? entry.data : []) { const record = item?.record; const id = Number(item?.anilistId); if (!record || typeof record !== 'object' || !Number.isSafeInteger(id) || id < 1) continue; if (record.format && !VALID_FORMATS.has(record.format)) continue; const existing = map.get(`anilist:${id}`); if (existing) fillFromRecord(existing, record, entry.fetchedAt); else map.set(`anilist:${id}`, candidateFromRecord(record, id, kind, entry.fetchedAt)); } } } function buildCandidates({ greatest, library, cacheDir, harvestCache = null }) { const map = new Map(); for (const entry of greatest.entries) { const candidate = candidateFromGreatest(entry, cacheDir); if (candidate.type === 'MANGA' || candidate.type === 'ONE_SHOT') map.set(candidate.id, candidate); } for (const manga of library) { const identity = manga.meta?.['x-anilist']; if (!identity?.id) continue; addRelations(map, { id: `anilist:${identity.id}`, title: manga.title, relations: identity.relations }, cacheDir); } for (const candidate of [...map.values()]) if (candidate.greatestId) addRelations(map, candidate, cacheDir); if (harvestCache) addHarvested(map, harvestCache); return [...map.values()]; } // Title-level isAdult or a strong adult tag; the Phase A regex only where AniList gave no isAdult. function isMature(candidate) { if (candidate.isAdult === undefined) return candidate.tags.some(tag => /adult|hentai|erotica/i.test(tag)); return candidate.isAdult === true || (candidate.tagRecords || []).some(tag => tag?.isAdult && Number(tag.rank) >= MATURE_TAG_RANK); } function ownedKeySet(library) { const ids = new Set(), names = new Set(); for (const manga of library) { const id = Number(manga.meta?.['x-anilist']?.id); if (id) ids.add(`anilist:${id}`); names.add(`${normalize(manga.title)}:${Number(manga.meta?.year) || '?'}`); } return { ids, names }; } function labelForVote(greatest, id) { return greatest.entries.find(entry => entry.id === id)?.title || null; } function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [], harvestCache = null, now = () => new Date() }) { const settings = user.settings || {}; const today = now(); if (!settings.discoveryEnabled) return { generatedAt: today.toISOString(), items: [], disabled: true }; const votes = db.greatestRatings?.[user.name] || {}; const feedback = db.discoveryFeedback?.[user.name] || {}; const owned = ownedKeySet(library); const queued = new Set(queuedTitles.map(normalize)); const positive = Object.entries(votes).filter(([, vote]) => vote === 'flame' || vote === 'good') .map(([id, vote]) => ({ id, vote, title: labelForVote(greatest, id), entry: greatest.entries.find(e => e.id === id) })).filter(x => x.entry); const likedTags = new Map(), likedPeriods = new Map(); for (const anchor of positive) { likedPeriods.set(anchor.entry.period, anchor); const candidate = candidates.find(c => c.greatestId === anchor.id); for (const tag of candidate?.tags || []) likedTags.set(tag.toLowerCase(), anchor); } const ownedTags = new Map(); for (const manga of library) for (const tag of stringsOf(manga.meta?.tags)) ownedTags.set(tag.toLowerCase(), manga.title); const readTags = new Map(); if (settings.discoveryReadingActivity) for (const manga of library) { const progress = db.progress?.[user.name]?.[manga.id] || {}; if (!Object.values(progress).some(chapter => chapter?.done)) continue; for (const tag of stringsOf(manga.meta?.tags)) readTags.set(tag.toLowerCase(), manga.title); } const seeds = seedIndexFor({ anchors: anchorsFor({ user, db, library, greatest }), harvestCache, mature: !!settings.discoveryMature }); const rows = []; for (const candidate of candidates) { if (candidate.greatestId && votes[candidate.greatestId]) continue; if (['dismissed', 'hidden'].includes(feedback[candidate.id])) continue; if (owned.ids.has(candidate.id) || owned.names.has(`${normalize(candidate.title)}:${candidate.year || '?'}`)) continue; if (queued.has(normalize(candidate.title))) continue; if (!settings.discoveryMature && isMature(candidate)) continue; const matchingLikedTag = candidate.tags.find(tag => likedTags.has(tag.toLowerCase())); const matchingOwnedTag = candidate.tags.find(tag => ownedTags.has(tag.toLowerCase())); const matchingReadTag = candidate.tags.find(tag => readTags.has(tag.toLowerCase())); const era = likedPeriods.get(candidate.period); const relation = candidate.relatedTo; const scoreParts = { editorial: candidate.greatestId ? Math.max(0, 6 - Math.min(6, Number(candidate.editorialRank) || 6)) : 0, relation: relation ? 3 : 0, sharedTaste: matchingLikedTag ? 4 : 0, sharedLibrary: matchingOwnedTag ? 1 : 0, era: era ? 2 : 0, reading: matchingReadTag ? 2 : 0, interest: feedback[candidate.id] === 'interested' ? 5 : feedback[candidate.id] === 'maybe' ? 2 : 0, communityRec: 0, sameCreator: 0, tagTop: 0, fresh: 0 }; // Every applicable part brings its reason and, for taste parts, its personal evidence. const notes = []; if (scoreParts.editorial && candidate.evidence[0]?.note) notes.push({ part: 'editorial', text: candidate.evidence[0].note }); if (relation) notes.push({ part: 'relation', text: `${candidate.title} is listed as ${relation.type.toLowerCase().replaceAll('_', ' ')} to ${relation.title}. Check the edition and reading order first.` }); if (matchingLikedTag) { const anchor = likedTags.get(matchingLikedTag.toLowerCase()); notes.push({ part: 'sharedTaste', text: `You liked ${anchor.title}; this also carries the ${matchingLikedTag} tag.`, evidence: { id: `taste:greatest:${anchor.id}`, kind: 'personal', label: `Your Greatest vote for ${anchor.title}`, note: `Shared tag: ${matchingLikedTag}`, url: null } }); } if (era) notes.push({ part: 'era', text: `You liked ${era.title}; this is another curated work from ${candidate.period}.`, evidence: { id: `taste:greatest:${era.id}`, kind: 'personal', label: `Your Greatest vote for ${era.title}`, note: `Shared period: ${candidate.period}`, url: null } }); if (matchingReadTag) { const title = readTags.get(matchingReadTag.toLowerCase()); notes.push({ part: 'reading', text: `You have read ${title}; this also carries the ${matchingReadTag} tag.`, evidence: { id: `taste:read:${normalize(title)}`, kind: 'personal', label: `Your completed reading in ${title}`, note: `Shared tag: ${matchingReadTag}`, url: null } }); } if (matchingOwnedTag) { const title = ownedTags.get(matchingOwnedTag.toLowerCase()); notes.push({ part: 'sharedLibrary', text: `This shares the ${matchingOwnedTag} tag with ${title} in your library.`, evidence: { id: `taste:library:${normalize(title)}`, kind: 'personal', label: `${title} in your library`, note: `Shared tag: ${matchingOwnedTag}`, url: null } }); } for (const note of seedNotes(candidate, seeds, today.getFullYear())) { scoreParts[note.part] = note.value; notes.push(note); } notes.sort((a, b) => scoreParts[b.part] - scoreParts[a.part] || partRank(a.part) - partRank(b.part)); const reasons = notes.map(note => String(note.text).slice(0, 280)); const fallback = candidate.evidence[0]?.note || (candidate.greatestId || relation ? 'A curated manga worth a closer look.' : 'A manga from AniList worth a closer look.'); const evidence = [...candidate.evidence]; for (const note of notes) if (note.evidence && !evidence.some(e => e.id === note.evidence.id)) evidence.push(note.evidence); rows.push({ candidateId: candidate.id, reason: (reasons[0] || fallback).slice(0, 280), reasons, evidence, evidenceIds: evidence.map(e => e.id), scoreParts, score: round2(Object.values(scoreParts).reduce((a, b) => a + b, 0)), period: candidate.period || 'Related works', group: groupFor(scoreParts, candidate.period), tags: candidate.tags }); } // Tags only steer the slot mix; the published row does not repeat them. const items = mixSlots(rows, { topTags: seeds.topTags }).map(({ tags, ...row }) => row); return { generatedAt: today.toISOString(), items }; } // Sample across eras (and seed kinds) before the work cap; insertion order in the curated // catalog must not make recent manga invisible. function candidatePool({ greatest, library, cacheDir, harvestCache }) { const all = buildCandidates({ greatest, library, cacheDir, harvestCache }); const buckets = new Map(); for (const candidate of all) { const key = candidate.period || candidate.seed || 'Related works'; if (!buckets.has(key)) buckets.set(key, []); buckets.get(key).push(candidate); } const candidates = []; while (candidates.length < MAX_CANDIDATES && [...buckets.values()].some(values => values.length)) { for (const values of buckets.values()) { if (values.length && candidates.length < MAX_CANDIDATES) candidates.push(values.shift()); } } return { all, candidates }; } // Only candidates a feed shows are published; the rest of the pool (up to 2000, with AniList // records) would bloat discovery.json for nothing. examined/clipped still describe the whole pool. function runResult(all, candidates, feeds) { const shown = new Set(); for (const feed of Object.values(feeds)) for (const item of feed?.items || []) shown.add(item.candidateId); return { candidates: Object.fromEntries(candidates.filter(c => shown.has(c.id)).map(c => [c.id, c])), feeds, examined: all.length, clipped: Math.max(0, all.length - candidates.length) }; } function generate({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], now }) { const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache }); const feeds = {}; for (const user of Object.values(db.users || {})) feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now }); return runResult(all, candidates, feeds); } // Same result as generate(), but yields to the event loop between users so the reader stays responsive. async function generateAsync({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], onlyUser, now }) { const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache }); const feeds = {}; for (const user of Object.values(db.users || {})) { if (onlyUser !== undefined && user.name !== onlyUser) continue; await new Promise(setImmediate); feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now }); } return runResult(all, candidates, feeds); } // A per-user rescore replaces one feed; the others and the candidates they show are carried over. function mergeUserFeed(prior, result, userName) { const feeds = { ...(prior?.feeds || {}) }; if (result.feeds?.[userName]) feeds[userName] = result.feeds[userName]; else delete feeds[userName]; const candidates = {}; for (const [name, feed] of Object.entries(feeds)) { if (name === userName) continue; for (const item of feed?.items || []) { const candidate = prior?.candidates?.[item.candidateId]; if (candidate) candidates[item.candidateId] = candidate; } } Object.assign(candidates, result.candidates || {}); return { candidates, feeds }; } module.exports = { generate, generateAsync, buildCandidates, candidatePool, feedForUser, mixSlots, mergeUserFeed, cachedMetadata, isMature, MAX_CANDIDATES, MAX_FEED, SLOTS };