258 lines
16 KiB
JavaScript
258 lines
16 KiB
JavaScript
'use strict';
|
|
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
const { normalize } = require('../greatest');
|
|
const { anchorsFor } = require('./anchors');
|
|
const { MAX_FEED, SLOTS, round2, seedIndexFor, seedNotes, groupFor, partRank, mixSlots } = require('./scoring');
|
|
|
|
const MAX_CANDIDATES = 2000;
|
|
const SEED_KINDS = ['recs', 'creator', 'tagTop', 'fresh'];
|
|
const MATURE_TAG_RANK = 60;
|
|
const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']);
|
|
const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null;
|
|
const unique = values => [...new Set(values.filter(Boolean))];
|
|
|
|
function cachedMetadata(dir, id) {
|
|
if (!Number.isSafeInteger(Number(id)) || Number(id) < 1) return null;
|
|
try {
|
|
const value = JSON.parse(fs.readFileSync(path.join(dir, `id-${Number(id)}.json`), 'utf8'));
|
|
return value._cacheVersion === 4 && value.metadata?.['x-anilist']?.id === Number(id) ? value.metadata : null;
|
|
} catch { return null; }
|
|
}
|
|
function candidateFromGreatest(entry, cacheDir) {
|
|
const id = Number(entry.metadata?.id);
|
|
const cached = id ? cachedMetadata(cacheDir, id) : null;
|
|
return {
|
|
id: id ? `anilist:${id}` : `greatest:${entry.id}`,
|
|
title: cached?.title || entry.title, year: cached?.year || entry.year || null,
|
|
type: cached?.['x-anilist']?.format || 'MANGA', status: cached?.status || entry.status || 'unknown',
|
|
cover: safeUrl(cached?.['x-anilist']?.coverUrl) || safeUrl(entry.cover),
|
|
metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(entry.metadata?.url) || null,
|
|
tags: unique(cached?.tags || []), authors: unique(cached?.authors || []),
|
|
period: entry.period, editorialRank: entry.rank, greatestId: entry.id,
|
|
evidence: [{ id: `greatest:${entry.id}`, kind: 'curation', label: 'Greatest of All Time selection',
|
|
note: String(entry.note || '').slice(0, 600), url: safeUrl(entry.source?.url) || safeUrl(entry.metadata?.url) }],
|
|
relations: cached?.['x-anilist']?.relations || [],
|
|
};
|
|
}
|
|
function addRelations(map, anchor, cacheDir) {
|
|
for (const relation of anchor.relations || []) {
|
|
if (!VALID_FORMATS.has(relation.format) || !Number.isSafeInteger(Number(relation.id))) continue;
|
|
const id = `anilist:${Number(relation.id)}`;
|
|
if (map.has(id)) continue;
|
|
const cached = cachedMetadata(cacheDir, relation.id);
|
|
map.set(id, {
|
|
id, title: cached?.title || relation.title || `AniList ${relation.id}`,
|
|
year: cached?.year || null, type: relation.format, status: cached?.status || 'unknown',
|
|
cover: safeUrl(cached?.['x-anilist']?.coverUrl),
|
|
metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}`,
|
|
tags: unique(cached?.tags || []), authors: unique(cached?.authors || []), period: null,
|
|
editorialRank: null, greatestId: null,
|
|
evidence: [{ id: `relation:${anchor.id}:${id}`, kind: 'relation', label: `${relation.type || 'RELATED'} to ${anchor.title}`,
|
|
note: '', url: safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}` }],
|
|
relatedTo: { id: anchor.id, title: anchor.title, type: relation.type || 'RELATED' },
|
|
relations: cached?.['x-anilist']?.relations || [],
|
|
});
|
|
}
|
|
}
|
|
const textList = values => unique(Array.isArray(values) ? values.map(value => value?.name).filter(name => typeof name === 'string') : []);
|
|
const recordList = values => Array.isArray(values) ? values.filter(value => value && typeof value === 'object') : [];
|
|
function candidateFromRecord(record, id, seed, fetchedAt) {
|
|
return {
|
|
id: `anilist:${id}`, title: String(record.title || `AniList ${id}`), year: Number(record.year) || null,
|
|
type: record.format || 'MANGA', status: record.status || 'unknown', cover: safeUrl(record.coverUrl),
|
|
metadataUrl: safeUrl(record.siteUrl) || `https://anilist.co/manga/${id}`,
|
|
tags: textList(record.tags), authors: textList(record.authors), period: null, editorialRank: null, greatestId: null,
|
|
evidence: [], relations: [], isAdult: record.isAdult === true,
|
|
averageScore: Number.isFinite(record.averageScore) ? record.averageScore : null,
|
|
tagRecords: recordList(record.tags), authorRecords: recordList(record.authors), fetchedAt, seed,
|
|
};
|
|
}
|
|
// Greatest and relation candidates keep their own fields; AniList only fills what they lack.
|
|
function fillFromRecord(candidate, record, fetchedAt) {
|
|
if (candidate.isAdult === undefined) candidate.isAdult = record.isAdult === true;
|
|
if (!candidate.tagRecords) candidate.tagRecords = recordList(record.tags);
|
|
if (!candidate.authorRecords) candidate.authorRecords = recordList(record.authors);
|
|
if (candidate.averageScore == null && Number.isFinite(record.averageScore)) candidate.averageScore = record.averageScore;
|
|
if (!candidate.fetchedAt) candidate.fetchedAt = fetchedAt;
|
|
}
|
|
function addHarvested(map, harvestCache) {
|
|
for (const kind of SEED_KINDS) {
|
|
const entries = harvestCache.list(kind).sort((a, b) => String(a.key).localeCompare(String(b.key)));
|
|
for (const entry of entries) for (const item of Array.isArray(entry.data) ? entry.data : []) {
|
|
const record = item?.record;
|
|
const id = Number(item?.anilistId);
|
|
if (!record || typeof record !== 'object' || !Number.isSafeInteger(id) || id < 1) continue;
|
|
if (record.format && !VALID_FORMATS.has(record.format)) continue;
|
|
const existing = map.get(`anilist:${id}`);
|
|
if (existing) fillFromRecord(existing, record, entry.fetchedAt);
|
|
else map.set(`anilist:${id}`, candidateFromRecord(record, id, kind, entry.fetchedAt));
|
|
}
|
|
}
|
|
}
|
|
function buildCandidates({ greatest, library, cacheDir, harvestCache = null }) {
|
|
const map = new Map();
|
|
for (const entry of greatest.entries) {
|
|
const candidate = candidateFromGreatest(entry, cacheDir);
|
|
if (candidate.type === 'MANGA' || candidate.type === 'ONE_SHOT') map.set(candidate.id, candidate);
|
|
}
|
|
for (const manga of library) {
|
|
const identity = manga.meta?.['x-anilist'];
|
|
if (!identity?.id) continue;
|
|
addRelations(map, { id: `anilist:${identity.id}`, title: manga.title, relations: identity.relations }, cacheDir);
|
|
}
|
|
for (const candidate of [...map.values()]) if (candidate.greatestId) addRelations(map, candidate, cacheDir);
|
|
if (harvestCache) addHarvested(map, harvestCache);
|
|
return [...map.values()];
|
|
}
|
|
// Title-level isAdult or a strong adult tag; the Phase A regex only where AniList gave no isAdult.
|
|
function isMature(candidate) {
|
|
if (candidate.isAdult === undefined) return candidate.tags.some(tag => /adult|hentai|erotica/i.test(tag));
|
|
return candidate.isAdult === true
|
|
|| (candidate.tagRecords || []).some(tag => tag?.isAdult && Number(tag.rank) >= MATURE_TAG_RANK);
|
|
}
|
|
function ownedKeySet(library) {
|
|
const ids = new Set(), names = new Set();
|
|
for (const manga of library) {
|
|
const id = Number(manga.meta?.['x-anilist']?.id);
|
|
if (id) ids.add(`anilist:${id}`);
|
|
names.add(`${normalize(manga.title)}:${Number(manga.meta?.year) || '?'}`);
|
|
}
|
|
return { ids, names };
|
|
}
|
|
function labelForVote(greatest, id) { return greatest.entries.find(entry => entry.id === id)?.title || null; }
|
|
function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [], harvestCache = null, now = () => new Date() }) {
|
|
const settings = user.settings || {};
|
|
const today = now();
|
|
if (!settings.discoveryEnabled) return { generatedAt: today.toISOString(), items: [], disabled: true };
|
|
const votes = db.greatestRatings?.[user.name] || {};
|
|
const feedback = db.discoveryFeedback?.[user.name] || {};
|
|
const owned = ownedKeySet(library);
|
|
const queued = new Set(queuedTitles.map(normalize));
|
|
const positive = Object.entries(votes).filter(([, vote]) => vote === 'flame' || vote === 'good')
|
|
.map(([id, vote]) => ({ id, vote, title: labelForVote(greatest, id), entry: greatest.entries.find(e => e.id === id) })).filter(x => x.entry);
|
|
const likedTags = new Map(), likedPeriods = new Map();
|
|
for (const anchor of positive) {
|
|
likedPeriods.set(anchor.entry.period, anchor);
|
|
const candidate = candidates.find(c => c.greatestId === anchor.id);
|
|
for (const tag of candidate?.tags || []) likedTags.set(tag.toLowerCase(), anchor);
|
|
}
|
|
const ownedTags = new Map();
|
|
for (const manga of library) for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') ownedTags.set(tag.toLowerCase(), manga.title);
|
|
const readTags = new Map();
|
|
if (settings.discoveryReadingActivity) for (const manga of library) {
|
|
const progress = db.progress?.[user.name]?.[manga.id] || {};
|
|
if (!Object.values(progress).some(chapter => chapter?.done)) continue;
|
|
for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') readTags.set(tag.toLowerCase(), manga.title);
|
|
}
|
|
const seeds = seedIndexFor({ anchors: anchorsFor({ user, db, library, greatest }), harvestCache, mature: !!settings.discoveryMature });
|
|
const rows = [];
|
|
for (const candidate of candidates) {
|
|
if (candidate.greatestId && votes[candidate.greatestId]) continue;
|
|
if (['dismissed', 'hidden'].includes(feedback[candidate.id])) continue;
|
|
if (owned.ids.has(candidate.id) || owned.names.has(`${normalize(candidate.title)}:${candidate.year || '?'}`)) continue;
|
|
if (queued.has(normalize(candidate.title))) continue;
|
|
if (!settings.discoveryMature && isMature(candidate)) continue;
|
|
const matchingLikedTag = candidate.tags.find(tag => likedTags.has(tag.toLowerCase()));
|
|
const matchingOwnedTag = candidate.tags.find(tag => ownedTags.has(tag.toLowerCase()));
|
|
const matchingReadTag = candidate.tags.find(tag => readTags.has(tag.toLowerCase()));
|
|
const era = likedPeriods.get(candidate.period);
|
|
const relation = candidate.relatedTo;
|
|
const scoreParts = { editorial: candidate.greatestId ? Math.max(0, 6 - Math.min(6, Number(candidate.editorialRank) || 6)) : 0,
|
|
relation: relation ? 3 : 0, sharedTaste: matchingLikedTag ? 4 : 0, sharedLibrary: matchingOwnedTag ? 1 : 0,
|
|
era: era ? 2 : 0, reading: matchingReadTag ? 2 : 0,
|
|
interest: feedback[candidate.id] === 'interested' ? 5 : feedback[candidate.id] === 'maybe' ? 2 : 0,
|
|
communityRec: 0, sameCreator: 0, tagTop: 0, fresh: 0 };
|
|
// Every applicable part brings its reason and, for taste parts, its personal evidence.
|
|
const notes = [];
|
|
if (scoreParts.editorial && candidate.evidence[0]?.note) notes.push({ part: 'editorial', text: candidate.evidence[0].note });
|
|
if (relation) notes.push({ part: 'relation', text: `${candidate.title} is listed as ${relation.type.toLowerCase().replaceAll('_', ' ')} to ${relation.title}. Check the edition and reading order first.` });
|
|
if (matchingLikedTag) {
|
|
const anchor = likedTags.get(matchingLikedTag.toLowerCase());
|
|
notes.push({ part: 'sharedTaste', text: `You liked ${anchor.title}; this also carries the ${matchingLikedTag} tag.`,
|
|
evidence: { id: `taste:greatest:${anchor.id}`, kind: 'personal', label: `Your Greatest vote for ${anchor.title}`, note: `Shared tag: ${matchingLikedTag}`, url: null } });
|
|
}
|
|
if (era) notes.push({ part: 'era', text: `You liked ${era.title}; this is another curated work from ${candidate.period}.`,
|
|
evidence: { id: `taste:greatest:${era.id}`, kind: 'personal', label: `Your Greatest vote for ${era.title}`, note: `Shared period: ${candidate.period}`, url: null } });
|
|
if (matchingReadTag) {
|
|
const title = readTags.get(matchingReadTag.toLowerCase());
|
|
notes.push({ part: 'reading', text: `You have read ${title}; this also carries the ${matchingReadTag} tag.`,
|
|
evidence: { id: `taste:read:${normalize(title)}`, kind: 'personal', label: `Your completed reading in ${title}`, note: `Shared tag: ${matchingReadTag}`, url: null } });
|
|
}
|
|
if (matchingOwnedTag) {
|
|
const title = ownedTags.get(matchingOwnedTag.toLowerCase());
|
|
notes.push({ part: 'sharedLibrary', text: `This shares the ${matchingOwnedTag} tag with ${title} in your library.`,
|
|
evidence: { id: `taste:library:${normalize(title)}`, kind: 'personal', label: `${title} in your library`, note: `Shared tag: ${matchingOwnedTag}`, url: null } });
|
|
}
|
|
for (const note of seedNotes(candidate, seeds, today.getFullYear())) { scoreParts[note.part] = note.value; notes.push(note); }
|
|
notes.sort((a, b) => scoreParts[b.part] - scoreParts[a.part] || partRank(a.part) - partRank(b.part));
|
|
const reasons = notes.map(note => String(note.text).slice(0, 280));
|
|
const fallback = candidate.evidence[0]?.note || (candidate.greatestId || relation ? 'A curated manga worth a closer look.' : 'A manga from AniList worth a closer look.');
|
|
const evidence = [...candidate.evidence];
|
|
for (const note of notes) if (note.evidence && !evidence.some(e => e.id === note.evidence.id)) evidence.push(note.evidence);
|
|
rows.push({ candidateId: candidate.id, reason: (reasons[0] || fallback).slice(0, 280), reasons, evidence,
|
|
evidenceIds: evidence.map(e => e.id), scoreParts, score: round2(Object.values(scoreParts).reduce((a, b) => a + b, 0)),
|
|
period: candidate.period || 'Related works', group: groupFor(scoreParts, candidate.period), tags: candidate.tags });
|
|
}
|
|
// Tags only steer the slot mix; the published row does not repeat them.
|
|
const items = mixSlots(rows, { topTags: seeds.topTags }).map(({ tags, ...row }) => row);
|
|
return { generatedAt: today.toISOString(), items };
|
|
}
|
|
// Sample across eras (and seed kinds) before the work cap; insertion order in the curated
|
|
// catalog must not make recent manga invisible.
|
|
function candidatePool({ greatest, library, cacheDir, harvestCache }) {
|
|
const all = buildCandidates({ greatest, library, cacheDir, harvestCache });
|
|
const buckets = new Map();
|
|
for (const candidate of all) {
|
|
const key = candidate.period || candidate.seed || 'Related works';
|
|
if (!buckets.has(key)) buckets.set(key, []);
|
|
buckets.get(key).push(candidate);
|
|
}
|
|
const candidates = [];
|
|
while (candidates.length < MAX_CANDIDATES && [...buckets.values()].some(values => values.length)) {
|
|
for (const values of buckets.values()) {
|
|
if (values.length && candidates.length < MAX_CANDIDATES) candidates.push(values.shift());
|
|
}
|
|
}
|
|
return { all, candidates };
|
|
}
|
|
const runResult = (all, candidates, feeds) => ({ candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds,
|
|
examined: all.length, clipped: Math.max(0, all.length - candidates.length) });
|
|
function generate({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], now }) {
|
|
const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache });
|
|
const feeds = {};
|
|
for (const user of Object.values(db.users || {}))
|
|
feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now });
|
|
return runResult(all, candidates, feeds);
|
|
}
|
|
// Same result as generate(), but yields to the event loop between users so the reader stays responsive.
|
|
async function generateAsync({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], onlyUser, now }) {
|
|
const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache });
|
|
const feeds = {};
|
|
for (const user of Object.values(db.users || {})) {
|
|
if (onlyUser !== undefined && user.name !== onlyUser) continue;
|
|
await new Promise(setImmediate);
|
|
feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now });
|
|
}
|
|
return runResult(all, candidates, feeds);
|
|
}
|
|
// A per-user rescore replaces one feed; the others and the candidates they show are carried over.
|
|
function mergeUserFeed(prior, result, userName) {
|
|
const feeds = { ...(prior?.feeds || {}) };
|
|
if (result.feeds?.[userName]) feeds[userName] = result.feeds[userName];
|
|
else delete feeds[userName];
|
|
const candidates = {};
|
|
for (const [name, feed] of Object.entries(feeds)) {
|
|
if (name === userName) continue;
|
|
for (const item of feed?.items || []) {
|
|
const candidate = prior?.candidates?.[item.candidateId];
|
|
if (candidate) candidates[item.candidateId] = candidate;
|
|
}
|
|
}
|
|
Object.assign(candidates, result.candidates || {});
|
|
return { candidates, feeds };
|
|
}
|
|
|
|
module.exports = { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, cachedMetadata,
|
|
isMature, MAX_CANDIDATES, MAX_FEED, SLOTS }; |