Discovery scores AniList seeds with slot mix and isAdult filter (BL-0051)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
MaashkinoozheandClaude Opus 5.5 committed
1 parent
552a599475
commit
5de5eb6a08
4 files changed
+515
-48
No files matched your search
+129
-45
@@ -3,9 +3,12 @@
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { normalize } = require('../greatest');
|
||||
const { anchorsFor } = require('./anchors');
|
||||
const { MAX_FEED, SLOTS, round2, seedIndexFor, seedNotes, groupFor, partRank, mixSlots } = require('./scoring');
|
||||
|
||||
const MAX_CANDIDATES = 50;
|
||||
const MAX_FEED = 20;
|
||||
const MAX_CANDIDATES = 2000;
|
||||
const SEED_KINDS = ['recs', 'creator', 'tagTop', 'fresh'];
|
||||
const MATURE_TAG_RANK = 60;
|
||||
const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']);
|
||||
const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null;
|
||||
const unique = values => [...new Set(values.filter(Boolean))];
|
||||
@@ -53,7 +56,42 @@ function addRelations(map, anchor, cacheDir) {
|
||||
});
|
||||
}
|
||||
}
|
||||
function buildCandidates({ greatest, library, cacheDir }) {
|
||||
const textList = values => unique(Array.isArray(values) ? values.map(value => value?.name).filter(name => typeof name === 'string') : []);
|
||||
const recordList = values => Array.isArray(values) ? values.filter(value => value && typeof value === 'object') : [];
|
||||
function candidateFromRecord(record, id, seed, fetchedAt) {
|
||||
return {
|
||||
id: `anilist:${id}`, title: String(record.title || `AniList ${id}`), year: Number(record.year) || null,
|
||||
type: record.format || 'MANGA', status: record.status || 'unknown', cover: safeUrl(record.coverUrl),
|
||||
metadataUrl: safeUrl(record.siteUrl) || `https://anilist.co/manga/${id}`,
|
||||
tags: textList(record.tags), authors: textList(record.authors), period: null, editorialRank: null, greatestId: null,
|
||||
evidence: [], relations: [], isAdult: record.isAdult === true,
|
||||
averageScore: Number.isFinite(record.averageScore) ? record.averageScore : null,
|
||||
tagRecords: recordList(record.tags), authorRecords: recordList(record.authors), fetchedAt, seed,
|
||||
};
|
||||
}
|
||||
// Greatest and relation candidates keep their own fields; AniList only fills what they lack.
|
||||
function fillFromRecord(candidate, record, fetchedAt) {
|
||||
if (candidate.isAdult === undefined) candidate.isAdult = record.isAdult === true;
|
||||
if (!candidate.tagRecords) candidate.tagRecords = recordList(record.tags);
|
||||
if (!candidate.authorRecords) candidate.authorRecords = recordList(record.authors);
|
||||
if (candidate.averageScore == null && Number.isFinite(record.averageScore)) candidate.averageScore = record.averageScore;
|
||||
if (!candidate.fetchedAt) candidate.fetchedAt = fetchedAt;
|
||||
}
|
||||
function addHarvested(map, harvestCache) {
|
||||
for (const kind of SEED_KINDS) {
|
||||
const entries = harvestCache.list(kind).sort((a, b) => String(a.key).localeCompare(String(b.key)));
|
||||
for (const entry of entries) for (const item of Array.isArray(entry.data) ? entry.data : []) {
|
||||
const record = item?.record;
|
||||
const id = Number(item?.anilistId);
|
||||
if (!record || typeof record !== 'object' || !Number.isSafeInteger(id) || id < 1) continue;
|
||||
if (record.format && !VALID_FORMATS.has(record.format)) continue;
|
||||
const existing = map.get(`anilist:${id}`);
|
||||
if (existing) fillFromRecord(existing, record, entry.fetchedAt);
|
||||
else map.set(`anilist:${id}`, candidateFromRecord(record, id, kind, entry.fetchedAt));
|
||||
}
|
||||
}
|
||||
}
|
||||
function buildCandidates({ greatest, library, cacheDir, harvestCache = null }) {
|
||||
const map = new Map();
|
||||
for (const entry of greatest.entries) {
|
||||
const candidate = candidateFromGreatest(entry, cacheDir);
|
||||
@@ -65,8 +103,15 @@ function buildCandidates({ greatest, library, cacheDir }) {
|
||||
addRelations(map, { id: `anilist:${identity.id}`, title: manga.title, relations: identity.relations }, cacheDir);
|
||||
}
|
||||
for (const candidate of [...map.values()]) if (candidate.greatestId) addRelations(map, candidate, cacheDir);
|
||||
if (harvestCache) addHarvested(map, harvestCache);
|
||||
return [...map.values()];
|
||||
}
|
||||
// Title-level isAdult or a strong adult tag; the Phase A regex only where AniList gave no isAdult.
|
||||
function isMature(candidate) {
|
||||
if (candidate.isAdult === undefined) return candidate.tags.some(tag => /adult|hentai|erotica/i.test(tag));
|
||||
return candidate.isAdult === true
|
||||
|| (candidate.tagRecords || []).some(tag => tag?.isAdult && Number(tag.rank) >= MATURE_TAG_RANK);
|
||||
}
|
||||
function ownedKeySet(library) {
|
||||
const ids = new Set(), names = new Set();
|
||||
for (const manga of library) {
|
||||
@@ -77,9 +122,10 @@ function ownedKeySet(library) {
|
||||
return { ids, names };
|
||||
}
|
||||
function labelForVote(greatest, id) { return greatest.entries.find(entry => entry.id === id)?.title || null; }
|
||||
function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [] }) {
|
||||
function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [], harvestCache = null, now = () => new Date() }) {
|
||||
const settings = user.settings || {};
|
||||
if (!settings.discoveryEnabled) return { generatedAt: new Date().toISOString(), items: [], disabled: true };
|
||||
const today = now();
|
||||
if (!settings.discoveryEnabled) return { generatedAt: today.toISOString(), items: [], disabled: true };
|
||||
const votes = db.greatestRatings?.[user.name] || {};
|
||||
const feedback = db.discoveryFeedback?.[user.name] || {};
|
||||
const owned = ownedKeySet(library);
|
||||
@@ -100,13 +146,14 @@ function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [
|
||||
if (!Object.values(progress).some(chapter => chapter?.done)) continue;
|
||||
for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') readTags.set(tag.toLowerCase(), manga.title);
|
||||
}
|
||||
const seeds = seedIndexFor({ anchors: anchorsFor({ user, db, library, greatest }), harvestCache, mature: !!settings.discoveryMature });
|
||||
const rows = [];
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.greatestId && votes[candidate.greatestId]) continue;
|
||||
if (['dismissed', 'hidden'].includes(feedback[candidate.id])) continue;
|
||||
if (owned.ids.has(candidate.id) || owned.names.has(`${normalize(candidate.title)}:${candidate.year || '?'}`)) continue;
|
||||
if (queued.has(normalize(candidate.title))) continue;
|
||||
if (!settings.discoveryMature && candidate.tags.some(tag => /adult|hentai|erotica/i.test(tag))) continue;
|
||||
if (!settings.discoveryMature && isMature(candidate)) continue;
|
||||
const matchingLikedTag = candidate.tags.find(tag => likedTags.has(tag.toLowerCase()));
|
||||
const matchingOwnedTag = candidate.tags.find(tag => ownedTags.has(tag.toLowerCase()));
|
||||
const matchingReadTag = candidate.tags.find(tag => readTags.has(tag.toLowerCase()));
|
||||
@@ -115,47 +162,50 @@ function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [
|
||||
const scoreParts = { editorial: candidate.greatestId ? Math.max(0, 6 - Math.min(6, Number(candidate.editorialRank) || 6)) : 0,
|
||||
relation: relation ? 3 : 0, sharedTaste: matchingLikedTag ? 4 : 0, sharedLibrary: matchingOwnedTag ? 1 : 0,
|
||||
era: era ? 2 : 0, reading: matchingReadTag ? 2 : 0,
|
||||
interest: feedback[candidate.id] === 'interested' ? 5 : feedback[candidate.id] === 'maybe' ? 2 : 0 };
|
||||
let reason, personalEvidence = null;
|
||||
if (relation) reason = `${candidate.title} is listed as ${relation.type.toLowerCase().replaceAll('_', ' ')} to ${relation.title}. Check the edition and reading order first.`;
|
||||
else if (matchingLikedTag) {
|
||||
interest: feedback[candidate.id] === 'interested' ? 5 : feedback[candidate.id] === 'maybe' ? 2 : 0,
|
||||
communityRec: 0, sameCreator: 0, tagTop: 0, fresh: 0 };
|
||||
// Every applicable part brings its reason and, for taste parts, its personal evidence.
|
||||
const notes = [];
|
||||
if (scoreParts.editorial && candidate.evidence[0]?.note) notes.push({ part: 'editorial', text: candidate.evidence[0].note });
|
||||
if (relation) notes.push({ part: 'relation', text: `${candidate.title} is listed as ${relation.type.toLowerCase().replaceAll('_', ' ')} to ${relation.title}. Check the edition and reading order first.` });
|
||||
if (matchingLikedTag) {
|
||||
const anchor = likedTags.get(matchingLikedTag.toLowerCase());
|
||||
reason = `You liked ${anchor.title}; this also carries the ${matchingLikedTag} tag.`;
|
||||
personalEvidence = { id: `taste:greatest:${anchor.id}`, kind: 'personal', label: `Your Greatest vote for ${anchor.title}`, note: `Shared tag: ${matchingLikedTag}`, url: null };
|
||||
} else if (era) {
|
||||
reason = `You liked ${era.title}; this is another curated work from ${candidate.period}.`;
|
||||
personalEvidence = { id: `taste:greatest:${era.id}`, kind: 'personal', label: `Your Greatest vote for ${era.title}`, note: `Shared period: ${candidate.period}`, url: null };
|
||||
} else if (matchingReadTag) {
|
||||
const title = readTags.get(matchingReadTag.toLowerCase());
|
||||
reason = `You have read ${title}; this also carries the ${matchingReadTag} tag.`;
|
||||
personalEvidence = { id: `taste:read:${normalize(title)}`, kind: 'personal', label: `Your completed reading in ${title}`, note: `Shared tag: ${matchingReadTag}`, url: null };
|
||||
} else if (matchingOwnedTag) {
|
||||
const title = ownedTags.get(matchingOwnedTag.toLowerCase());
|
||||
reason = `This shares the ${matchingOwnedTag} tag with ${title} in your library.`;
|
||||
personalEvidence = { id: `taste:library:${normalize(title)}`, kind: 'personal', label: `${title} in your library`, note: `Shared tag: ${matchingOwnedTag}`, url: null };
|
||||
notes.push({ part: 'sharedTaste', text: `You liked ${anchor.title}; this also carries the ${matchingLikedTag} tag.`,
|
||||
evidence: { id: `taste:greatest:${anchor.id}`, kind: 'personal', label: `Your Greatest vote for ${anchor.title}`, note: `Shared tag: ${matchingLikedTag}`, url: null } });
|
||||
}
|
||||
else reason = candidate.evidence[0]?.note || 'A curated manga worth a closer look.';
|
||||
reason = reason.slice(0, 280);
|
||||
const evidence = [...candidate.evidence, ...(personalEvidence ? [personalEvidence] : [])];
|
||||
rows.push({ candidateId: candidate.id, reason, evidence, evidenceIds: evidence.map(e => e.id), scoreParts,
|
||||
score: Object.values(scoreParts).reduce((a, b) => a + b, 0), period: candidate.period || 'Related works' });
|
||||
if (era) notes.push({ part: 'era', text: `You liked ${era.title}; this is another curated work from ${candidate.period}.`,
|
||||
evidence: { id: `taste:greatest:${era.id}`, kind: 'personal', label: `Your Greatest vote for ${era.title}`, note: `Shared period: ${candidate.period}`, url: null } });
|
||||
if (matchingReadTag) {
|
||||
const title = readTags.get(matchingReadTag.toLowerCase());
|
||||
notes.push({ part: 'reading', text: `You have read ${title}; this also carries the ${matchingReadTag} tag.`,
|
||||
evidence: { id: `taste:read:${normalize(title)}`, kind: 'personal', label: `Your completed reading in ${title}`, note: `Shared tag: ${matchingReadTag}`, url: null } });
|
||||
}
|
||||
if (matchingOwnedTag) {
|
||||
const title = ownedTags.get(matchingOwnedTag.toLowerCase());
|
||||
notes.push({ part: 'sharedLibrary', text: `This shares the ${matchingOwnedTag} tag with ${title} in your library.`,
|
||||
evidence: { id: `taste:library:${normalize(title)}`, kind: 'personal', label: `${title} in your library`, note: `Shared tag: ${matchingOwnedTag}`, url: null } });
|
||||
}
|
||||
for (const note of seedNotes(candidate, seeds, today.getFullYear())) { scoreParts[note.part] = note.value; notes.push(note); }
|
||||
notes.sort((a, b) => scoreParts[b.part] - scoreParts[a.part] || partRank(a.part) - partRank(b.part));
|
||||
const reasons = notes.map(note => String(note.text).slice(0, 280));
|
||||
const fallback = candidate.evidence[0]?.note || (candidate.greatestId || relation ? 'A curated manga worth a closer look.' : 'A manga from AniList worth a closer look.');
|
||||
const evidence = [...candidate.evidence];
|
||||
for (const note of notes) if (note.evidence && !evidence.some(e => e.id === note.evidence.id)) evidence.push(note.evidence);
|
||||
rows.push({ candidateId: candidate.id, reason: (reasons[0] || fallback).slice(0, 280), reasons, evidence,
|
||||
evidenceIds: evidence.map(e => e.id), scoreParts, score: round2(Object.values(scoreParts).reduce((a, b) => a + b, 0)),
|
||||
period: candidate.period || 'Related works', group: groupFor(scoreParts, candidate.period), tags: candidate.tags });
|
||||
}
|
||||
rows.sort((a, b) => b.score - a.score || a.candidateId.localeCompare(b.candidateId));
|
||||
const picked = [], periodCounts = new Map();
|
||||
for (const row of rows) {
|
||||
if (picked.length >= MAX_FEED) break;
|
||||
if ((periodCounts.get(row.period) || 0) >= 4) continue;
|
||||
picked.push(row); periodCounts.set(row.period, (periodCounts.get(row.period) || 0) + 1);
|
||||
}
|
||||
return { generatedAt: new Date().toISOString(), items: picked.map((row, index) => ({ ...row, rank: index + 1 })) };
|
||||
// Tags only steer the slot mix; the published row does not repeat them.
|
||||
const items = mixSlots(rows, { topTags: seeds.topTags }).map(({ tags, ...row }) => row);
|
||||
return { generatedAt: today.toISOString(), items };
|
||||
}
|
||||
function generate({ db, library, greatest, cacheDir, queuedTitles = [] }) {
|
||||
const all = buildCandidates({ greatest, library, cacheDir });
|
||||
// Sample across eras before the work cap; insertion order in the curated catalog
|
||||
// must not make recent manga invisible.
|
||||
// Sample across eras (and seed kinds) before the work cap; insertion order in the curated
|
||||
// catalog must not make recent manga invisible.
|
||||
function candidatePool({ greatest, library, cacheDir, harvestCache }) {
|
||||
const all = buildCandidates({ greatest, library, cacheDir, harvestCache });
|
||||
const buckets = new Map();
|
||||
for (const candidate of all) {
|
||||
const key = candidate.period || 'Related works';
|
||||
const key = candidate.period || candidate.seed || 'Related works';
|
||||
if (!buckets.has(key)) buckets.set(key, []);
|
||||
buckets.get(key).push(candidate);
|
||||
}
|
||||
@@ -165,10 +215,44 @@ function generate({ db, library, greatest, cacheDir, queuedTitles = [] }) {
|
||||
if (values.length && candidates.length < MAX_CANDIDATES) candidates.push(values.shift());
|
||||
}
|
||||
}
|
||||
return { all, candidates };
|
||||
}
|
||||
const runResult = (all, candidates, feeds) => ({ candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds,
|
||||
examined: all.length, clipped: Math.max(0, all.length - candidates.length) });
|
||||
function generate({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], now }) {
|
||||
const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache });
|
||||
const feeds = {};
|
||||
for (const user of Object.values(db.users || {})) feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles });
|
||||
return { candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds,
|
||||
examined: all.length, clipped: Math.max(0, all.length - candidates.length) };
|
||||
for (const user of Object.values(db.users || {}))
|
||||
feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now });
|
||||
return runResult(all, candidates, feeds);
|
||||
}
|
||||
// Same result as generate(), but yields to the event loop between users so the reader stays responsive.
|
||||
async function generateAsync({ db, library, greatest, cacheDir, harvestCache = null, queuedTitles = [], onlyUser, now }) {
|
||||
const { all, candidates } = candidatePool({ greatest, library, cacheDir, harvestCache });
|
||||
const feeds = {};
|
||||
for (const user of Object.values(db.users || {})) {
|
||||
if (onlyUser !== undefined && user.name !== onlyUser) continue;
|
||||
await new Promise(setImmediate);
|
||||
feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles, harvestCache, now });
|
||||
}
|
||||
return runResult(all, candidates, feeds);
|
||||
}
|
||||
// A per-user rescore replaces one feed; the others and the candidates they show are carried over.
|
||||
function mergeUserFeed(prior, result, userName) {
|
||||
const feeds = { ...(prior?.feeds || {}) };
|
||||
if (result.feeds?.[userName]) feeds[userName] = result.feeds[userName];
|
||||
else delete feeds[userName];
|
||||
const candidates = {};
|
||||
for (const [name, feed] of Object.entries(feeds)) {
|
||||
if (name === userName) continue;
|
||||
for (const item of feed?.items || []) {
|
||||
const candidate = prior?.candidates?.[item.candidateId];
|
||||
if (candidate) candidates[item.candidateId] = candidate;
|
||||
}
|
||||
}
|
||||
Object.assign(candidates, result.candidates || {});
|
||||
return { candidates, feeds };
|
||||
}
|
||||
|
||||
module.exports = { generate, buildCandidates, feedForUser, cachedMetadata, MAX_CANDIDATES, MAX_FEED };
|
||||
module.exports = { generate, generateAsync, buildCandidates, feedForUser, mixSlots, mergeUserFeed, cachedMetadata,
|
||||
isMature, MAX_CANDIDATES, MAX_FEED, SLOTS };
|
||||
Reference in new issue
Block a user