Checkpoint peering identity and opt-in discovery

discovery/cli.js

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+'use strict';
+// Read-only CLI. Never writes the reader's discovery.json or scraper queue.
+const fs = require('node:fs');
+const path = require('node:path');
+const { loadCatalog } = require('../greatest');
+const { generate } = require('./engine');
+
+if (!process.argv.includes('--dry-run')) {
+  console.error('Usage: node discovery/cli.js run --dry-run [--data DIR] [--catalog FILE] [--show]');
+  process.exit(2);
+}
+const arg = (name, fallback) => { const i = process.argv.indexOf(name); return i < 0 ? fallback : process.argv[i + 1]; };
+const data = path.resolve(arg('--data', process.env.DATA || 'data'));
+const db = JSON.parse(fs.readFileSync(path.join(data, 'db.json'), 'utf8'));
+let library = [];
+try { library = JSON.parse(fs.readFileSync(path.join(data, 'library-index.json'), 'utf8')).mangas || []; }
+catch (error) { if (error.code !== 'ENOENT') throw error; }
+const greatest = loadCatalog(arg('--catalog', path.join(__dirname, '..', 'greatest', 'catalog.json')));
+const result = generate({ db, library, greatest, cacheDir: path.join(data, 'scraper', 'metadata', 'anilist') });
+const summary = { examined: result.examined, candidates: Object.keys(result.candidates).length,
+  optedInUsers: Object.values(db.users || {}).filter(u => u.settings?.discoveryEnabled).length,
+  feedSizes: Object.values(result.feeds).filter(f => !f.disabled).map(f => f.items.length) };
+if (process.argv.includes('--show')) summary.titles = Object.values(result.feeds).find(f => !f.disabled)?.items
+  .map(item => result.candidates[item.candidateId]?.title) || [];
+console.log(JSON.stringify(summary, null, 2));

discovery/engine.js

@@ -0,0 +1,174 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { normalize } = require('../greatest');
+
+const MAX_CANDIDATES = 50;
+const MAX_FEED = 20;
+const VALID_FORMATS = new Set(['MANGA', 'ONE_SHOT']);
+const safeUrl = value => /^https:\/\/[^\s]+$/i.test(String(value || '')) ? String(value) : null;
+const unique = values => [...new Set(values.filter(Boolean))];
+
+function cachedMetadata(dir, id) {
+  if (!Number.isSafeInteger(Number(id)) || Number(id) < 1) return null;
+  try {
+    const value = JSON.parse(fs.readFileSync(path.join(dir, `id-${Number(id)}.json`), 'utf8'));
+    return value._cacheVersion === 4 && value.metadata?.['x-anilist']?.id === Number(id) ? value.metadata : null;
+  } catch { return null; }
+}
+function candidateFromGreatest(entry, cacheDir) {
+  const id = Number(entry.metadata?.id);
+  const cached = id ? cachedMetadata(cacheDir, id) : null;
+  return {
+    id: id ? `anilist:${id}` : `greatest:${entry.id}`,
+    title: cached?.title || entry.title, year: cached?.year || entry.year || null,
+    type: cached?.['x-anilist']?.format || 'MANGA', status: cached?.status || entry.status || 'unknown',
+    cover: safeUrl(cached?.['x-anilist']?.coverUrl) || safeUrl(entry.cover),
+    metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(entry.metadata?.url) || null,
+    tags: unique(cached?.tags || []), authors: unique(cached?.authors || []),
+    period: entry.period, editorialRank: entry.rank, greatestId: entry.id,
+    evidence: [{ id: `greatest:${entry.id}`, kind: 'curation', label: 'Greatest of All Time selection',
+      note: String(entry.note || '').slice(0, 600), url: safeUrl(entry.source?.url) || safeUrl(entry.metadata?.url) }],
+    relations: cached?.['x-anilist']?.relations || [],
+  };
+}
+function addRelations(map, anchor, cacheDir) {
+  for (const relation of anchor.relations || []) {
+    if (!VALID_FORMATS.has(relation.format) || !Number.isSafeInteger(Number(relation.id))) continue;
+    const id = `anilist:${Number(relation.id)}`;
+    if (map.has(id)) continue;
+    const cached = cachedMetadata(cacheDir, relation.id);
+    map.set(id, {
+      id, title: cached?.title || relation.title || `AniList ${relation.id}`,
+      year: cached?.year || null, type: relation.format, status: cached?.status || 'unknown',
+      cover: safeUrl(cached?.['x-anilist']?.coverUrl),
+      metadataUrl: safeUrl(cached?.['x-anilist']?.url) || safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}`,
+      tags: unique(cached?.tags || []), authors: unique(cached?.authors || []), period: null,
+      editorialRank: null, greatestId: null,
+      evidence: [{ id: `relation:${anchor.id}:${id}`, kind: 'relation', label: `${relation.type || 'RELATED'} to ${anchor.title}`,
+        note: '', url: safeUrl(relation.url) || `https://anilist.co/manga/${relation.id}` }],
+      relatedTo: { id: anchor.id, title: anchor.title, type: relation.type || 'RELATED' },
+      relations: cached?.['x-anilist']?.relations || [],
+    });
+  }
+}
+function buildCandidates({ greatest, library, cacheDir }) {
+  const map = new Map();
+  for (const entry of greatest.entries) {
+    const candidate = candidateFromGreatest(entry, cacheDir);
+    if (candidate.type === 'MANGA' || candidate.type === 'ONE_SHOT') map.set(candidate.id, candidate);
+  }
+  for (const manga of library) {
+    const identity = manga.meta?.['x-anilist'];
+    if (!identity?.id) continue;
+    addRelations(map, { id: `anilist:${identity.id}`, title: manga.title, relations: identity.relations }, cacheDir);
+  }
+  for (const candidate of [...map.values()]) if (candidate.greatestId) addRelations(map, candidate, cacheDir);
+  return [...map.values()];
+}
+function ownedKeySet(library) {
+  const ids = new Set(), names = new Set();
+  for (const manga of library) {
+    const id = Number(manga.meta?.['x-anilist']?.id);
+    if (id) ids.add(`anilist:${id}`);
+    names.add(`${normalize(manga.title)}:${Number(manga.meta?.year) || '?'}`);
+  }
+  return { ids, names };
+}
+function labelForVote(greatest, id) { return greatest.entries.find(entry => entry.id === id)?.title || null; }
+function feedForUser({ user, db, candidates, greatest, library, queuedTitles = [] }) {
+  const settings = user.settings || {};
+  if (!settings.discoveryEnabled) return { generatedAt: new Date().toISOString(), items: [], disabled: true };
+  const votes = db.greatestRatings?.[user.name] || {};
+  const feedback = db.discoveryFeedback?.[user.name] || {};
+  const owned = ownedKeySet(library);
+  const queued = new Set(queuedTitles.map(normalize));
+  const positive = Object.entries(votes).filter(([, vote]) => vote === 'flame' || vote === 'good')
+    .map(([id, vote]) => ({ id, vote, title: labelForVote(greatest, id), entry: greatest.entries.find(e => e.id === id) })).filter(x => x.entry);
+  const likedTags = new Map(), likedPeriods = new Map();
+  for (const anchor of positive) {
+    likedPeriods.set(anchor.entry.period, anchor);
+    const candidate = candidates.find(c => c.greatestId === anchor.id);
+    for (const tag of candidate?.tags || []) likedTags.set(tag.toLowerCase(), anchor);
+  }
+  const ownedTags = new Map();
+  for (const manga of library) for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') ownedTags.set(tag.toLowerCase(), manga.title);
+  const readTags = new Map();
+  if (settings.discoveryReadingActivity) for (const manga of library) {
+    const progress = db.progress?.[user.name]?.[manga.id] || {};
+    if (!Object.values(progress).some(chapter => chapter?.done)) continue;
+    for (const tag of manga.meta?.tags || []) if (typeof tag === 'string') readTags.set(tag.toLowerCase(), manga.title);
+  }
+  const rows = [];
+  for (const candidate of candidates) {
+    if (candidate.greatestId && votes[candidate.greatestId]) continue;
+    if (['dismissed', 'hidden'].includes(feedback[candidate.id])) continue;
+    if (owned.ids.has(candidate.id) || owned.names.has(`${normalize(candidate.title)}:${candidate.year || '?'}`)) continue;
+    if (queued.has(normalize(candidate.title))) continue;
+    if (!settings.discoveryMature && candidate.tags.some(tag => /adult|hentai|erotica/i.test(tag))) continue;
+    const matchingLikedTag = candidate.tags.find(tag => likedTags.has(tag.toLowerCase()));
+    const matchingOwnedTag = candidate.tags.find(tag => ownedTags.has(tag.toLowerCase()));
+    const matchingReadTag = candidate.tags.find(tag => readTags.has(tag.toLowerCase()));
+    const era = likedPeriods.get(candidate.period);
+    const relation = candidate.relatedTo;
+    const scoreParts = { editorial: candidate.greatestId ? Math.max(0, 6 - Math.min(6, Number(candidate.editorialRank) || 6)) : 0,
+      relation: relation ? 3 : 0, sharedTaste: matchingLikedTag ? 4 : 0, sharedLibrary: matchingOwnedTag ? 1 : 0,
+      era: era ? 2 : 0, reading: matchingReadTag ? 2 : 0,
+      interest: feedback[candidate.id] === 'interested' ? 5 : feedback[candidate.id] === 'maybe' ? 2 : 0 };
+    let reason, personalEvidence = null;
+    if (relation) reason = `${candidate.title} is listed as ${relation.type.toLowerCase().replaceAll('_', ' ')} to ${relation.title}. Check the edition and reading order first.`;
+    else if (matchingLikedTag) {
+      const anchor = likedTags.get(matchingLikedTag.toLowerCase());
+      reason = `You liked ${anchor.title}; this also carries the ${matchingLikedTag} tag.`;
+      personalEvidence = { id: `taste:greatest:${anchor.id}`, kind: 'personal', label: `Your Greatest vote for ${anchor.title}`, note: `Shared tag: ${matchingLikedTag}`, url: null };
+    } else if (era) {
+      reason = `You liked ${era.title}; this is another curated work from ${candidate.period}.`;
+      personalEvidence = { id: `taste:greatest:${era.id}`, kind: 'personal', label: `Your Greatest vote for ${era.title}`, note: `Shared period: ${candidate.period}`, url: null };
+    } else if (matchingReadTag) {
+      const title = readTags.get(matchingReadTag.toLowerCase());
+      reason = `You have read ${title}; this also carries the ${matchingReadTag} tag.`;
+      personalEvidence = { id: `taste:read:${normalize(title)}`, kind: 'personal', label: `Your completed reading in ${title}`, note: `Shared tag: ${matchingReadTag}`, url: null };
+    } else if (matchingOwnedTag) {
+      const title = ownedTags.get(matchingOwnedTag.toLowerCase());
+      reason = `This shares the ${matchingOwnedTag} tag with ${title} in your library.`;
+      personalEvidence = { id: `taste:library:${normalize(title)}`, kind: 'personal', label: `${title} in your library`, note: `Shared tag: ${matchingOwnedTag}`, url: null };
+    }
+    else reason = candidate.evidence[0]?.note || 'A curated manga worth a closer look.';
+    reason = reason.slice(0, 280);
+    const evidence = [...candidate.evidence, ...(personalEvidence ? [personalEvidence] : [])];
+    rows.push({ candidateId: candidate.id, reason, evidence, evidenceIds: evidence.map(e => e.id), scoreParts,
+      score: Object.values(scoreParts).reduce((a, b) => a + b, 0), period: candidate.period || 'Related works' });
+  }
+  rows.sort((a, b) => b.score - a.score || a.candidateId.localeCompare(b.candidateId));
+  const picked = [], periodCounts = new Map();
+  for (const row of rows) {
+    if (picked.length >= MAX_FEED) break;
+    if ((periodCounts.get(row.period) || 0) >= 4) continue;
+    picked.push(row); periodCounts.set(row.period, (periodCounts.get(row.period) || 0) + 1);
+  }
+  return { generatedAt: new Date().toISOString(), items: picked.map((row, index) => ({ ...row, rank: index + 1 })) };
+}
+function generate({ db, library, greatest, cacheDir, queuedTitles = [] }) {
+  const all = buildCandidates({ greatest, library, cacheDir });
+  // Sample across eras before the work cap; insertion order in the curated catalog
+  // must not make recent manga invisible.
+  const buckets = new Map();
+  for (const candidate of all) {
+    const key = candidate.period || 'Related works';
+    if (!buckets.has(key)) buckets.set(key, []);
+    buckets.get(key).push(candidate);
+  }
+  const candidates = [];
+  while (candidates.length < MAX_CANDIDATES && [...buckets.values()].some(values => values.length)) {
+    for (const values of buckets.values()) {
+      if (values.length && candidates.length < MAX_CANDIDATES) candidates.push(values.shift());
+    }
+  }
+  const feeds = {};
+  for (const user of Object.values(db.users || {})) feeds[user.name] = feedForUser({ user, db, candidates, greatest, library, queuedTitles });
+  return { candidates: Object.fromEntries(candidates.map(c => [c.id, c])), feeds,
+    examined: all.length, clipped: Math.max(0, all.length - candidates.length) };
+}
+
+module.exports = { generate, buildCandidates, feedForUser, cachedMetadata, MAX_CANDIDATES, MAX_FEED };

discovery/store.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+
+class DiscoveryStore {
+  constructor(dataDir) { this.file = path.join(path.resolve(dataDir), 'discovery.json'); }
+  load() {
+    try {
+      const data = JSON.parse(fs.readFileSync(this.file, 'utf8'));
+      if (data.schema !== 1 || !Number.isSafeInteger(data.revision) || !data.candidates || !data.feeds || !data.lastRun)
+        throw new Error('Invalid discovery store schema');
+      return data;
+    } catch (error) {
+      if (error.code === 'ENOENT') return { schema: 1, revision: 0, candidates: {}, feeds: {}, lastRun: { status: 'never' } };
+      throw new Error(`Discovery store unavailable; refusing to overwrite it: ${error.message}`);
+    }
+  }
+  publish(value) {
+    const prior = this.load();
+    const data = { schema: 1, revision: prior.revision + 1, candidates: value.candidates, feeds: value.feeds,
+      lastRun: { status: 'success', startedAt: value.startedAt, finishedAt: new Date().toISOString(),
+        candidateCount: Object.keys(value.candidates).length, examined: value.examined, clipped: value.clipped } };
+    fs.mkdirSync(path.dirname(this.file), { recursive: true });
+    const temp = `${this.file}.${crypto.randomBytes(6).toString('hex')}.tmp`;
+    try {
+      fs.writeFileSync(temp, JSON.stringify(data, null, 2) + '\n', { mode: 0o600 });
+      if (fs.existsSync(this.file)) fs.copyFileSync(this.file, `${this.file}.prev`);
+      fs.renameSync(temp, this.file);
+    } finally { try { fs.rmSync(temp, { force: true }); } catch {} }
+    return data;
+  }
+}
+module.exports = { DiscoveryStore };