Checkpoint peering identity and opt-in discovery

scraper/lib/library-inventory.js

@@ -6,6 +6,7 @@ const { assertInside } = require('./names');
 const { chapterNumber } = require('./scope');
 const { isSpecialKey, parseSpecialFolder, parseKey } = require('../../chapter-identity');
 const { writeJsonAtomic } = require('./json-file');
+const { resolveWorkIdentity, sameWorkIdentity } = require('./work-identity');
 
 async function readJson(file) {
   try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return {}; }
@@ -20,7 +21,9 @@ function keyFromName(name) { return parseKey(name)?.key || null; }
 
 function sourceKey(url, name = '') {
   try {
-    const host = new URL(String(url)).hostname.replace(/^www\./, '').toLowerCase();
+    const parsed = new URL(String(url));
+    const host = parsed.hostname.replace(/^www\./, '').toLowerCase();
+    if (parsed.protocol === 'peer:' && /^p[0-9a-f]{24}$/.test(host)) return `peer-${host}`;
     if (host === 'weebcentral.com') return 'weebcentral';
     if (host === 'mangadex.org') return 'mangadex';
     return host;
@@ -53,7 +56,7 @@ async function scanLibrary(libraryRoot) {
         const chapterMeta = await readJson(path.join(bookPath, versionEntry.name, 'chapter.json'));
         const source = typeof chapterMeta.source === 'string' ? { name: chapterMeta.source, url: '' } : (chapterMeta.source || {});
         chapters.push({
-          book, number, version: versionEntry.name,
+          book, bookFolder: bookEntry.name, number, version: versionEntry.name,
           sourceName: String(source.name || ''), sourceUrl: String(source.url || ''),
           sourceKey: sourceKey(source.url, source.name),
         });
@@ -75,6 +78,13 @@ function findManga(inventory, identity = {}) {
   const anilist = identity.anilist == null ? '' : String(identity.anilist);
   const explicit = identity.libraryMangaId == null ? '' : String(identity.libraryMangaId);
   if (explicit) return inventory.mangas.find(manga => manga.id === explicit) || null;
+  if (identity.workIdentity) {
+    const matches = inventory.mangas.filter(manga => sameWorkIdentity(
+      resolveWorkIdentity(manga.meta, manga.id, identity.localServerId), identity.workIdentity));
+    if (matches.length !== 1) return null;
+    if (source && seriesId && matches[0].ids[source] != null && String(matches[0].ids[source]) !== seriesId) return null;
+    return matches[0];
+  }
   if (source && seriesId) {
     const bySourceId = inventory.mangas.find(manga => String(manga.ids[source] ?? '') === seriesId);
     if (bySourceId) return bySourceId;

scraper/lib/metadata-merge.js

@@ -2,6 +2,7 @@
 
 const IMPORTED_FIELDS = ['title', 'altTitles', 'authors', 'status', 'year', 'tags', 'description'];
 const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+const { normalizeWorkIdentity, sameWorkIdentity } = require('./work-identity');
 
 // The effective fields remain at the top of manga.json. A snapshot of the last
 // imported values lets later imports refresh unchanged fields without replacing
@@ -25,6 +26,12 @@ function mergeImportedMetadata(prior, imported, fallbackTitle) {
     throw new Error(`AniList identity conflict: existing ${oldId}, imported ${newId}; review the manga match`);
   }
   if (imported['x-anilist']) result['x-anilist'] = { ...(prior['x-anilist'] || {}), ...imported['x-anilist'] };
+  const oldWorkIdentity = normalizeWorkIdentity(prior['x-work-identity']);
+  const newWorkIdentity = normalizeWorkIdentity(imported['x-work-identity']);
+  if (oldWorkIdentity && newWorkIdentity && !sameWorkIdentity(oldWorkIdentity, newWorkIdentity)) {
+    throw new Error(`MangaCircus work identity conflict: existing ${oldWorkIdentity.kind} identity differs from imported ${newWorkIdentity.kind} identity`);
+  }
+  if (newWorkIdentity) result['x-work-identity'] = newWorkIdentity;
   const snapshot = { ...previous };
   for (const field of IMPORTED_FIELDS) if (Object.hasOwn(imported, field)) snapshot[field] = imported[field];
   if (Object.keys(snapshot).length) result['x-metadata'] = { ...(prior['x-metadata'] || {}), imported: snapshot };

scraper/lib/publisher.js

@@ -10,6 +10,7 @@ const { cleanPart, chapterFolder, assertInside } = require('./names');
 const { sizeFromBuffer, imageSize } = require('../../library');
 const { directionFields } = require('../../reading-direction');
 const { mergeImportedMetadata } = require('./metadata-merge');
+const { normalizeWorkIdentity } = require('./work-identity');
 
 const token = () => crypto.randomBytes(5).toString('hex');
 // Staging is a recovery asset, not disposable garbage. Legacy stages without an
@@ -204,7 +205,16 @@ async function publishManifest(manifest, options = {}) {
   const isNew = !mangaDir;
   const finalName = cleanPart(manifest.series.folder || manifest.series.title);
   const finalDir = assertInside(libraryRoot, path.join(libraryRoot, finalName));
-  if (!mangaDir && await exists(finalDir)) throw new Error(`Folder already belongs to another work: ${finalDir}`);
+  if (!mangaDir && await exists(finalDir)) {
+    const existing = await readMetadata(path.join(finalDir, 'manga.json'));
+    const incomingIdentity = normalizeWorkIdentity(manifest.series.metadata?.['x-work-identity']);
+    const existingIdentity = normalizeWorkIdentity(existing['x-work-identity']);
+    if (incomingIdentity && existingIdentity && incomingIdentity.kind === existingIdentity.kind
+      && incomingIdentity.value === existingIdentity.value) {
+      throw new Error(`Work identity matches the existing “${existing.title || finalName}” folder, but its source association was not resolved. Review the existing manga target; no files were overwritten.`);
+    }
+    throw new Error(`A library folder named “${finalName}” already exists, but this download could not be safely matched to it. No files were overwritten. Choose the existing manga as the target or review its work identity before syncing.`);
+  }
   mangaDir = mangaDir || path.join(libraryRoot, `.staging-mcscraper-resume-${digest([finalName, [...sourceUrls].sort()])}`);
   await fsp.mkdir(mangaDir, { recursive: true, mode: 0o755 });
   const http = options.httpClient || new SlowHttp(options.http || manifest.http || {});

scraper/lib/source-ledger.js

@@ -0,0 +1,54 @@
+'use strict';
+
+const { normalizeWorkIdentity } = require('./work-identity');
+
+function workKey(item) {
+  const identity = normalizeWorkIdentity(item.workIdentity);
+  if (identity) return `${identity.scheme}:${identity.kind}:${identity.value}`;
+  if (item.libraryMangaId) return `library:${item.libraryMangaId}`;
+  return `source:${item.source}:${item.seriesId || item.url}`;
+}
+
+function sourceKey(item) {
+  return `${item.source}:${item.seriesId || item.url}`;
+}
+
+function recordSource(ledger, item, observation = {}) {
+  const key = sourceKey(item);
+  const prior = ledger[key] || {};
+  const availability = observation.availability || item.availability;
+  const count = observation.catalogTotal ?? item.catalogTotal;
+  const observedAt = observation.observedAt || (count != null ? item.checked || item.completed : null) || null;
+  const catalog = count != null ? 'found' : prior.catalog || 'unverified';
+  const adapter = count != null
+    ? Number(count) > 0 ? 'compatible' : availability?.reason === 'unrecognized-labels' ? 'unsupported-labels' : 'no-selected-pages'
+    : prior.adapter || 'unverified';
+  ledger[key] = {
+    ...prior, workKey: workKey(item), source: item.source, seriesId: item.seriesId || null,
+    url: item.url || prior.url || null, title: item.title || prior.title || null,
+    sourceTitle: observation.sourceTitle || item.sourceTitle || prior.sourceTitle || null,
+    language: availability?.language || prior.language || null,
+    catalog, adapter, rights: prior.rights || 'unverified',
+    hostedCount: count == null ? prior.hostedCount ?? null : Number(count),
+    externalCount: availability?.externalReleases ?? prior.externalCount ?? null,
+    coverage: availability?.coverage || prior.coverage || 'unknown',
+    reason: observation.availability ? availability.reason || null : availability?.reason || prior.reason || null,
+    lastObservedAt: observedAt || prior.lastObservedAt || null,
+    evidence: count != null && observedAt ? { kind: 'scraper-catalog', jobId: item.id || null } : prior.evidence || { kind: 'queue-url', jobId: item.id || null },
+  };
+  if (observation.catalogTotal != null && Number(count) > 0) delete ledger[key].lastFailure;
+  return ledger[key];
+}
+
+function recordFailure(ledger, item, failure, at = new Date().toISOString()) {
+  const row = recordSource(ledger, item);
+  row.lastFailure = { kind: failure.kind, message: String(failure.message || '').slice(0, 300), at };
+  return row;
+}
+
+function ledgerRows(ledger) {
+  return Object.values(ledger || {}).sort((a, b) => String(a.title || '').localeCompare(String(b.title || ''))
+    || String(a.source || '').localeCompare(String(b.source || '')));
+}
+
+module.exports = { workKey, sourceKey, recordSource, recordFailure, ledgerRows };

scraper/lib/work-identity.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const crypto = require('node:crypto');
+
+const SCHEME = 'mangacircus-work-v1';
+const KINDS = new Set(['anilist', 'mangadex', 'weebcentral', 'hoster', 'origin']);
+
+function normalizeWorkIdentity(value) {
+  if (!value || typeof value !== 'object' || value.scheme !== SCHEME
+    || !KINDS.has(value.kind) || typeof value.value !== 'string'
+    || !value.value.trim() || value.value.length > 300) return null;
+  return { scheme: SCHEME, kind: value.kind, value: value.value.trim() };
+}
+
+function resolveWorkIdentity(meta = {}, mangaId, serverId) {
+  const carried = normalizeWorkIdentity(meta['x-work-identity']);
+  if (carried) return carried;
+
+  const anilist = meta['x-anilist']?.id ?? meta['x-scraper']?.ids?.anilist;
+  if (anilist != null && String(anilist).trim()) return { scheme: SCHEME, kind: 'anilist', value: String(anilist).trim() };
+
+  const ids = meta['x-scraper']?.ids || {};
+  for (const source of ['mangadex', 'weebcentral']) {
+    if (ids[source] != null && String(ids[source]).trim()) {
+      return { scheme: SCHEME, kind: source, value: String(ids[source]).trim() };
+    }
+  }
+  for (const source of Object.keys(ids).sort()) {
+    if (!['anilist', 'mangadex', 'weebcentral'].includes(source) && !source.startsWith('peer-')
+      && ids[source] != null && String(ids[source]).trim()) {
+      return { scheme: SCHEME, kind: 'hoster', value: `${source}:${String(ids[source]).trim()}` };
+    }
+  }
+
+  // A private, stable lineage ID lets a title safely make a round trip even if
+  // neither server has external metadata. It reveals neither the folder nor title.
+  if (typeof mangaId === 'string' && mangaId && typeof serverId === 'string' && serverId) {
+    const value = crypto.createHash('sha256').update(`${serverId}\0${mangaId}`).digest('hex');
+    return { scheme: SCHEME, kind: 'origin', value };
+  }
+  return null;
+}
+
+function sameWorkIdentity(left, right) {
+  const a = normalizeWorkIdentity(left), b = normalizeWorkIdentity(right);
+  return !!a && !!b && a.scheme === b.scheme && a.kind === b.kind && a.value === b.value;
+}
+
+module.exports = { SCHEME, normalizeWorkIdentity, resolveWorkIdentity, sameWorkIdentity };