Consolidate MangaCircus app, scraper, and Compose baseline

scraper/lib/control.js

@@ -0,0 +1,11 @@
+'use strict';
+
+class ArchiveControlError extends Error {
+  constructor(reason = 'paused') {
+    super(`Archive ${reason}`);
+    this.name = 'ArchiveControlError';
+    this.reason = reason;
+  }
+}
+
+module.exports = { ArchiveControlError };

scraper/lib/delay.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+class PoliteGate {
+  constructor({ minMs = 8000, maxMs = 15000 } = {}) {
+    if (!Number.isFinite(minMs) || !Number.isFinite(maxMs) || minMs < 0 || maxMs < minMs) {
+      throw new Error('Invalid delay range');
+    }
+    this.minMs = minMs;
+    this.maxMs = maxMs;
+    this.lastRequestAt = 0;
+  }
+
+  async wait() {
+    if (!this.lastRequestAt) return;
+    const target = this.minMs + Math.floor(Math.random() * (this.maxMs - this.minMs + 1));
+    const remaining = target - (Date.now() - this.lastRequestAt);
+    if (remaining > 0) await sleep(remaining);
+  }
+
+  mark() {
+    this.lastRequestAt = Date.now();
+  }
+}
+
+module.exports = { PoliteGate, sleep };

scraper/lib/failures.js

@@ -0,0 +1,23 @@
+'use strict';
+
+function safeMessage(error) {
+  return String(error?.message || error || 'Unknown error').replace(/https?:\/\/[^\s]+/gi, raw => {
+    try { const u = new URL(raw); return u.origin + u.pathname; } catch { return '[source URL]'; }
+  }).slice(0, 1500);
+}
+function classifyFailure(error) {
+  const message = safeMessage(error);
+  const status = Number(error?.status || /(?:HTTP|returned)\s+(\d{3})/i.exec(message)?.[1]);
+  const code = error?.code || error?.cause?.code;
+  const failure = (kind, scope, retryable = false) => ({ kind, scope, retryable, message, ...(status ? { httpStatus: status } : {}) });
+  if ([401, 403, 429].includes(status) || /captcha|cloudflare|challenge|access.denied|account.ban|paywall|login required/i.test(message)) return failure('access', 'source');
+  if (['ENOSPC', 'EROFS', 'EACCES', 'EPERM', 'EIO'].includes(code)) return failure('storage', 'global');
+  if (error?.failureKind === 'unavailable' || /No downloadable English|title was not found|HTTP 404/i.test(message)) return failure('unavailable', 'job');
+  if (error?.failureKind === 'transient' || status >= 500 || /network|fetch failed|timed? ?out|timeout|socket|connection reset/i.test(message)
+    || ['ETIMEDOUT', 'ECONNRESET', 'ENOTFOUND', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT'].includes(code)) return failure('transient', 'source', true);
+  return failure('adapter', 'job');
+}
+function progressKey(job) {
+  return JSON.stringify([job?.completed || 0, job?.total || 0, job?.current?.sourceId || '', job?.current?.downloadedPages || 0, job?.lastChapter || '']);
+}
+module.exports = { classifyFailure, safeMessage, progressKey };

scraper/lib/http.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const { PoliteGate, sleep } = require('./delay');
+
+class StopScrapeError extends Error {}
+
+class SlowHttp {
+  constructor(options = {}) {
+    this.gate = new PoliteGate(options);
+    this.userAgent = options.userAgent || 'MangaCircus-Scraper/1.0 (personal archival; sequential requests)';
+    this.maxRetries = options.maxRetries ?? 2;
+    this.onEvent = options.onEvent;
+    this.timeoutMs = options.timeoutMs || 60000;
+  }
+
+  async get(url, options = {}) {
+    for (let attempt = 0; ; attempt++) {
+      await this.gate.wait();
+      let response;
+      try {
+        response = await fetch(url, {
+          redirect: 'follow',
+          signal: AbortSignal.timeout(this.timeoutMs),
+          headers: { 'user-agent': this.userAgent, accept: options.accept || '*/*', ...(options.headers || {}) },
+        });
+        this.gate.mark();
+      } catch (error) {
+        this.gate.mark();
+        if (attempt >= this.maxRetries) throw new Error(`Network request failed after ${attempt + 1} attempts: ${url}`, { cause: error });
+        const backoff = 30000 * (2 ** attempt);
+        const message = `Network failure; retrying in ${backoff / 1000}s (${attempt + 1}/${this.maxRetries})`;
+        console.warn(`[http] ${message}`);
+        if (this.onEvent) await this.onEvent({ level: 'warning', message, url });
+        await sleep(backoff);
+        continue;
+      }
+
+      if ([401, 403, 429].includes(response.status)) {
+        if (this.onEvent) await this.onEvent({ level: 'error', message: `Host returned ${response.status}; stopping`, url, status: response.status });
+        throw Object.assign(new StopScrapeError(`Host returned ${response.status}; stop and require human review: ${url}`), { status: response.status });
+      }
+      if (response.status >= 500 && attempt < this.maxRetries) {
+        await sleep(30000 * (2 ** attempt));
+        continue;
+      }
+      if (!response.ok) throw Object.assign(new Error(`HTTP ${response.status}: ${url}`), { status: response.status });
+      return response;
+    }
+  }
+}
+
+module.exports = { SlowHttp, StopScrapeError };

scraper/lib/images.js

@@ -0,0 +1,16 @@
+'use strict';
+
+const TYPES = [
+  { ext: 'jpg', test: b => b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff },
+  { ext: 'png', test: b => b.length >= 8 && b.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) },
+  { ext: 'gif', test: b => b.length >= 6 && /^GIF8[79]a$/.test(b.subarray(0, 6).toString('ascii')) },
+  { ext: 'webp', test: b => b.length >= 12 && b.subarray(0, 4).toString('ascii') === 'RIFF' && b.subarray(8, 12).toString('ascii') === 'WEBP' },
+  { ext: 'bmp', test: b => b.length >= 2 && b.subarray(0, 2).toString('ascii') === 'BM' },
+  { ext: 'avif', test: b => b.length >= 12 && b.subarray(4, 12).toString('ascii').includes('ftyp') && /avif|avis/.test(b.subarray(8, 32).toString('ascii')) },
+];
+
+function imageExtension(buffer) {
+  return TYPES.find(type => type.test(buffer))?.ext || null;
+}
+
+module.exports = { imageExtension };

scraper/lib/json-file.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const fs = require('fs').promises;
+const path = require('path');
+const crypto = require('crypto');
+
+async function writeJsonAtomic(file, value, io = fs) {
+  const folder = path.dirname(file), name = path.basename(file);
+  const temp = path.join(folder, `.${name}.${crypto.randomUUID()}.tmp`);
+  await io.writeFile(temp, JSON.stringify(value, null, 2) + '\n');
+  try {
+    try { await io.rename(temp, file); return; }
+    catch (error) {
+      if (!['EPERM', 'EACCES', 'EBUSY', 'EEXIST'].includes(error.code)) throw error;
+    }
+    // Windows may refuse rename-over-existing. Keep the old JSON recoverable
+    // until the replacement is safely in place; never delete it first.
+    const backup = path.join(folder, `.old-${name}-${crypto.randomUUID()}`);
+    await io.rename(file, backup);
+    try { await io.rename(temp, file); }
+    catch (error) {
+      try { await io.rename(backup, file); }
+      catch (restoreError) { error.message += `; previous JSON remains at ${backup}: ${restoreError.message}`; }
+      throw error;
+    }
+    await io.rm(backup, { force: true });
+  } finally { await io.rm(temp, { force: true }); }
+}
+
+module.exports = { writeJsonAtomic };

scraper/lib/legacy-publication.js

@@ -0,0 +1,179 @@
+'use strict';
+
+const fs = require('fs').promises;
+const path = require('path');
+const { imageExtension } = require('./images');
+const { assertInside } = require('./names');
+
+const PAGE = /^(\d{4,5})(?:_[a-z0-9]+)*\.(jpe?g|png|webp|gif|avif|bmp)$/i;
+const BOOK = /^(?:v|s|x)\d{2,3}(?: |$)/i;
+const CHAPTER = /^(?:c\d{4}(?:\.\d*[1-9])?|x\d{2})(?: |$)/i;
+const OWN_STAGE = /^\.staging-mcscraper-[a-f0-9]+$/;
+
+async function exists(target) {
+  try { await fs.access(target); return true; } catch { return false; }
+}
+
+async function renameRetry(from, to, tries = 5) {
+  for (let attempt = 0; ; attempt++) {
+    try { await fs.rename(from, to); return; } catch (error) {
+      if (attempt >= tries || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
+      await new Promise(resolve => setTimeout(resolve, 200 * (attempt + 1)));
+    }
+  }
+}
+
+async function writeJsonAtomic(file, value) {
+  const temp = `${file}.migration.tmp`;
+  const old = `${file}.migration.old`;
+  await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o644 });
+  await fs.rm(old, { force: true });
+  await renameRetry(file, old);
+  try {
+    await renameRetry(temp, file);
+  } catch (error) {
+    await renameRetry(old, file).catch(() => {});
+    await fs.rm(temp, { force: true });
+    throw error;
+  }
+  await fs.rm(old, { force: true }).catch(() => {});
+}
+
+async function directories(folder) {
+  return (await fs.readdir(folder, { withFileTypes: true })).filter(entry => entry.isDirectory());
+}
+
+async function validatePage(file, expected) {
+  const handle = await fs.open(file, 'r');
+  try {
+    const prefix = Buffer.alloc(16);
+    const { bytesRead } = await handle.read(prefix, 0, prefix.length, 0);
+    const actual = imageExtension(prefix.subarray(0, bytesRead));
+    if (!actual) throw new Error(`Unsupported image bytes: ${file}`);
+    const normalized = expected.toLowerCase() === 'jpeg' ? 'jpg' : expected.toLowerCase();
+    if (actual !== normalized) throw new Error(`Image extension does not match bytes: ${file}`);
+  } finally {
+    await handle.close();
+  }
+}
+
+async function validateChapter(folder) {
+  const entries = await fs.readdir(folder, { withFileTypes: true });
+  if (entries.some(entry => entry.isDirectory())) throw new Error(`Chapter contains a subdirectory: ${folder}`);
+  const chapterFile = path.join(folder, 'chapter.json');
+  const chapter = JSON.parse(await fs.readFile(chapterFile, 'utf8'));
+  if (!chapter.source?.name || !chapter.source?.url) throw new Error(`Invalid chapter source: ${chapterFile}`);
+  const pages = entries.filter(entry => entry.isFile() && entry.name !== 'chapter.json').map(entry => {
+    const match = PAGE.exec(entry.name);
+    if (!match) throw new Error(`Invalid chapter file: ${path.join(folder, entry.name)}`);
+    return { name: entry.name, number: Number(match[1]), extension: match[2] };
+  }).sort((a, b) => a.number - b.number);
+  if (!pages.length) throw new Error(`Chapter has no pages: ${folder}`);
+  for (let index = 0; index < pages.length; index++) {
+    if (pages[index].number !== index + 1) throw new Error(`Non-contiguous pages in ${folder}`);
+    await validatePage(path.join(folder, pages[index].name), pages[index].extension);
+  }
+  return pages.length;
+}
+
+async function inspectLegacyWork({ libraryRoot, seriesId }) {
+  const id = String(seriesId || '');
+  if (!/^[A-Za-z0-9-]+$/.test(id)) throw new Error(`Unsafe legacy series id: ${id}`);
+  const root = path.resolve(libraryRoot);
+  const workRoot = assertInside(root, path.join(root, `.mcscraper-work-${id}`));
+  const stateFile = path.join(workRoot, 'job.json');
+  if (!(await exists(stateFile))) return null;
+  const state = JSON.parse(await fs.readFile(stateFile, 'utf8'));
+  if (state.publicationMode === 'incremental') return null;
+  const mangaDirs = (await directories(workRoot)).filter(entry => !entry.name.startsWith('.'));
+  if (!mangaDirs.length) return null;
+  if (mangaDirs.length !== 1) throw new Error(`Legacy work ${id} must contain exactly one manga folder`);
+  const mangaDir = path.join(workRoot, mangaDirs[0].name);
+  const target = assertInside(root, path.join(root, mangaDirs[0].name));
+  if (await exists(target)) throw new Error(`Cannot publish legacy work; target already exists: ${target}`);
+  const meta = JSON.parse(await fs.readFile(path.join(mangaDir, 'manga.json'), 'utf8'));
+  if (!meta.title || !Array.isArray(meta.sources) || !meta.sources.length) throw new Error(`Invalid legacy manga metadata: ${mangaDir}`);
+
+  let chapterCount = 0, pageCount = 0;
+  const incompleteStages = [];
+  for (const book of await directories(mangaDir)) {
+    if (book.name.startsWith('.')) throw new Error(`Unexpected hidden folder remains in ${mangaDir}`);
+    if (!BOOK.test(book.name)) throw new Error(`Invalid legacy book folder: ${book.name}`);
+    const bookDir = path.join(mangaDir, book.name);
+    for (const chapter of await directories(bookDir)) {
+      if (chapter.name.startsWith('.')) {
+        if (!OWN_STAGE.test(chapter.name)) throw new Error(`Unexpected hidden folder remains in ${bookDir}: ${chapter.name}`);
+        incompleteStages.push(path.join(bookDir, chapter.name));
+        continue;
+      }
+      if (!CHAPTER.test(chapter.name)) throw new Error(`Invalid legacy chapter folder: ${chapter.name}`);
+      pageCount += await validateChapter(path.join(bookDir, chapter.name));
+      chapterCount++;
+    }
+  }
+  const completedCount = new Set(state.completed || []).size;
+  if (!chapterCount || chapterCount !== completedCount) {
+    throw new Error(`Legacy work ${id} has ${chapterCount} complete folders but ${completedCount} completed checkpoint entries`);
+  }
+  return {
+    id, root, workRoot, stateFile, state, mangaDir, target,
+    mangaId: path.basename(target), chapterCount, pageCount, incompleteStages,
+  };
+}
+
+async function promoteLegacyWork(inspection) {
+  const { stateFile, mangaDir, target, state, workRoot } = inspection;
+  const quarantineRoot = path.join(workRoot, '.legacy-incomplete');
+  const quarantined = [];
+  if (inspection.incompleteStages.length) await fs.mkdir(quarantineRoot, { recursive: true, mode: 0o755 });
+  for (const stage of inspection.incompleteStages) {
+    const relative = path.relative(mangaDir, stage).split(path.sep).join('__');
+    const quarantine = path.join(quarantineRoot, relative);
+    await renameRetry(stage, quarantine);
+    quarantined.push({ stage, quarantine });
+  }
+  try {
+    await renameRetry(mangaDir, target);
+  } catch (error) {
+    for (const item of quarantined.reverse()) await renameRetry(item.quarantine, item.stage).catch(() => {});
+    throw error;
+  }
+  const migrated = {
+    ...state,
+    publicationMode: 'incremental',
+    status: 'paused',
+    current: null,
+    jobCompleted: 0,
+    total: 0,
+    migratedToIncremental: {
+      time: new Date().toISOString(),
+      mangaId: inspection.mangaId,
+      chaptersPublished: inspection.chapterCount,
+      pagesPublished: inspection.pageCount,
+      incompleteStagesQuarantined: quarantined.length,
+    },
+  };
+  try {
+    await writeJsonAtomic(stateFile, migrated);
+  } catch (error) {
+    await renameRetry(target, mangaDir).catch(() => {});
+    for (const item of quarantined.reverse()) await renameRetry(item.quarantine, item.stage).catch(() => {});
+    throw error;
+  }
+  await fs.appendFile(path.join(workRoot, 'job.log'), JSON.stringify({
+    time: migrated.migratedToIncremental.time,
+    level: 'info',
+    message: `Published ${inspection.chapterCount} completed legacy chapters and switched to incremental mode`,
+    mangaId: inspection.mangaId,
+  }) + '\n').catch(() => {});
+  return {
+    seriesId: inspection.id,
+    mangaId: inspection.mangaId,
+    finalPath: target,
+    chaptersPublished: inspection.chapterCount,
+    pagesPublished: inspection.pageCount,
+    incompleteStagesQuarantined: quarantined.length,
+  };
+}
+
+module.exports = { inspectLegacyWork, promoteLegacyWork };

scraper/lib/library-inventory.js

@@ -0,0 +1,158 @@
+'use strict';
+
+const fs = require('fs').promises;
+const path = require('path');
+const { assertInside } = require('./names');
+const { chapterNumber } = require('./scope');
+const { isSpecialKey, parseSpecialFolder, parseKey } = require('../../chapter-identity');
+const { writeJsonAtomic } = require('./json-file');
+
+async function readJson(file) {
+  try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return {}; }
+}
+
+async function directories(folder) {
+  try { return (await fs.readdir(folder, { withFileTypes: true })).filter(entry => entry.isDirectory() && !entry.name.startsWith('.')); }
+  catch { return []; }
+}
+
+function keyFromName(name) { return parseKey(name)?.key || null; }
+
+function sourceKey(url, name = '') {
+  try {
+    const host = new URL(String(url)).hostname.replace(/^www\./, '').toLowerCase();
+    if (host === 'weebcentral.com') return 'weebcentral';
+    if (host === 'mangadex.org') return 'mangadex';
+    return host;
+  } catch {}
+  const label = String(name).toLowerCase();
+  if (label.startsWith('mangadex')) return 'mangadex';
+  if (label.startsWith('weebcentral')) return 'weebcentral';
+  return label.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
+}
+
+function autoUpdateFor(meta) {
+  const configured = meta?.['x-scraper']?.autoUpdate;
+  return configured === undefined ? String(meta?.status || '').toLowerCase() === 'ongoing' : !!configured;
+}
+
+async function scanLibrary(libraryRoot) {
+  const root = path.resolve(libraryRoot);
+  const mangas = [];
+  for (const mangaEntry of await directories(root)) {
+    const mangaPath = path.join(root, mangaEntry.name);
+    const meta = await readJson(path.join(mangaPath, 'manga.json'));
+    const chapters = [];
+    for (const bookEntry of await directories(mangaPath)) {
+      const book = keyFromName(bookEntry.name);
+      if (!book) continue;
+      const bookPath = path.join(mangaPath, bookEntry.name);
+      for (const versionEntry of await directories(bookPath)) {
+        const number = parseSpecialFolder(versionEntry.name)?.key || keyFromName(versionEntry.name);
+        if (!number) continue;
+        const chapterMeta = await readJson(path.join(bookPath, versionEntry.name, 'chapter.json'));
+        const source = typeof chapterMeta.source === 'string' ? { name: chapterMeta.source, url: '' } : (chapterMeta.source || {});
+        chapters.push({
+          book, number, version: versionEntry.name,
+          sourceName: String(source.name || ''), sourceUrl: String(source.url || ''),
+          sourceKey: sourceKey(source.url, source.name),
+        });
+      }
+    }
+    mangas.push({
+      id: mangaEntry.name, path: mangaPath, title: String(meta.title || mangaEntry.name), meta,
+      status: String(meta.status || 'unknown').toLowerCase(), autoUpdate: autoUpdateFor(meta),
+      ids: { ...(meta['x-scraper']?.ids || {}) }, sources: Array.isArray(meta.sources) ? meta.sources : [], chapters,
+    });
+  }
+  return { root, mangas };
+}
+
+function findManga(inventory, identity = {}) {
+  const source = String(identity.source || '');
+  const seriesId = identity.seriesId == null ? '' : String(identity.seriesId);
+  const sourceUrl = String(identity.sourceUrl || '');
+  const anilist = identity.anilist == null ? '' : String(identity.anilist);
+  const explicit = identity.libraryMangaId == null ? '' : String(identity.libraryMangaId);
+  if (explicit) return inventory.mangas.find(manga => manga.id === explicit) || null;
+  if (source && seriesId) {
+    const bySourceId = inventory.mangas.find(manga => String(manga.ids[source] ?? '') === seriesId);
+    if (bySourceId) return bySourceId;
+  }
+  if (sourceUrl) {
+    const byUrl = inventory.mangas.find(manga => manga.sources.some(item => item.url === sourceUrl));
+    if (byUrl) return byUrl;
+  }
+  if (anilist) {
+    const matches = inventory.mangas.filter(manga => String(manga.ids.anilist ?? manga.meta?.['x-anilist']?.id ?? '') === anilist);
+    // AniList identifies the work, not every pilot/edition on a source. Never
+    // silently merge a different listing from that same host into its sibling.
+    if (matches.length !== 1) return null;
+    if (source && seriesId && matches[0].ids[source] != null && String(matches[0].ids[source]) !== seriesId) return null;
+    return matches[0];
+  }
+  return null;
+}
+
+function chapterBook(ref, manga) {
+  if (manga?.meta?.['x-scraper']?.strategy === 'volumes' && ref.volume != null) return chapterNumber(ref.volume, 'volume');
+  return chapterNumber(ref.book?.number || 1, 'book');
+}
+
+function existingVersions(manga, ref) {
+  if (!manga) return [];
+  const book = chapterBook(ref, manga);
+  const number = isSpecialKey(ref.number) ? String(ref.number).toLowerCase() : chapterNumber(ref.number);
+  return manga.chapters.filter(item => item.book === book && item.number === number);
+}
+
+function shouldDownload(manga, ref, source, existingPolicy = 'skip-any') {
+  const versions = existingVersions(manga, ref);
+  if (!versions.length) return true;
+  if (existingPolicy === 'add-source') return !versions.some(item => item.sourceKey === source);
+  return false;
+}
+
+function isNewerThanLibrary(manga, ref) {
+  if (isSpecialKey(ref.number)) return !existingVersions(manga, ref).length;
+  if (!manga?.chapters?.length) return true;
+  const regular = manga.chapters.filter(item => !String(item.book).startsWith('x') && /^\d+(?:\.\d+)?$/.test(String(item.number)));
+  if (!regular.length) return true;
+  const latest = regular.reduce((best, item) => {
+    if (!best) return item;
+    return Number(item.book) > Number(best.book)
+      || (Number(item.book) === Number(best.book) && Number(item.number) > Number(best.number)) ? item : best;
+  }, null);
+  const candidate = { book: chapterBook(ref, manga), number: chapterNumber(ref.number) };
+  return Number(candidate.book) > Number(latest.book)
+    || (Number(candidate.book) === Number(latest.book) && Number(candidate.number) > Number(latest.number));
+}
+
+function summarizeInventory(inventory, supported) {
+  return inventory.mangas.map(manga => ({
+    id: manga.id, title: manga.title, status: manga.status, autoUpdate: manga.autoUpdate,
+    chapters: new Set(manga.chapters.map(item => `${item.book}/${item.number}`)).size,
+    versions: manga.chapters.length,
+    sources: manga.sources.map(source => {
+      const found = supported(source.url);
+      return { name: source.name, url: source.url, adapter: found?.adapter.id || null };
+    }),
+  }));
+}
+
+async function setAutoUpdate(libraryRoot, mangaId, enabled) {
+  const root = path.resolve(libraryRoot);
+  const folder = assertInside(root, path.join(root, String(mangaId)));
+  const stat = await fs.stat(folder).catch(() => null);
+  if (!stat?.isDirectory() || path.basename(folder).startsWith('.')) throw new Error('Library manga not found');
+  const file = path.join(folder, 'manga.json');
+  const meta = await readJson(file);
+  meta['x-scraper'] = { ...(meta['x-scraper'] || {}), autoUpdate: !!enabled };
+  await writeJsonAtomic(file, meta);
+  return { id: mangaId, autoUpdate: !!enabled };
+}
+
+module.exports = {
+  scanLibrary, findManga, existingVersions, shouldDownload, summarizeInventory,
+  isNewerThanLibrary, setAutoUpdate, sourceKey, autoUpdateFor, keyFromName,
+};

scraper/lib/metadata-merge.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const IMPORTED_FIELDS = ['title', 'altTitles', 'authors', 'status', 'year', 'tags', 'description'];
+const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+// The effective fields remain at the top of manga.json. A snapshot of the last
+// imported values lets later imports refresh unchanged fields without replacing
+// values somebody edited in the library. Legacy files have no snapshot, so their
+// existing values are conservatively treated as intentional.
+function mergeImportedMetadata(prior, imported, fallbackTitle) {
+  const previous = prior['x-metadata']?.imported || {};
+  const result = { ...prior };
+  for (const field of IMPORTED_FIELDS) {
+    if (!Object.hasOwn(imported, field)) continue;
+    if (result[field] === undefined || result[field] === null || result[field] === ''
+      || (Object.hasOwn(previous, field) && same(result[field], previous[field]))) {
+      result[field] = imported[field];
+    }
+  }
+  if (!result.title) result.title = fallbackTitle;
+
+  const oldId = prior['x-anilist']?.id;
+  const newId = imported['x-anilist']?.id;
+  if (oldId != null && newId != null && String(oldId) !== String(newId)) {
+    throw new Error(`AniList identity conflict: existing ${oldId}, imported ${newId}; review the manga match`);
+  }
+  if (imported['x-anilist']) result['x-anilist'] = { ...(prior['x-anilist'] || {}), ...imported['x-anilist'] };
+  const snapshot = { ...previous };
+  for (const field of IMPORTED_FIELDS) if (Object.hasOwn(imported, field)) snapshot[field] = imported[field];
+  if (Object.keys(snapshot).length) result['x-metadata'] = { ...(prior['x-metadata'] || {}), imported: snapshot };
+  return result;
+}
+
+module.exports = { mergeImportedMetadata, IMPORTED_FIELDS };

scraper/lib/names.js

@@ -0,0 +1,50 @@
+'use strict';
+
+const path = require('path');
+const { isSpecialKey } = require('../../chapter-identity');
+
+const RESERVED = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])$/i;
+
+function cleanPart(value, max = 120) {
+  let result = String(value || '')
+    .normalize('NFC')
+    .replace(/[<>:"/\\|?*\x00-\x1f]/g, '')
+    .replace(/[\[\]]/g, '')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .replace(/[ .]+$/g, '');
+  if (!result) result = 'Untitled';
+  if (RESERVED.test(result)) result += '-manga';
+  return result.slice(0, max).replace(/[ .]+$/g, '');
+}
+
+function chapterKey(value) {
+  const raw = String(value).trim();
+  if (isSpecialKey(raw)) return raw.toLowerCase();
+  if (/^\d+(?:\.\d+)?$/.test(raw)) {
+    const [integer, decimal = ''] = raw.split('.');
+    const normalized = decimal.replace(/0+$/g, '');
+    return `c${integer.padStart(4, '0')}${normalized ? `.${normalized}` : ''}`;
+  }
+  const extra = raw.match(/^x(\d+)$/i);
+  if (extra) return `x${extra[1].padStart(2, '0')}`;
+  throw new Error(`Unsupported chapter number: ${value}`);
+}
+
+function chapterFolder(chapter) {
+  const parts = [chapterKey(chapter.number)];
+  if (chapter.title) parts.push(cleanPart(chapter.title));
+  parts.push(`[${cleanPart(chapter.source.name, 50)}]`);
+  return parts.join(' ');
+}
+
+function assertInside(parent, child) {
+  const root = path.resolve(parent);
+  const target = path.resolve(child);
+  if (target !== root && !target.startsWith(root + path.sep)) {
+    throw new Error(`Path escapes library root: ${target}`);
+  }
+  return target;
+}
+
+module.exports = { cleanPart, chapterKey, chapterFolder, assertInside };

scraper/lib/publisher.js

@@ -0,0 +1,279 @@
+'use strict';
+
+const fs = require('fs');
+const fsp = fs.promises;
+const path = require('path');
+const crypto = require('crypto');
+const { SlowHttp } = require('./http');
+const { imageExtension } = require('./images');
+const { cleanPart, chapterFolder, assertInside } = require('./names');
+const { sizeFromBuffer, imageSize } = require('../../library');
+const { directionFields } = require('../../reading-direction');
+const { mergeImportedMetadata } = require('./metadata-merge');
+
+const token = () => crypto.randomBytes(5).toString('hex');
+// Staging is a recovery asset, not disposable garbage. Legacy stages without an
+// identity must be reviewed manually; never infer a chapter from page filenames.
+const ownTemp = name => /^\.old-mcscraper-[a-f0-9]+$/.test(name);
+const digest = value => crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex');
+const coverFile = name => /^cover\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name);
+
+async function exists(file) {
+  try { await fsp.access(file); return true; } catch { return false; }
+}
+
+async function readMetadata(file) {
+  try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^\uFEFF/, '')); }
+  catch (error) { if (error.code === 'ENOENT') return {}; throw error; }
+}
+
+function pageDimensions(size, tags = []) {
+  const rotation = tags.find(tag => /^rot(90|180|270)$/.test(tag));
+  return { w: size?.w || 0, h: size?.h || 0, r: rotation ? Number(rotation.slice(3)) : 0 };
+}
+
+async function publishedDimensions(folder) {
+  const pages = [];
+  for (const name of (await fsp.readdir(folder)).sort()) {
+    if (!/\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name)) continue;
+    const tags = name.slice(0, name.lastIndexOf('.')).split('_').slice(1);
+    pages.push(pageDimensions(await imageSize(path.join(folder, name)).catch(() => null), tags));
+  }
+  return pages;
+}
+
+async function renameRetry(from, to, tries = 5) {
+  for (let attempt = 0; ; attempt++) {
+    try { await fsp.rename(from, to); return; } catch (error) {
+      if (attempt >= tries || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
+      await new Promise(resolve => setTimeout(resolve, 200 * (attempt + 1)));
+    }
+  }
+}
+
+async function writeJsonAtomic(file, value) {
+  const temp = path.join(path.dirname(file), `.${path.basename(file)}.tmp`);
+  await fsp.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o644 });
+  if (!(await exists(file))) return renameRetry(temp, file);
+  try {
+    await renameRetry(temp, file);
+  } catch (error) {
+    if (!['EPERM', 'EBUSY', 'EACCES', 'EEXIST'].includes(error.code)) throw error;
+    const old = path.join(path.dirname(file), `.old-mcscraper-${token()}`);
+    await renameRetry(file, old);
+    try { await renameRetry(temp, file); } catch (moveError) { await renameRetry(old, file); throw moveError; }
+    await fsp.rm(old, { force: true });
+  }
+}
+
+async function cleanupOwnTemps(folder, maxAgeMs = 6 * 60 * 60 * 1000) {
+  if (!(await exists(folder))) return;
+  const now = Date.now();
+  for (const entry of await fsp.readdir(folder, { withFileTypes: true })) {
+    if (!entry.isDirectory() || !ownTemp(entry.name)) continue;
+    const target = path.join(folder, entry.name);
+    const stat = await fsp.stat(target);
+    if (now - stat.mtimeMs > maxAgeMs) await fsp.rm(target, { recursive: true, force: true });
+  }
+}
+
+async function findManga(libraryRoot, sourceUrls) {
+  if (!(await exists(libraryRoot))) return null;
+  for (const entry of await fsp.readdir(libraryRoot, { withFileTypes: true })) {
+    if (!entry.isDirectory() || entry.name.startsWith('.')) continue;
+    const file = path.join(libraryRoot, entry.name, 'manga.json');
+    try {
+      const json = JSON.parse(await fsp.readFile(file, 'utf8'));
+      if ((json.sources || []).some(source => sourceUrls.has(source.url))) return path.join(libraryRoot, entry.name);
+    } catch {}
+  }
+  return null;
+}
+
+function validateManifest(manifest) {
+  if (!manifest || manifest.version !== 1) throw new Error('Manifest version must be 1');
+  if (!manifest.series?.title) throw new Error('series.title is required');
+  if (!['rtl', 'ltr', 'ttb'].includes(manifest.series.direction)) throw new Error('series.direction must be rtl, ltr, or ttb');
+  if (!Array.isArray(manifest.series.sources) || !manifest.series.sources.length) throw new Error('series.sources is required');
+  if (!Array.isArray(manifest.chapters) || !manifest.chapters.length) throw new Error('At least one chapter is required');
+  for (const chapter of manifest.chapters) {
+    if (chapter.number == null || chapter.number === '' || !chapter.source?.name || !chapter.source?.url) throw new Error('Every chapter needs number and source {name,url}');
+    if (!Array.isArray(chapter.pages) || !chapter.pages.length) throw new Error(`Chapter ${chapter.number} has no pages`);
+    for (const page of chapter.pages) {
+      if (!!page.url === !!page.path) throw new Error('Every page must have exactly one of url or path');
+      for (const tag of page.tags || []) if (!/^[a-z0-9]+$/.test(tag)) throw new Error(`Invalid page tag: ${tag}`);
+    }
+  }
+}
+
+async function pageBytes(page, http) {
+  if (page.path) return fsp.readFile(path.resolve(page.path));
+  const response = await http.get(page.url, { headers: page.headers || {} });
+  return Buffer.from(await response.arrayBuffer());
+}
+
+function validateCoverUrl(value) {
+  const url = new URL(value);
+  if (url.protocol !== 'https:' || !(url.hostname === 'anilist.co' || url.hostname.endsWith('.anilist.co'))) {
+    throw new Error('Series cover must use the AniList HTTPS image host');
+  }
+  return url.href;
+}
+
+async function installSeriesCover(mangaDir, coverUrl, http) {
+  const files = await fsp.readdir(mangaDir);
+  if (files.some(coverFile)) return { status: 'existing' };
+  const response = await http.get(validateCoverUrl(coverUrl));
+  const data = Buffer.from(await response.arrayBuffer());
+  if (data.length > 10 * 1024 * 1024) throw new Error('Series cover exceeds 10 MiB');
+  const ext = imageExtension(data);
+  if (!ext) throw new Error('Series cover is not a supported image');
+  const temp = path.join(mangaDir, `.cover-mcscraper-${token()}.${ext}`);
+  const target = path.join(mangaDir, `cover.${ext}`);
+  await fsp.writeFile(temp, data, { mode: 0o644 });
+  try { await renameRetry(temp, target); } catch (error) { await fsp.rm(temp, { force: true }); throw error; }
+  return { status: 'installed', file: path.basename(target) };
+}
+
+async function stageChapter(bookDir, chapter, http, onPage, priorMeta = {}) {
+  // Full manifest fingerprint prevents reusing a page from a changed release.
+  // Store only the hash, never image URLs/headers (which may contain signatures).
+  const identity = digest([chapterFolder(chapter), chapter.source, chapter.pages.map(p => [p.url || path.resolve(p.path), p.tags || []])]);
+  const stage = path.join(bookDir, `.staging-mcscraper-resume-${identity}`);
+  await fsp.mkdir(stage, { recursive: true, mode: 0o755 });
+  const dimensions = [];
+  {
+    for (let index = 0; index < chapter.pages.length; index++) {
+      const tags = (chapter.pages[index].tags || []).map(tag => `_${tag}`).join('');
+      const prefix = `${String(index + 1).padStart(4, '0')}${tags}`;
+      const prior = (await fsp.readdir(stage)).filter(name => name.startsWith(prefix + '.') && !name.endsWith('.tmp'));
+      if (prior.length === 1) {
+        const bytes = await fsp.readFile(path.join(stage, prior[0]));
+        if (imageExtension(bytes) === path.extname(prior[0]).slice(1)) {
+          dimensions.push(pageDimensions(sizeFromBuffer(bytes), chapter.pages[index].tags));
+          if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length });
+          continue;
+        }
+      }
+      if (prior.length) throw new Error(`Partial page ${index + 1} requires review: invalid or ambiguous image`);
+      const data = await pageBytes(chapter.pages[index], http);
+      const ext = imageExtension(data);
+      if (!ext) throw new Error(`Page ${index + 1} is not a supported image`);
+      dimensions.push(pageDimensions(sizeFromBuffer(data), chapter.pages[index].tags));
+      const target = path.join(stage, `${prefix}.${ext}`);
+      const pending = path.join(stage, `.${prefix}.tmp`);
+      await fsp.writeFile(pending, data, { mode: 0o644 });
+      await renameRetry(pending, target);
+      if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length });
+    }
+    await fsp.writeFile(path.join(stage, 'chapter.json'), JSON.stringify({
+      ...priorMeta,
+      title: chapter.title || '', source: chapter.source,
+      added: chapter.added || new Date().toISOString(),
+      priority: Number(chapter.priority || 0), extra: !!chapter.extra,
+      ...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId, sourceNumber: chapter.sourceNumber ?? null } : {}),
+      ...directionFields([priorMeta, chapter], dimensions),
+    }, null, 2) + '\n', { mode: 0o644 });
+    // A process interruption during a write can leave a hidden pending file.
+    for (const name of await fsp.readdir(stage)) {
+      if (/^\.\d{4,5}(?:_[a-z0-9]+)*\.tmp$/.test(name)) await fsp.rm(path.join(stage, name), { force: true });
+    }
+    return { stage, dimensions };
+  }
+}
+
+async function publishStaged(stage, target, replace) {
+  if (!(await exists(target))) return renameRetry(stage, target);
+  if (!replace) { await fsp.rm(stage, { recursive: true, force: true }); return false; }
+  const old = path.join(path.dirname(target), `.old-mcscraper-${token()}`);
+  await renameRetry(target, old);
+  try { await renameRetry(stage, target); } catch (error) { await renameRetry(old, target); throw error; }
+  await fsp.rm(old, { recursive: true, force: true });
+  return true;
+}
+
+async function publishManifest(manifest, options = {}) {
+  validateManifest(manifest);
+  const libraryRoot = path.resolve(options.libraryRoot || manifest.libraryRoot || 'library');
+  await fsp.mkdir(libraryRoot, { recursive: true, mode: 0o755 });
+  await cleanupOwnTemps(libraryRoot);
+  const sourceUrls = new Set(manifest.series.sources.map(source => source.url));
+  let mangaDir = options.mangaDir ? assertInside(libraryRoot, options.mangaDir) : await findManga(libraryRoot, sourceUrls);
+  if (mangaDir && !(await exists(mangaDir))) throw new Error(`Target manga folder does not exist: ${mangaDir}`);
+  if (mangaDir && path.basename(mangaDir).startsWith('.')) throw new Error('Target manga folder must be published');
+  const isNew = !mangaDir;
+  const finalName = cleanPart(manifest.series.folder || manifest.series.title);
+  const finalDir = assertInside(libraryRoot, path.join(libraryRoot, finalName));
+  if (!mangaDir && await exists(finalDir)) throw new Error(`Folder already belongs to another work: ${finalDir}`);
+  mangaDir = mangaDir || path.join(libraryRoot, `.staging-mcscraper-resume-${digest([finalName, [...sourceUrls].sort()])}`);
+  await fsp.mkdir(mangaDir, { recursive: true, mode: 0o755 });
+  const http = options.httpClient || new SlowHttp(options.http || manifest.http || {});
+  const results = [];
+  const dimensions = [];
+  let cover = null;
+  const prior = await readMetadata(path.join(mangaDir, 'manga.json'));
+  const owned = manifest.series.metadata || {};
+  // Validate identity before fetching or exposing any chapter pages.
+  const merged = mergeImportedMetadata(prior, owned, manifest.series.title);
+
+  try {
+    for (const chapter of manifest.chapters) {
+      const bookNumber = Number(chapter.book?.number || 1);
+      if (!Number.isInteger(bookNumber) || bookNumber < 1) throw new Error(`Invalid book number for chapter ${chapter.number}`);
+      const bookName = `v${String(bookNumber).padStart(2, '0')}${chapter.book?.title ? ` ${cleanPart(chapter.book.title)}` : ''}`;
+      const bookDir = path.join(mangaDir, bookName);
+      await fsp.mkdir(bookDir, { recursive: true, mode: 0o755 });
+      await cleanupOwnTemps(bookDir);
+      if (chapter.book?.title) {
+        const file = path.join(bookDir, 'book.json'), priorBook = await readMetadata(file);
+        await writeJsonAtomic(file, { ...priorBook, title: chapter.book.title,
+          ...(chapter.book.direction || chapter.book.directionOverride
+            ? directionFields([priorBook, chapter.book], []) : {}) });
+      }
+      const folder = chapterFolder(chapter);
+      const target = path.join(bookDir, folder);
+      if (await exists(target) && !options.replace) {
+        dimensions.push(...await publishedDimensions(target));
+        results.push({ chapter: chapter.number, status: 'skipped' }); continue;
+      }
+      const priorChapter = await readMetadata(path.join(target, 'chapter.json'));
+      const staged = await stageChapter(bookDir, chapter, http, options.onPage, priorChapter);
+      dimensions.push(...staged.dimensions);
+      await publishStaged(staged.stage, target, !!options.replace);
+      results.push({ chapter: chapter.number, status: 'published', folder });
+    }
+
+    const sources = [...(prior.sources || [])];
+    for (const source of manifest.series.sources) {
+      const index = sources.findIndex(item => item.url === source.url);
+      if (index < 0) sources.push(source); else sources[index] = { ...sources[index], ...source };
+    }
+    const priorScraper = prior['x-scraper'] || {};
+    const nextScraper = { ...priorScraper, strategy: priorScraper.strategy || 'flat', ...(manifest.series.scraper || {}) };
+    await writeJsonAtomic(path.join(mangaDir, 'manga.json'), {
+      ...merged,
+      ...directionFields([prior, merged, manifest.series], dimensions),
+      sources,
+      'x-scraper': nextScraper,
+    });
+
+    if (manifest.series.coverUrl && !manifest.series.coverAttempted) {
+      manifest.series.coverAttempted = true;
+      try {
+        cover = await installSeriesCover(mangaDir, manifest.series.coverUrl, http);
+      } catch (error) {
+        cover = { status: 'error', message: error.message };
+        console.warn(`[publisher] cover unavailable for ${manifest.series.title}: ${error.message}`);
+      }
+    }
+
+    if (isNew) await renameRetry(mangaDir, finalDir);
+    return { mangaDir: isNew ? finalDir : mangaDir, results, cover };
+  } catch (error) {
+    // Preserve hidden validated pages on pause, transient failure or shutdown.
+    // Publication still occurs only after the whole chapter validates.
+    throw error;
+  }
+}
+
+module.exports = { publishManifest, validateManifest, cleanupOwnTemps, installSeriesCover, validateCoverUrl, writeJsonAtomic };

scraper/lib/scope.js

@@ -0,0 +1,74 @@
+'use strict';
+
+const PUBLIC_MODES = new Set(['missing', 'latest', 'last', 'first', 'chapter', 'after', 'volume']);
+
+function chapterNumber(value, field = 'chapter') {
+  const raw = String(value ?? '').trim();
+  if (!/^\d+(?:\.\d+)?$/.test(raw)) throw new Error(`${field} must be a numeric chapter`);
+  const [whole, decimal = ''] = raw.split('.');
+  const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0';
+  const normalizedDecimal = decimal.replace(/0+$/, '');
+  return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole;
+}
+
+function positiveCount(value) {
+  const count = Number(value);
+  if (!Number.isInteger(count) || count < 1 || count > 10000) throw new Error('count must be an integer from 1 to 10000');
+  return count;
+}
+
+function normalizeScope(input = {}) {
+  const raw = input && typeof input === 'object' ? input : {};
+  const mode = String(raw.mode || 'missing').toLowerCase();
+  if (mode === 'ids') {
+    if (!Array.isArray(raw.ids) || !raw.ids.length || raw.ids.length > 10000) throw new Error('ids scope requires 1 to 10000 source chapter ids');
+    return { mode, ids: [...new Set(raw.ids.map(value => String(value).trim()).filter(Boolean))] };
+  }
+  if (!PUBLIC_MODES.has(mode)) throw new Error(`Unsupported queue scope: ${mode}`);
+  if (mode === 'last' || mode === 'first') return { mode, count: positiveCount(raw.count) };
+  if (mode === 'chapter' || mode === 'after') return { mode, chapter: chapterNumber(raw.chapter) };
+  if (mode === 'volume') {
+    const volume = String(raw.volume ?? '').trim();
+    if (!/^\d+(?:\.\d+)?$/.test(volume)) throw new Error('volume must be numeric');
+    return { mode, volume: chapterNumber(volume, 'volume') };
+  }
+  return { mode };
+}
+
+function volumeOf(ref) {
+  const value = ref.volume ?? ref.book?.number;
+  if (value === undefined || value === null || value === '') return null;
+  try { return chapterNumber(value, 'volume'); } catch { return null; }
+}
+
+function selectByScope(chapters, input) {
+  const scope = normalizeScope(input);
+  const list = [...chapters];
+  if (scope.mode === 'missing') return list;
+  if (scope.mode === 'latest') return list.length ? [list[list.length - 1]] : [];
+  if (scope.mode === 'last') return list.slice(-scope.count);
+  if (scope.mode === 'first') return list.slice(0, scope.count);
+  const numbered = list.filter(ref => /^\d+(?:\.\d+)?$/.test(String(ref.number)));
+  if (scope.mode === 'chapter') return numbered.filter(ref => chapterNumber(ref.number) === scope.chapter);
+  if (scope.mode === 'after') return numbered.filter(ref => Number(chapterNumber(ref.number)) > Number(scope.chapter));
+  if (scope.mode === 'volume') return list.filter(ref => volumeOf(ref) === scope.volume);
+  if (scope.mode === 'ids') {
+    const ids = new Set(scope.ids);
+    return list.filter(ref => ids.has(String(ref.id)));
+  }
+  return [];
+}
+
+function scopeLabel(input) {
+  const scope = normalizeScope(input);
+  if (scope.mode === 'missing') return 'all missing chapters';
+  if (scope.mode === 'latest') return 'newest chapter';
+  if (scope.mode === 'last') return `last ${scope.count} chapters`;
+  if (scope.mode === 'first') return `first ${scope.count} chapters`;
+  if (scope.mode === 'chapter') return `chapter ${scope.chapter}`;
+  if (scope.mode === 'after') return `chapters after ${scope.chapter}`;
+  if (scope.mode === 'volume') return `volume ${scope.volume}`;
+  return `${scope.ids.length} discovered chapters`;
+}
+
+module.exports = { normalizeScope, selectByScope, scopeLabel, chapterNumber, volumeOf };