'use strict'; const fs = require('fs'); const fsp = fs.promises; const path = require('path'); const crypto = require('crypto'); const { SlowHttp } = require('./http'); const { imageExtension } = require('./images'); const { cleanPart, chapterFolder, assertInside } = require('./names'); const { sizeFromBuffer, imageSize } = require('../../library'); const { directionFields } = require('../../reading-direction'); const { mergeImportedMetadata } = require('./metadata-merge'); const { normalizeWorkIdentity } = require('./work-identity'); const { parseKey } = require('../../chapter-identity'); const { publishedDates } = require('../../chapter-dates'); const token = () => crypto.randomBytes(5).toString('hex'); // Staging is a recovery asset, not disposable garbage. Legacy stages without an // identity must be reviewed manually; never infer a chapter from page filenames. const ownTemp = name => /^\.old-mcscraper-[a-f0-9]+$/.test(name); const digest = value => crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex'); const coverFile = name => /^cover\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name); async function exists(file) { try { await fsp.access(file); return true; } catch { return false; } } async function readMetadata(file) { try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^\uFEFF/, '')); } catch (error) { if (error.code === 'ENOENT') return {}; throw error; } } function pageDimensions(size, tags = []) { const rotation = tags.find(tag => /^rot(90|180|270)$/.test(tag)); return { w: size?.w || 0, h: size?.h || 0, r: rotation ? Number(rotation.slice(3)) : 0 }; } async function publishedDimensions(folder) { const pages = []; for (const name of (await fsp.readdir(folder)).sort()) { if (!/\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name)) continue; const tags = name.slice(0, name.lastIndexOf('.')).split('_').slice(1); pages.push(pageDimensions(await imageSize(path.join(folder, name)).catch(() => null), tags)); } return pages; } async function renameRetry(from, to, tries = 5) { for (let attempt = 0; ; attempt++) { try { await fsp.rename(from, to); return; } catch (error) { if (attempt >= tries || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error; await new Promise(resolve => setTimeout(resolve, 200 * (attempt + 1))); } } } async function writeJsonAtomic(file, value) { const temp = path.join(path.dirname(file), `.${path.basename(file)}.tmp`); await fsp.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o644 }); if (!(await exists(file))) return renameRetry(temp, file); try { await renameRetry(temp, file); } catch (error) { if (!['EPERM', 'EBUSY', 'EACCES', 'EEXIST'].includes(error.code)) throw error; const old = path.join(path.dirname(file), `.old-mcscraper-${token()}`); await renameRetry(file, old); try { await renameRetry(temp, file); } catch (moveError) { await renameRetry(old, file); throw moveError; } await fsp.rm(old, { force: true }); } } async function cleanupOwnTemps(folder, maxAgeMs = 6 * 60 * 60 * 1000) { if (!(await exists(folder))) return; const now = Date.now(); for (const entry of await fsp.readdir(folder, { withFileTypes: true })) { if (!entry.isDirectory() || !ownTemp(entry.name)) continue; const target = path.join(folder, entry.name); const stat = await fsp.stat(target); if (now - stat.mtimeMs > maxAgeMs) await fsp.rm(target, { recursive: true, force: true }); } } async function findManga(libraryRoot, sourceUrls) { if (!(await exists(libraryRoot))) return null; for (const entry of await fsp.readdir(libraryRoot, { withFileTypes: true })) { if (!entry.isDirectory() || entry.name.startsWith('.')) continue; const file = path.join(libraryRoot, entry.name, 'manga.json'); try { const json = JSON.parse(await fsp.readFile(file, 'utf8')); if ((json.sources || []).some(source => sourceUrls.has(source.url))) return path.join(libraryRoot, entry.name); } catch {} } return null; } async function bookDirectory(mangaDir, bookNumber, bookTitle) { const names = (await fsp.readdir(mangaDir, { withFileTypes: true })) .filter(entry => entry.isDirectory() && parseKey(entry.name)?.key === String(bookNumber)) .map(entry => entry.name); if (names.length > 1) throw new Error(`Ambiguous library book ${bookNumber}: ${names.join(', ')}`); const defaultName = `v${String(bookNumber).padStart(2, '0')}`; return path.join(mangaDir, names[0] || (bookTitle ? `${defaultName} ${cleanPart(bookTitle)}` : defaultName)); } function validateManifest(manifest) { if (!manifest || manifest.version !== 1) throw new Error('Manifest version must be 1'); if (!manifest.series?.title) throw new Error('series.title is required'); if (!['rtl', 'ltr', 'ttb'].includes(manifest.series.direction)) throw new Error('series.direction must be rtl, ltr, or ttb'); if (!Array.isArray(manifest.series.sources) || !manifest.series.sources.length) throw new Error('series.sources is required'); if (!Array.isArray(manifest.chapters) || !manifest.chapters.length) throw new Error('At least one chapter is required'); for (const chapter of manifest.chapters) { if (chapter.number == null || chapter.number === '' || !chapter.source?.name || !chapter.source?.url) throw new Error('Every chapter needs number and source {name,url}'); if (!Array.isArray(chapter.pages) || !chapter.pages.length) throw new Error(`Chapter ${chapter.number} has no pages`); for (const page of chapter.pages) { if (!!page.url === !!page.path) throw new Error('Every page must have exactly one of url or path'); for (const tag of page.tags || []) if (!/^[a-z0-9]+$/.test(tag)) throw new Error(`Invalid page tag: ${tag}`); } } } async function pageBytes(page, http) { if (page.path) return fsp.readFile(path.resolve(page.path)); const response = await http.get(page.url, { headers: page.headers || {} }); return Buffer.from(await response.arrayBuffer()); } function validateCoverUrl(value) { const url = new URL(value); if (url.protocol !== 'https:' || !(url.hostname === 'anilist.co' || url.hostname.endsWith('.anilist.co'))) { throw new Error('Series cover must use the AniList HTTPS image host'); } return url.href; } async function installSeriesCover(mangaDir, coverUrl, http) { const files = await fsp.readdir(mangaDir); if (files.some(coverFile)) return { status: 'existing' }; const response = await http.get(validateCoverUrl(coverUrl)); const data = Buffer.from(await response.arrayBuffer()); if (data.length > 10 * 1024 * 1024) throw new Error('Series cover exceeds 10 MiB'); const ext = imageExtension(data); if (!ext) throw new Error('Series cover is not a supported image'); const temp = path.join(mangaDir, `.cover-mcscraper-${token()}.${ext}`); const target = path.join(mangaDir, `cover.${ext}`); await fsp.writeFile(temp, data, { mode: 0o644 }); try { await renameRetry(temp, target); } catch (error) { await fsp.rm(temp, { force: true }); throw error; } return { status: 'installed', file: path.basename(target) }; } async function stageChapter(bookDir, chapter, http, onPage, priorMeta = {}) { // Full manifest fingerprint prevents reusing a page from a changed release. // Store only the hash, never image URLs/headers (which may contain signatures). const identity = digest([chapterFolder(chapter), chapter.source, chapter.pages.map(p => [p.url || path.resolve(p.path), p.tags || []])]); const stage = path.join(bookDir, `.staging-mcscraper-resume-${identity}`); await fsp.mkdir(stage, { recursive: true, mode: 0o755 }); const dimensions = []; { for (let index = 0; index < chapter.pages.length; index++) { const tags = (chapter.pages[index].tags || []).map(tag => `_${tag}`).join(''); const prefix = `${String(index + 1).padStart(4, '0')}${tags}`; const prior = (await fsp.readdir(stage)).filter(name => name.startsWith(prefix + '.') && !name.endsWith('.tmp')); if (prior.length === 1) { const bytes = await fsp.readFile(path.join(stage, prior[0])); if (imageExtension(bytes) === path.extname(prior[0]).slice(1)) { dimensions.push(pageDimensions(sizeFromBuffer(bytes), chapter.pages[index].tags)); if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length }); continue; } } if (prior.length) throw new Error(`Partial page ${index + 1} requires review: invalid or ambiguous image`); const data = await pageBytes(chapter.pages[index], http); const ext = imageExtension(data); if (!ext) throw new Error(`Page ${index + 1} is not a supported image`); dimensions.push(pageDimensions(sizeFromBuffer(data), chapter.pages[index].tags)); const target = path.join(stage, `${prefix}.${ext}`); const pending = path.join(stage, `.${prefix}.tmp`); await fsp.writeFile(pending, data, { mode: 0o644 }); await renameRetry(pending, target); if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length }); } await fsp.writeFile(path.join(stage, 'chapter.json'), JSON.stringify({ ...priorMeta, title: chapter.title || '', source: chapter.source, // "added" stays the first arrival in this library; x-dates keeps source/release/ecosystem. ...publishedDates(priorMeta, { ...(chapter.dates || {}), added: chapter.added }), priority: Number(chapter.priority || 0), extra: !!chapter.extra, ...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId, sourceNumber: chapter.sourceNumber ?? null, ...(chapter.releaseKind ? { releaseKind: chapter.releaseKind } : {}), ...(chapter.volume ? { volume: chapter.volume } : {}) } : {}), ...directionFields([priorMeta, chapter], dimensions), }, null, 2) + '\n', { mode: 0o644 }); // A process interruption during a write can leave a hidden pending file. for (const name of await fsp.readdir(stage)) { if (/^\.\d{4,5}(?:_[a-z0-9]+)*\.tmp$/.test(name)) await fsp.rm(path.join(stage, name), { force: true }); } return { stage, dimensions }; } } async function publishStaged(stage, target, replace) { if (!(await exists(target))) return renameRetry(stage, target); if (!replace) { await fsp.rm(stage, { recursive: true, force: true }); return false; } const old = path.join(path.dirname(target), `.old-mcscraper-${token()}`); await renameRetry(target, old); try { await renameRetry(stage, target); } catch (error) { await renameRetry(old, target); throw error; } await fsp.rm(old, { recursive: true, force: true }); return true; } async function publishManifest(manifest, options = {}) { validateManifest(manifest); const libraryRoot = path.resolve(options.libraryRoot || manifest.libraryRoot || 'library'); await fsp.mkdir(libraryRoot, { recursive: true, mode: 0o755 }); await cleanupOwnTemps(libraryRoot); const sourceUrls = new Set(manifest.series.sources.map(source => source.url)); let mangaDir = options.mangaDir ? assertInside(libraryRoot, options.mangaDir) : await findManga(libraryRoot, sourceUrls); if (mangaDir && !(await exists(mangaDir))) throw new Error(`Target manga folder does not exist: ${mangaDir}`); if (mangaDir && path.basename(mangaDir).startsWith('.')) throw new Error('Target manga folder must be published'); const isNew = !mangaDir; const finalName = cleanPart(manifest.series.folder || manifest.series.title); const finalDir = assertInside(libraryRoot, path.join(libraryRoot, finalName)); if (!mangaDir && await exists(finalDir)) { const existing = await readMetadata(path.join(finalDir, 'manga.json')); const incomingIdentity = normalizeWorkIdentity(manifest.series.metadata?.['x-work-identity']); const existingIdentity = normalizeWorkIdentity(existing['x-work-identity']); if (incomingIdentity && existingIdentity && incomingIdentity.kind === existingIdentity.kind && incomingIdentity.value === existingIdentity.value) { throw new Error(`Work identity matches the existing “${existing.title || finalName}” folder, but its source association was not resolved. Review the existing manga target; no files were overwritten.`); } throw new Error(`A library folder named “${finalName}” already exists, but this download could not be safely matched to it. No files were overwritten. Choose the existing manga as the target or review its work identity before syncing.`); } mangaDir = mangaDir || path.join(libraryRoot, `.staging-mcscraper-resume-${digest([finalName, [...sourceUrls].sort()])}`); await fsp.mkdir(mangaDir, { recursive: true, mode: 0o755 }); const http = options.httpClient || new SlowHttp(options.http || manifest.http || {}); const results = []; const dimensions = []; let cover = null; const prior = await readMetadata(path.join(mangaDir, 'manga.json')); const owned = manifest.series.metadata || {}; // Validate identity before fetching or exposing any chapter pages. const merged = mergeImportedMetadata(prior, owned, manifest.series.title); try { for (const chapter of manifest.chapters) { const bookNumber = Number(chapter.book?.number || 1); if (!Number.isInteger(bookNumber) || bookNumber < 1) throw new Error(`Invalid book number for chapter ${chapter.number}`); const bookDir = await bookDirectory(mangaDir, bookNumber, chapter.book?.title); await fsp.mkdir(bookDir, { recursive: true, mode: 0o755 }); await cleanupOwnTemps(bookDir); if (chapter.book?.title) { const file = path.join(bookDir, 'book.json'), priorBook = await readMetadata(file); await writeJsonAtomic(file, { ...priorBook, title: chapter.book.title, ...(chapter.book.direction || chapter.book.directionOverride ? directionFields([priorBook, chapter.book], []) : {}) }); } const folder = chapterFolder(chapter); const target = path.join(bookDir, folder); if (await exists(target) && !options.replace) { dimensions.push(...await publishedDimensions(target)); results.push({ chapter: chapter.number, status: 'skipped' }); continue; } const priorChapter = await readMetadata(path.join(target, 'chapter.json')); const staged = await stageChapter(bookDir, chapter, http, options.onPage, priorChapter); dimensions.push(...staged.dimensions); await publishStaged(staged.stage, target, !!options.replace); results.push({ chapter: chapter.number, status: 'published', folder }); } const sources = [...(prior.sources || [])]; for (const source of manifest.series.sources) { const index = sources.findIndex(item => item.url === source.url); if (index < 0) sources.push(source); else sources[index] = { ...sources[index], ...source }; } const priorScraper = prior['x-scraper'] || {}; const nextScraper = { ...priorScraper, strategy: priorScraper.strategy || 'flat', ...(manifest.series.scraper || {}) }; await writeJsonAtomic(path.join(mangaDir, 'manga.json'), { ...merged, ...directionFields([prior, merged, manifest.series], dimensions), sources, 'x-scraper': nextScraper, }); if (manifest.series.coverUrl && !manifest.series.coverAttempted) { manifest.series.coverAttempted = true; try { cover = await installSeriesCover(mangaDir, manifest.series.coverUrl, http); } catch (error) { cover = { status: 'error', message: error.message }; console.warn(`[publisher] cover unavailable for ${manifest.series.title}: ${error.message}`); } } if (isNew) await renameRetry(mangaDir, finalDir); return { mangaDir: isNew ? finalDir : mangaDir, results, cover }; } catch (error) { // Preserve hidden validated pages on pause, transient failure or shutdown. // Publication still occurs only after the whole chapter validates. throw error; } } module.exports = { publishManifest, validateManifest, cleanupOwnTemps, installSeriesCover, validateCoverUrl, writeJsonAtomic };