302 lines
16 KiB
JavaScript
302 lines
16 KiB
JavaScript
'use strict';
|
|
|
|
const fs = require('fs');
|
|
const fsp = fs.promises;
|
|
const path = require('path');
|
|
const crypto = require('crypto');
|
|
const { SlowHttp } = require('./http');
|
|
const { imageExtension } = require('./images');
|
|
const { cleanPart, chapterFolder, assertInside } = require('./names');
|
|
const { sizeFromBuffer, imageSize } = require('../../library');
|
|
const { directionFields } = require('../../reading-direction');
|
|
const { mergeImportedMetadata } = require('./metadata-merge');
|
|
const { normalizeWorkIdentity } = require('./work-identity');
|
|
const { parseKey } = require('../../chapter-identity');
|
|
|
|
const token = () => crypto.randomBytes(5).toString('hex');
|
|
// Staging is a recovery asset, not disposable garbage. Legacy stages without an
|
|
// identity must be reviewed manually; never infer a chapter from page filenames.
|
|
const ownTemp = name => /^\.old-mcscraper-[a-f0-9]+$/.test(name);
|
|
const digest = value => crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
const coverFile = name => /^cover\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name);
|
|
|
|
async function exists(file) {
|
|
try { await fsp.access(file); return true; } catch { return false; }
|
|
}
|
|
|
|
async function readMetadata(file) {
|
|
try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^\uFEFF/, '')); }
|
|
catch (error) { if (error.code === 'ENOENT') return {}; throw error; }
|
|
}
|
|
|
|
function pageDimensions(size, tags = []) {
|
|
const rotation = tags.find(tag => /^rot(90|180|270)$/.test(tag));
|
|
return { w: size?.w || 0, h: size?.h || 0, r: rotation ? Number(rotation.slice(3)) : 0 };
|
|
}
|
|
|
|
async function publishedDimensions(folder) {
|
|
const pages = [];
|
|
for (const name of (await fsp.readdir(folder)).sort()) {
|
|
if (!/\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name)) continue;
|
|
const tags = name.slice(0, name.lastIndexOf('.')).split('_').slice(1);
|
|
pages.push(pageDimensions(await imageSize(path.join(folder, name)).catch(() => null), tags));
|
|
}
|
|
return pages;
|
|
}
|
|
|
|
async function renameRetry(from, to, tries = 5) {
|
|
for (let attempt = 0; ; attempt++) {
|
|
try { await fsp.rename(from, to); return; } catch (error) {
|
|
if (attempt >= tries || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
|
await new Promise(resolve => setTimeout(resolve, 200 * (attempt + 1)));
|
|
}
|
|
}
|
|
}
|
|
|
|
async function writeJsonAtomic(file, value) {
|
|
const temp = path.join(path.dirname(file), `.${path.basename(file)}.tmp`);
|
|
await fsp.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o644 });
|
|
if (!(await exists(file))) return renameRetry(temp, file);
|
|
try {
|
|
await renameRetry(temp, file);
|
|
} catch (error) {
|
|
if (!['EPERM', 'EBUSY', 'EACCES', 'EEXIST'].includes(error.code)) throw error;
|
|
const old = path.join(path.dirname(file), `.old-mcscraper-${token()}`);
|
|
await renameRetry(file, old);
|
|
try { await renameRetry(temp, file); } catch (moveError) { await renameRetry(old, file); throw moveError; }
|
|
await fsp.rm(old, { force: true });
|
|
}
|
|
}
|
|
|
|
async function cleanupOwnTemps(folder, maxAgeMs = 6 * 60 * 60 * 1000) {
|
|
if (!(await exists(folder))) return;
|
|
const now = Date.now();
|
|
for (const entry of await fsp.readdir(folder, { withFileTypes: true })) {
|
|
if (!entry.isDirectory() || !ownTemp(entry.name)) continue;
|
|
const target = path.join(folder, entry.name);
|
|
const stat = await fsp.stat(target);
|
|
if (now - stat.mtimeMs > maxAgeMs) await fsp.rm(target, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function findManga(libraryRoot, sourceUrls) {
|
|
if (!(await exists(libraryRoot))) return null;
|
|
for (const entry of await fsp.readdir(libraryRoot, { withFileTypes: true })) {
|
|
if (!entry.isDirectory() || entry.name.startsWith('.')) continue;
|
|
const file = path.join(libraryRoot, entry.name, 'manga.json');
|
|
try {
|
|
const json = JSON.parse(await fsp.readFile(file, 'utf8'));
|
|
if ((json.sources || []).some(source => sourceUrls.has(source.url))) return path.join(libraryRoot, entry.name);
|
|
} catch {}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function bookDirectory(mangaDir, bookNumber, bookTitle) {
|
|
const names = (await fsp.readdir(mangaDir, { withFileTypes: true }))
|
|
.filter(entry => entry.isDirectory() && parseKey(entry.name)?.key === String(bookNumber))
|
|
.map(entry => entry.name);
|
|
if (names.length > 1) throw new Error(`Ambiguous library book ${bookNumber}: ${names.join(', ')}`);
|
|
const defaultName = `v${String(bookNumber).padStart(2, '0')}`;
|
|
return path.join(mangaDir, names[0] || (bookTitle ? `${defaultName} ${cleanPart(bookTitle)}` : defaultName));
|
|
}
|
|
|
|
function validateManifest(manifest) {
|
|
if (!manifest || manifest.version !== 1) throw new Error('Manifest version must be 1');
|
|
if (!manifest.series?.title) throw new Error('series.title is required');
|
|
if (!['rtl', 'ltr', 'ttb'].includes(manifest.series.direction)) throw new Error('series.direction must be rtl, ltr, or ttb');
|
|
if (!Array.isArray(manifest.series.sources) || !manifest.series.sources.length) throw new Error('series.sources is required');
|
|
if (!Array.isArray(manifest.chapters) || !manifest.chapters.length) throw new Error('At least one chapter is required');
|
|
for (const chapter of manifest.chapters) {
|
|
if (chapter.number == null || chapter.number === '' || !chapter.source?.name || !chapter.source?.url) throw new Error('Every chapter needs number and source {name,url}');
|
|
if (!Array.isArray(chapter.pages) || !chapter.pages.length) throw new Error(`Chapter ${chapter.number} has no pages`);
|
|
for (const page of chapter.pages) {
|
|
if (!!page.url === !!page.path) throw new Error('Every page must have exactly one of url or path');
|
|
for (const tag of page.tags || []) if (!/^[a-z0-9]+$/.test(tag)) throw new Error(`Invalid page tag: ${tag}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function pageBytes(page, http) {
|
|
if (page.path) return fsp.readFile(path.resolve(page.path));
|
|
const response = await http.get(page.url, { headers: page.headers || {} });
|
|
return Buffer.from(await response.arrayBuffer());
|
|
}
|
|
|
|
function validateCoverUrl(value) {
|
|
const url = new URL(value);
|
|
if (url.protocol !== 'https:' || !(url.hostname === 'anilist.co' || url.hostname.endsWith('.anilist.co'))) {
|
|
throw new Error('Series cover must use the AniList HTTPS image host');
|
|
}
|
|
return url.href;
|
|
}
|
|
|
|
async function installSeriesCover(mangaDir, coverUrl, http) {
|
|
const files = await fsp.readdir(mangaDir);
|
|
if (files.some(coverFile)) return { status: 'existing' };
|
|
const response = await http.get(validateCoverUrl(coverUrl));
|
|
const data = Buffer.from(await response.arrayBuffer());
|
|
if (data.length > 10 * 1024 * 1024) throw new Error('Series cover exceeds 10 MiB');
|
|
const ext = imageExtension(data);
|
|
if (!ext) throw new Error('Series cover is not a supported image');
|
|
const temp = path.join(mangaDir, `.cover-mcscraper-${token()}.${ext}`);
|
|
const target = path.join(mangaDir, `cover.${ext}`);
|
|
await fsp.writeFile(temp, data, { mode: 0o644 });
|
|
try { await renameRetry(temp, target); } catch (error) { await fsp.rm(temp, { force: true }); throw error; }
|
|
return { status: 'installed', file: path.basename(target) };
|
|
}
|
|
|
|
async function stageChapter(bookDir, chapter, http, onPage, priorMeta = {}) {
|
|
// Full manifest fingerprint prevents reusing a page from a changed release.
|
|
// Store only the hash, never image URLs/headers (which may contain signatures).
|
|
const identity = digest([chapterFolder(chapter), chapter.source, chapter.pages.map(p => [p.url || path.resolve(p.path), p.tags || []])]);
|
|
const stage = path.join(bookDir, `.staging-mcscraper-resume-${identity}`);
|
|
await fsp.mkdir(stage, { recursive: true, mode: 0o755 });
|
|
const dimensions = [];
|
|
{
|
|
for (let index = 0; index < chapter.pages.length; index++) {
|
|
const tags = (chapter.pages[index].tags || []).map(tag => `_${tag}`).join('');
|
|
const prefix = `${String(index + 1).padStart(4, '0')}${tags}`;
|
|
const prior = (await fsp.readdir(stage)).filter(name => name.startsWith(prefix + '.') && !name.endsWith('.tmp'));
|
|
if (prior.length === 1) {
|
|
const bytes = await fsp.readFile(path.join(stage, prior[0]));
|
|
if (imageExtension(bytes) === path.extname(prior[0]).slice(1)) {
|
|
dimensions.push(pageDimensions(sizeFromBuffer(bytes), chapter.pages[index].tags));
|
|
if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length });
|
|
continue;
|
|
}
|
|
}
|
|
if (prior.length) throw new Error(`Partial page ${index + 1} requires review: invalid or ambiguous image`);
|
|
const data = await pageBytes(chapter.pages[index], http);
|
|
const ext = imageExtension(data);
|
|
if (!ext) throw new Error(`Page ${index + 1} is not a supported image`);
|
|
dimensions.push(pageDimensions(sizeFromBuffer(data), chapter.pages[index].tags));
|
|
const target = path.join(stage, `${prefix}.${ext}`);
|
|
const pending = path.join(stage, `.${prefix}.tmp`);
|
|
await fsp.writeFile(pending, data, { mode: 0o644 });
|
|
await renameRetry(pending, target);
|
|
if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length });
|
|
}
|
|
await fsp.writeFile(path.join(stage, 'chapter.json'), JSON.stringify({
|
|
...priorMeta,
|
|
title: chapter.title || '', source: chapter.source,
|
|
added: chapter.added || new Date().toISOString(),
|
|
priority: Number(chapter.priority || 0), extra: !!chapter.extra,
|
|
...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId,
|
|
sourceNumber: chapter.sourceNumber ?? null,
|
|
...(chapter.releaseKind ? { releaseKind: chapter.releaseKind } : {}),
|
|
...(chapter.volume ? { volume: chapter.volume } : {}) } : {}),
|
|
...directionFields([priorMeta, chapter], dimensions),
|
|
}, null, 2) + '\n', { mode: 0o644 });
|
|
// A process interruption during a write can leave a hidden pending file.
|
|
for (const name of await fsp.readdir(stage)) {
|
|
if (/^\.\d{4,5}(?:_[a-z0-9]+)*\.tmp$/.test(name)) await fsp.rm(path.join(stage, name), { force: true });
|
|
}
|
|
return { stage, dimensions };
|
|
}
|
|
}
|
|
|
|
async function publishStaged(stage, target, replace) {
|
|
if (!(await exists(target))) return renameRetry(stage, target);
|
|
if (!replace) { await fsp.rm(stage, { recursive: true, force: true }); return false; }
|
|
const old = path.join(path.dirname(target), `.old-mcscraper-${token()}`);
|
|
await renameRetry(target, old);
|
|
try { await renameRetry(stage, target); } catch (error) { await renameRetry(old, target); throw error; }
|
|
await fsp.rm(old, { recursive: true, force: true });
|
|
return true;
|
|
}
|
|
|
|
async function publishManifest(manifest, options = {}) {
|
|
validateManifest(manifest);
|
|
const libraryRoot = path.resolve(options.libraryRoot || manifest.libraryRoot || 'library');
|
|
await fsp.mkdir(libraryRoot, { recursive: true, mode: 0o755 });
|
|
await cleanupOwnTemps(libraryRoot);
|
|
const sourceUrls = new Set(manifest.series.sources.map(source => source.url));
|
|
let mangaDir = options.mangaDir ? assertInside(libraryRoot, options.mangaDir) : await findManga(libraryRoot, sourceUrls);
|
|
if (mangaDir && !(await exists(mangaDir))) throw new Error(`Target manga folder does not exist: ${mangaDir}`);
|
|
if (mangaDir && path.basename(mangaDir).startsWith('.')) throw new Error('Target manga folder must be published');
|
|
const isNew = !mangaDir;
|
|
const finalName = cleanPart(manifest.series.folder || manifest.series.title);
|
|
const finalDir = assertInside(libraryRoot, path.join(libraryRoot, finalName));
|
|
if (!mangaDir && await exists(finalDir)) {
|
|
const existing = await readMetadata(path.join(finalDir, 'manga.json'));
|
|
const incomingIdentity = normalizeWorkIdentity(manifest.series.metadata?.['x-work-identity']);
|
|
const existingIdentity = normalizeWorkIdentity(existing['x-work-identity']);
|
|
if (incomingIdentity && existingIdentity && incomingIdentity.kind === existingIdentity.kind
|
|
&& incomingIdentity.value === existingIdentity.value) {
|
|
throw new Error(`Work identity matches the existing “${existing.title || finalName}” folder, but its source association was not resolved. Review the existing manga target; no files were overwritten.`);
|
|
}
|
|
throw new Error(`A library folder named “${finalName}” already exists, but this download could not be safely matched to it. No files were overwritten. Choose the existing manga as the target or review its work identity before syncing.`);
|
|
}
|
|
mangaDir = mangaDir || path.join(libraryRoot, `.staging-mcscraper-resume-${digest([finalName, [...sourceUrls].sort()])}`);
|
|
await fsp.mkdir(mangaDir, { recursive: true, mode: 0o755 });
|
|
const http = options.httpClient || new SlowHttp(options.http || manifest.http || {});
|
|
const results = [];
|
|
const dimensions = [];
|
|
let cover = null;
|
|
const prior = await readMetadata(path.join(mangaDir, 'manga.json'));
|
|
const owned = manifest.series.metadata || {};
|
|
// Validate identity before fetching or exposing any chapter pages.
|
|
const merged = mergeImportedMetadata(prior, owned, manifest.series.title);
|
|
|
|
try {
|
|
for (const chapter of manifest.chapters) {
|
|
const bookNumber = Number(chapter.book?.number || 1);
|
|
if (!Number.isInteger(bookNumber) || bookNumber < 1) throw new Error(`Invalid book number for chapter ${chapter.number}`);
|
|
const bookDir = await bookDirectory(mangaDir, bookNumber, chapter.book?.title);
|
|
await fsp.mkdir(bookDir, { recursive: true, mode: 0o755 });
|
|
await cleanupOwnTemps(bookDir);
|
|
if (chapter.book?.title) {
|
|
const file = path.join(bookDir, 'book.json'), priorBook = await readMetadata(file);
|
|
await writeJsonAtomic(file, { ...priorBook, title: chapter.book.title,
|
|
...(chapter.book.direction || chapter.book.directionOverride
|
|
? directionFields([priorBook, chapter.book], []) : {}) });
|
|
}
|
|
const folder = chapterFolder(chapter);
|
|
const target = path.join(bookDir, folder);
|
|
if (await exists(target) && !options.replace) {
|
|
dimensions.push(...await publishedDimensions(target));
|
|
results.push({ chapter: chapter.number, status: 'skipped' }); continue;
|
|
}
|
|
const priorChapter = await readMetadata(path.join(target, 'chapter.json'));
|
|
const staged = await stageChapter(bookDir, chapter, http, options.onPage, priorChapter);
|
|
dimensions.push(...staged.dimensions);
|
|
await publishStaged(staged.stage, target, !!options.replace);
|
|
results.push({ chapter: chapter.number, status: 'published', folder });
|
|
}
|
|
|
|
const sources = [...(prior.sources || [])];
|
|
for (const source of manifest.series.sources) {
|
|
const index = sources.findIndex(item => item.url === source.url);
|
|
if (index < 0) sources.push(source); else sources[index] = { ...sources[index], ...source };
|
|
}
|
|
const priorScraper = prior['x-scraper'] || {};
|
|
const nextScraper = { ...priorScraper, strategy: priorScraper.strategy || 'flat', ...(manifest.series.scraper || {}) };
|
|
await writeJsonAtomic(path.join(mangaDir, 'manga.json'), {
|
|
...merged,
|
|
...directionFields([prior, merged, manifest.series], dimensions),
|
|
sources,
|
|
'x-scraper': nextScraper,
|
|
});
|
|
|
|
if (manifest.series.coverUrl && !manifest.series.coverAttempted) {
|
|
manifest.series.coverAttempted = true;
|
|
try {
|
|
cover = await installSeriesCover(mangaDir, manifest.series.coverUrl, http);
|
|
} catch (error) {
|
|
cover = { status: 'error', message: error.message };
|
|
console.warn(`[publisher] cover unavailable for ${manifest.series.title}: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
if (isNew) await renameRetry(mangaDir, finalDir);
|
|
return { mangaDir: isNew ? finalDir : mangaDir, results, cover };
|
|
} catch (error) {
|
|
// Preserve hidden validated pages on pause, transient failure or shutdown.
|
|
// Publication still occurs only after the whole chapter validates.
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
module.exports = { publishManifest, validateManifest, cleanupOwnTemps, installSeriesCover, validateCoverUrl, writeJsonAtomic };
|