Consolidate MangaCircus app, scraper, and Compose baseline
This commit is contained in:
Maashkinoozhe committed
1 parent
cfb2d79103
commit
eef4ffa07c
103 files changed
+9316
-172
No files matched your search
@@ -0,0 +1,279 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const fsp = fs.promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { SlowHttp } = require('./http');
|
||||
const { imageExtension } = require('./images');
|
||||
const { cleanPart, chapterFolder, assertInside } = require('./names');
|
||||
const { sizeFromBuffer, imageSize } = require('../../library');
|
||||
const { directionFields } = require('../../reading-direction');
|
||||
const { mergeImportedMetadata } = require('./metadata-merge');
|
||||
|
||||
const token = () => crypto.randomBytes(5).toString('hex');
|
||||
// Staging is a recovery asset, not disposable garbage. Legacy stages without an
|
||||
// identity must be reviewed manually; never infer a chapter from page filenames.
|
||||
const ownTemp = name => /^\.old-mcscraper-[a-f0-9]+$/.test(name);
|
||||
const digest = value => crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
||||
const coverFile = name => /^cover\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name);
|
||||
|
||||
async function exists(file) {
|
||||
try { await fsp.access(file); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
async function readMetadata(file) {
|
||||
try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^\uFEFF/, '')); }
|
||||
catch (error) { if (error.code === 'ENOENT') return {}; throw error; }
|
||||
}
|
||||
|
||||
function pageDimensions(size, tags = []) {
|
||||
const rotation = tags.find(tag => /^rot(90|180|270)$/.test(tag));
|
||||
return { w: size?.w || 0, h: size?.h || 0, r: rotation ? Number(rotation.slice(3)) : 0 };
|
||||
}
|
||||
|
||||
async function publishedDimensions(folder) {
|
||||
const pages = [];
|
||||
for (const name of (await fsp.readdir(folder)).sort()) {
|
||||
if (!/\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(name)) continue;
|
||||
const tags = name.slice(0, name.lastIndexOf('.')).split('_').slice(1);
|
||||
pages.push(pageDimensions(await imageSize(path.join(folder, name)).catch(() => null), tags));
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
async function renameRetry(from, to, tries = 5) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try { await fsp.rename(from, to); return; } catch (error) {
|
||||
if (attempt >= tries || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
||||
await new Promise(resolve => setTimeout(resolve, 200 * (attempt + 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function writeJsonAtomic(file, value) {
|
||||
const temp = path.join(path.dirname(file), `.${path.basename(file)}.tmp`);
|
||||
await fsp.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o644 });
|
||||
if (!(await exists(file))) return renameRetry(temp, file);
|
||||
try {
|
||||
await renameRetry(temp, file);
|
||||
} catch (error) {
|
||||
if (!['EPERM', 'EBUSY', 'EACCES', 'EEXIST'].includes(error.code)) throw error;
|
||||
const old = path.join(path.dirname(file), `.old-mcscraper-${token()}`);
|
||||
await renameRetry(file, old);
|
||||
try { await renameRetry(temp, file); } catch (moveError) { await renameRetry(old, file); throw moveError; }
|
||||
await fsp.rm(old, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanupOwnTemps(folder, maxAgeMs = 6 * 60 * 60 * 1000) {
|
||||
if (!(await exists(folder))) return;
|
||||
const now = Date.now();
|
||||
for (const entry of await fsp.readdir(folder, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || !ownTemp(entry.name)) continue;
|
||||
const target = path.join(folder, entry.name);
|
||||
const stat = await fsp.stat(target);
|
||||
if (now - stat.mtimeMs > maxAgeMs) await fsp.rm(target, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function findManga(libraryRoot, sourceUrls) {
|
||||
if (!(await exists(libraryRoot))) return null;
|
||||
for (const entry of await fsp.readdir(libraryRoot, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || entry.name.startsWith('.')) continue;
|
||||
const file = path.join(libraryRoot, entry.name, 'manga.json');
|
||||
try {
|
||||
const json = JSON.parse(await fsp.readFile(file, 'utf8'));
|
||||
if ((json.sources || []).some(source => sourceUrls.has(source.url))) return path.join(libraryRoot, entry.name);
|
||||
} catch {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function validateManifest(manifest) {
|
||||
if (!manifest || manifest.version !== 1) throw new Error('Manifest version must be 1');
|
||||
if (!manifest.series?.title) throw new Error('series.title is required');
|
||||
if (!['rtl', 'ltr', 'ttb'].includes(manifest.series.direction)) throw new Error('series.direction must be rtl, ltr, or ttb');
|
||||
if (!Array.isArray(manifest.series.sources) || !manifest.series.sources.length) throw new Error('series.sources is required');
|
||||
if (!Array.isArray(manifest.chapters) || !manifest.chapters.length) throw new Error('At least one chapter is required');
|
||||
for (const chapter of manifest.chapters) {
|
||||
if (chapter.number == null || chapter.number === '' || !chapter.source?.name || !chapter.source?.url) throw new Error('Every chapter needs number and source {name,url}');
|
||||
if (!Array.isArray(chapter.pages) || !chapter.pages.length) throw new Error(`Chapter ${chapter.number} has no pages`);
|
||||
for (const page of chapter.pages) {
|
||||
if (!!page.url === !!page.path) throw new Error('Every page must have exactly one of url or path');
|
||||
for (const tag of page.tags || []) if (!/^[a-z0-9]+$/.test(tag)) throw new Error(`Invalid page tag: ${tag}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function pageBytes(page, http) {
|
||||
if (page.path) return fsp.readFile(path.resolve(page.path));
|
||||
const response = await http.get(page.url, { headers: page.headers || {} });
|
||||
return Buffer.from(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
function validateCoverUrl(value) {
|
||||
const url = new URL(value);
|
||||
if (url.protocol !== 'https:' || !(url.hostname === 'anilist.co' || url.hostname.endsWith('.anilist.co'))) {
|
||||
throw new Error('Series cover must use the AniList HTTPS image host');
|
||||
}
|
||||
return url.href;
|
||||
}
|
||||
|
||||
async function installSeriesCover(mangaDir, coverUrl, http) {
|
||||
const files = await fsp.readdir(mangaDir);
|
||||
if (files.some(coverFile)) return { status: 'existing' };
|
||||
const response = await http.get(validateCoverUrl(coverUrl));
|
||||
const data = Buffer.from(await response.arrayBuffer());
|
||||
if (data.length > 10 * 1024 * 1024) throw new Error('Series cover exceeds 10 MiB');
|
||||
const ext = imageExtension(data);
|
||||
if (!ext) throw new Error('Series cover is not a supported image');
|
||||
const temp = path.join(mangaDir, `.cover-mcscraper-${token()}.${ext}`);
|
||||
const target = path.join(mangaDir, `cover.${ext}`);
|
||||
await fsp.writeFile(temp, data, { mode: 0o644 });
|
||||
try { await renameRetry(temp, target); } catch (error) { await fsp.rm(temp, { force: true }); throw error; }
|
||||
return { status: 'installed', file: path.basename(target) };
|
||||
}
|
||||
|
||||
async function stageChapter(bookDir, chapter, http, onPage, priorMeta = {}) {
|
||||
// Full manifest fingerprint prevents reusing a page from a changed release.
|
||||
// Store only the hash, never image URLs/headers (which may contain signatures).
|
||||
const identity = digest([chapterFolder(chapter), chapter.source, chapter.pages.map(p => [p.url || path.resolve(p.path), p.tags || []])]);
|
||||
const stage = path.join(bookDir, `.staging-mcscraper-resume-${identity}`);
|
||||
await fsp.mkdir(stage, { recursive: true, mode: 0o755 });
|
||||
const dimensions = [];
|
||||
{
|
||||
for (let index = 0; index < chapter.pages.length; index++) {
|
||||
const tags = (chapter.pages[index].tags || []).map(tag => `_${tag}`).join('');
|
||||
const prefix = `${String(index + 1).padStart(4, '0')}${tags}`;
|
||||
const prior = (await fsp.readdir(stage)).filter(name => name.startsWith(prefix + '.') && !name.endsWith('.tmp'));
|
||||
if (prior.length === 1) {
|
||||
const bytes = await fsp.readFile(path.join(stage, prior[0]));
|
||||
if (imageExtension(bytes) === path.extname(prior[0]).slice(1)) {
|
||||
dimensions.push(pageDimensions(sizeFromBuffer(bytes), chapter.pages[index].tags));
|
||||
if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (prior.length) throw new Error(`Partial page ${index + 1} requires review: invalid or ambiguous image`);
|
||||
const data = await pageBytes(chapter.pages[index], http);
|
||||
const ext = imageExtension(data);
|
||||
if (!ext) throw new Error(`Page ${index + 1} is not a supported image`);
|
||||
dimensions.push(pageDimensions(sizeFromBuffer(data), chapter.pages[index].tags));
|
||||
const target = path.join(stage, `${prefix}.${ext}`);
|
||||
const pending = path.join(stage, `.${prefix}.tmp`);
|
||||
await fsp.writeFile(pending, data, { mode: 0o644 });
|
||||
await renameRetry(pending, target);
|
||||
if (onPage) await onPage({ completed: index + 1, total: chapter.pages.length });
|
||||
}
|
||||
await fsp.writeFile(path.join(stage, 'chapter.json'), JSON.stringify({
|
||||
...priorMeta,
|
||||
title: chapter.title || '', source: chapter.source,
|
||||
added: chapter.added || new Date().toISOString(),
|
||||
priority: Number(chapter.priority || 0), extra: !!chapter.extra,
|
||||
...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId, sourceNumber: chapter.sourceNumber ?? null } : {}),
|
||||
...directionFields([priorMeta, chapter], dimensions),
|
||||
}, null, 2) + '\n', { mode: 0o644 });
|
||||
// A process interruption during a write can leave a hidden pending file.
|
||||
for (const name of await fsp.readdir(stage)) {
|
||||
if (/^\.\d{4,5}(?:_[a-z0-9]+)*\.tmp$/.test(name)) await fsp.rm(path.join(stage, name), { force: true });
|
||||
}
|
||||
return { stage, dimensions };
|
||||
}
|
||||
}
|
||||
|
||||
async function publishStaged(stage, target, replace) {
|
||||
if (!(await exists(target))) return renameRetry(stage, target);
|
||||
if (!replace) { await fsp.rm(stage, { recursive: true, force: true }); return false; }
|
||||
const old = path.join(path.dirname(target), `.old-mcscraper-${token()}`);
|
||||
await renameRetry(target, old);
|
||||
try { await renameRetry(stage, target); } catch (error) { await renameRetry(old, target); throw error; }
|
||||
await fsp.rm(old, { recursive: true, force: true });
|
||||
return true;
|
||||
}
|
||||
|
||||
async function publishManifest(manifest, options = {}) {
|
||||
validateManifest(manifest);
|
||||
const libraryRoot = path.resolve(options.libraryRoot || manifest.libraryRoot || 'library');
|
||||
await fsp.mkdir(libraryRoot, { recursive: true, mode: 0o755 });
|
||||
await cleanupOwnTemps(libraryRoot);
|
||||
const sourceUrls = new Set(manifest.series.sources.map(source => source.url));
|
||||
let mangaDir = options.mangaDir ? assertInside(libraryRoot, options.mangaDir) : await findManga(libraryRoot, sourceUrls);
|
||||
if (mangaDir && !(await exists(mangaDir))) throw new Error(`Target manga folder does not exist: ${mangaDir}`);
|
||||
if (mangaDir && path.basename(mangaDir).startsWith('.')) throw new Error('Target manga folder must be published');
|
||||
const isNew = !mangaDir;
|
||||
const finalName = cleanPart(manifest.series.folder || manifest.series.title);
|
||||
const finalDir = assertInside(libraryRoot, path.join(libraryRoot, finalName));
|
||||
if (!mangaDir && await exists(finalDir)) throw new Error(`Folder already belongs to another work: ${finalDir}`);
|
||||
mangaDir = mangaDir || path.join(libraryRoot, `.staging-mcscraper-resume-${digest([finalName, [...sourceUrls].sort()])}`);
|
||||
await fsp.mkdir(mangaDir, { recursive: true, mode: 0o755 });
|
||||
const http = options.httpClient || new SlowHttp(options.http || manifest.http || {});
|
||||
const results = [];
|
||||
const dimensions = [];
|
||||
let cover = null;
|
||||
const prior = await readMetadata(path.join(mangaDir, 'manga.json'));
|
||||
const owned = manifest.series.metadata || {};
|
||||
// Validate identity before fetching or exposing any chapter pages.
|
||||
const merged = mergeImportedMetadata(prior, owned, manifest.series.title);
|
||||
|
||||
try {
|
||||
for (const chapter of manifest.chapters) {
|
||||
const bookNumber = Number(chapter.book?.number || 1);
|
||||
if (!Number.isInteger(bookNumber) || bookNumber < 1) throw new Error(`Invalid book number for chapter ${chapter.number}`);
|
||||
const bookName = `v${String(bookNumber).padStart(2, '0')}${chapter.book?.title ? ` ${cleanPart(chapter.book.title)}` : ''}`;
|
||||
const bookDir = path.join(mangaDir, bookName);
|
||||
await fsp.mkdir(bookDir, { recursive: true, mode: 0o755 });
|
||||
await cleanupOwnTemps(bookDir);
|
||||
if (chapter.book?.title) {
|
||||
const file = path.join(bookDir, 'book.json'), priorBook = await readMetadata(file);
|
||||
await writeJsonAtomic(file, { ...priorBook, title: chapter.book.title,
|
||||
...(chapter.book.direction || chapter.book.directionOverride
|
||||
? directionFields([priorBook, chapter.book], []) : {}) });
|
||||
}
|
||||
const folder = chapterFolder(chapter);
|
||||
const target = path.join(bookDir, folder);
|
||||
if (await exists(target) && !options.replace) {
|
||||
dimensions.push(...await publishedDimensions(target));
|
||||
results.push({ chapter: chapter.number, status: 'skipped' }); continue;
|
||||
}
|
||||
const priorChapter = await readMetadata(path.join(target, 'chapter.json'));
|
||||
const staged = await stageChapter(bookDir, chapter, http, options.onPage, priorChapter);
|
||||
dimensions.push(...staged.dimensions);
|
||||
await publishStaged(staged.stage, target, !!options.replace);
|
||||
results.push({ chapter: chapter.number, status: 'published', folder });
|
||||
}
|
||||
|
||||
const sources = [...(prior.sources || [])];
|
||||
for (const source of manifest.series.sources) {
|
||||
const index = sources.findIndex(item => item.url === source.url);
|
||||
if (index < 0) sources.push(source); else sources[index] = { ...sources[index], ...source };
|
||||
}
|
||||
const priorScraper = prior['x-scraper'] || {};
|
||||
const nextScraper = { ...priorScraper, strategy: priorScraper.strategy || 'flat', ...(manifest.series.scraper || {}) };
|
||||
await writeJsonAtomic(path.join(mangaDir, 'manga.json'), {
|
||||
...merged,
|
||||
...directionFields([prior, merged, manifest.series], dimensions),
|
||||
sources,
|
||||
'x-scraper': nextScraper,
|
||||
});
|
||||
|
||||
if (manifest.series.coverUrl && !manifest.series.coverAttempted) {
|
||||
manifest.series.coverAttempted = true;
|
||||
try {
|
||||
cover = await installSeriesCover(mangaDir, manifest.series.coverUrl, http);
|
||||
} catch (error) {
|
||||
cover = { status: 'error', message: error.message };
|
||||
console.warn(`[publisher] cover unavailable for ${manifest.series.title}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (isNew) await renameRetry(mangaDir, finalDir);
|
||||
return { mangaDir: isNew ? finalDir : mangaDir, results, cover };
|
||||
} catch (error) {
|
||||
// Preserve hidden validated pages on pause, transient failure or shutdown.
|
||||
// Publication still occurs only after the whole chapter validates.
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { publishManifest, validateManifest, cleanupOwnTemps, installSeriesCover, validateCoverUrl, writeJsonAtomic };
|
||||
Reference in new issue
Block a user