Consolidate MangaCircus app, scraper, and Compose baseline
This commit is contained in:
Maashkinoozhe committed
1 parent
cfb2d79103
commit
eef4ffa07c
103 files changed
+9316
-172
No files matched your search
@@ -0,0 +1,179 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { imageExtension } = require('./images');
|
||||
const { assertInside } = require('./names');
|
||||
|
||||
const PAGE = /^(\d{4,5})(?:_[a-z0-9]+)*\.(jpe?g|png|webp|gif|avif|bmp)$/i;
|
||||
const BOOK = /^(?:v|s|x)\d{2,3}(?: |$)/i;
|
||||
const CHAPTER = /^(?:c\d{4}(?:\.\d*[1-9])?|x\d{2})(?: |$)/i;
|
||||
const OWN_STAGE = /^\.staging-mcscraper-[a-f0-9]+$/;
|
||||
|
||||
async function exists(target) {
|
||||
try { await fs.access(target); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
async function renameRetry(from, to, tries = 5) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try { await fs.rename(from, to); return; } catch (error) {
|
||||
if (attempt >= tries || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
||||
await new Promise(resolve => setTimeout(resolve, 200 * (attempt + 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function writeJsonAtomic(file, value) {
|
||||
const temp = `${file}.migration.tmp`;
|
||||
const old = `${file}.migration.old`;
|
||||
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o644 });
|
||||
await fs.rm(old, { force: true });
|
||||
await renameRetry(file, old);
|
||||
try {
|
||||
await renameRetry(temp, file);
|
||||
} catch (error) {
|
||||
await renameRetry(old, file).catch(() => {});
|
||||
await fs.rm(temp, { force: true });
|
||||
throw error;
|
||||
}
|
||||
await fs.rm(old, { force: true }).catch(() => {});
|
||||
}
|
||||
|
||||
async function directories(folder) {
|
||||
return (await fs.readdir(folder, { withFileTypes: true })).filter(entry => entry.isDirectory());
|
||||
}
|
||||
|
||||
async function validatePage(file, expected) {
|
||||
const handle = await fs.open(file, 'r');
|
||||
try {
|
||||
const prefix = Buffer.alloc(16);
|
||||
const { bytesRead } = await handle.read(prefix, 0, prefix.length, 0);
|
||||
const actual = imageExtension(prefix.subarray(0, bytesRead));
|
||||
if (!actual) throw new Error(`Unsupported image bytes: ${file}`);
|
||||
const normalized = expected.toLowerCase() === 'jpeg' ? 'jpg' : expected.toLowerCase();
|
||||
if (actual !== normalized) throw new Error(`Image extension does not match bytes: ${file}`);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function validateChapter(folder) {
|
||||
const entries = await fs.readdir(folder, { withFileTypes: true });
|
||||
if (entries.some(entry => entry.isDirectory())) throw new Error(`Chapter contains a subdirectory: ${folder}`);
|
||||
const chapterFile = path.join(folder, 'chapter.json');
|
||||
const chapter = JSON.parse(await fs.readFile(chapterFile, 'utf8'));
|
||||
if (!chapter.source?.name || !chapter.source?.url) throw new Error(`Invalid chapter source: ${chapterFile}`);
|
||||
const pages = entries.filter(entry => entry.isFile() && entry.name !== 'chapter.json').map(entry => {
|
||||
const match = PAGE.exec(entry.name);
|
||||
if (!match) throw new Error(`Invalid chapter file: ${path.join(folder, entry.name)}`);
|
||||
return { name: entry.name, number: Number(match[1]), extension: match[2] };
|
||||
}).sort((a, b) => a.number - b.number);
|
||||
if (!pages.length) throw new Error(`Chapter has no pages: ${folder}`);
|
||||
for (let index = 0; index < pages.length; index++) {
|
||||
if (pages[index].number !== index + 1) throw new Error(`Non-contiguous pages in ${folder}`);
|
||||
await validatePage(path.join(folder, pages[index].name), pages[index].extension);
|
||||
}
|
||||
return pages.length;
|
||||
}
|
||||
|
||||
async function inspectLegacyWork({ libraryRoot, seriesId }) {
|
||||
const id = String(seriesId || '');
|
||||
if (!/^[A-Za-z0-9-]+$/.test(id)) throw new Error(`Unsafe legacy series id: ${id}`);
|
||||
const root = path.resolve(libraryRoot);
|
||||
const workRoot = assertInside(root, path.join(root, `.mcscraper-work-${id}`));
|
||||
const stateFile = path.join(workRoot, 'job.json');
|
||||
if (!(await exists(stateFile))) return null;
|
||||
const state = JSON.parse(await fs.readFile(stateFile, 'utf8'));
|
||||
if (state.publicationMode === 'incremental') return null;
|
||||
const mangaDirs = (await directories(workRoot)).filter(entry => !entry.name.startsWith('.'));
|
||||
if (!mangaDirs.length) return null;
|
||||
if (mangaDirs.length !== 1) throw new Error(`Legacy work ${id} must contain exactly one manga folder`);
|
||||
const mangaDir = path.join(workRoot, mangaDirs[0].name);
|
||||
const target = assertInside(root, path.join(root, mangaDirs[0].name));
|
||||
if (await exists(target)) throw new Error(`Cannot publish legacy work; target already exists: ${target}`);
|
||||
const meta = JSON.parse(await fs.readFile(path.join(mangaDir, 'manga.json'), 'utf8'));
|
||||
if (!meta.title || !Array.isArray(meta.sources) || !meta.sources.length) throw new Error(`Invalid legacy manga metadata: ${mangaDir}`);
|
||||
|
||||
let chapterCount = 0, pageCount = 0;
|
||||
const incompleteStages = [];
|
||||
for (const book of await directories(mangaDir)) {
|
||||
if (book.name.startsWith('.')) throw new Error(`Unexpected hidden folder remains in ${mangaDir}`);
|
||||
if (!BOOK.test(book.name)) throw new Error(`Invalid legacy book folder: ${book.name}`);
|
||||
const bookDir = path.join(mangaDir, book.name);
|
||||
for (const chapter of await directories(bookDir)) {
|
||||
if (chapter.name.startsWith('.')) {
|
||||
if (!OWN_STAGE.test(chapter.name)) throw new Error(`Unexpected hidden folder remains in ${bookDir}: ${chapter.name}`);
|
||||
incompleteStages.push(path.join(bookDir, chapter.name));
|
||||
continue;
|
||||
}
|
||||
if (!CHAPTER.test(chapter.name)) throw new Error(`Invalid legacy chapter folder: ${chapter.name}`);
|
||||
pageCount += await validateChapter(path.join(bookDir, chapter.name));
|
||||
chapterCount++;
|
||||
}
|
||||
}
|
||||
const completedCount = new Set(state.completed || []).size;
|
||||
if (!chapterCount || chapterCount !== completedCount) {
|
||||
throw new Error(`Legacy work ${id} has ${chapterCount} complete folders but ${completedCount} completed checkpoint entries`);
|
||||
}
|
||||
return {
|
||||
id, root, workRoot, stateFile, state, mangaDir, target,
|
||||
mangaId: path.basename(target), chapterCount, pageCount, incompleteStages,
|
||||
};
|
||||
}
|
||||
|
||||
async function promoteLegacyWork(inspection) {
|
||||
const { stateFile, mangaDir, target, state, workRoot } = inspection;
|
||||
const quarantineRoot = path.join(workRoot, '.legacy-incomplete');
|
||||
const quarantined = [];
|
||||
if (inspection.incompleteStages.length) await fs.mkdir(quarantineRoot, { recursive: true, mode: 0o755 });
|
||||
for (const stage of inspection.incompleteStages) {
|
||||
const relative = path.relative(mangaDir, stage).split(path.sep).join('__');
|
||||
const quarantine = path.join(quarantineRoot, relative);
|
||||
await renameRetry(stage, quarantine);
|
||||
quarantined.push({ stage, quarantine });
|
||||
}
|
||||
try {
|
||||
await renameRetry(mangaDir, target);
|
||||
} catch (error) {
|
||||
for (const item of quarantined.reverse()) await renameRetry(item.quarantine, item.stage).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
const migrated = {
|
||||
...state,
|
||||
publicationMode: 'incremental',
|
||||
status: 'paused',
|
||||
current: null,
|
||||
jobCompleted: 0,
|
||||
total: 0,
|
||||
migratedToIncremental: {
|
||||
time: new Date().toISOString(),
|
||||
mangaId: inspection.mangaId,
|
||||
chaptersPublished: inspection.chapterCount,
|
||||
pagesPublished: inspection.pageCount,
|
||||
incompleteStagesQuarantined: quarantined.length,
|
||||
},
|
||||
};
|
||||
try {
|
||||
await writeJsonAtomic(stateFile, migrated);
|
||||
} catch (error) {
|
||||
await renameRetry(target, mangaDir).catch(() => {});
|
||||
for (const item of quarantined.reverse()) await renameRetry(item.quarantine, item.stage).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
await fs.appendFile(path.join(workRoot, 'job.log'), JSON.stringify({
|
||||
time: migrated.migratedToIncremental.time,
|
||||
level: 'info',
|
||||
message: `Published ${inspection.chapterCount} completed legacy chapters and switched to incremental mode`,
|
||||
mangaId: inspection.mangaId,
|
||||
}) + '\n').catch(() => {});
|
||||
return {
|
||||
seriesId: inspection.id,
|
||||
mangaId: inspection.mangaId,
|
||||
finalPath: target,
|
||||
chaptersPublished: inspection.chapterCount,
|
||||
pagesPublished: inspection.pageCount,
|
||||
incompleteStagesQuarantined: quarantined.length,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { inspectLegacyWork, promoteLegacyWork };
|
||||
Reference in new issue
Block a user