Consolidate MangaCircus app, scraper, and Compose baseline
This commit is contained in:
Maashkinoozhe committed
1 parent
cfb2d79103
commit
eef4ffa07c
103 files changed
+9316
-172
No files matched your search
@@ -0,0 +1,220 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { SlowHttp } = require('../lib/http');
|
||||
const { sleep } = require('../lib/delay');
|
||||
const { ArchiveControlError } = require('../lib/control');
|
||||
const { cleanPart } = require('../lib/names');
|
||||
const { publishManifest } = require('../lib/publisher');
|
||||
const { selectByScope } = require('../lib/scope');
|
||||
const { scanLibrary, findManga, shouldDownload, sourceKey } = require('../lib/library-inventory');
|
||||
const { lookupAniList } = require('../metadata/anilist');
|
||||
const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral');
|
||||
|
||||
function randomBetween(min, max) {
|
||||
return min + Math.floor(Math.random() * (max - min + 1));
|
||||
}
|
||||
|
||||
async function exists(file) {
|
||||
try { await fs.access(file); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
async function writeJsonAtomic(file, value) {
|
||||
const temp = `${file}.tmp`;
|
||||
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n');
|
||||
await fs.rename(temp, file).catch(async error => {
|
||||
if (!['EPERM', 'EEXIST'].includes(error.code)) throw error;
|
||||
await fs.rm(file, { force: true });
|
||||
await fs.rename(temp, file);
|
||||
});
|
||||
}
|
||||
|
||||
async function renameRetry(from, to) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try { await fs.rename(from, to); return; } catch (error) {
|
||||
if (attempt >= 5 || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
||||
await sleep(200 * (attempt + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runArchive(options) {
|
||||
const checkControl = () => {
|
||||
const reason = options.shouldStop && options.shouldStop();
|
||||
if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason));
|
||||
};
|
||||
const libraryRoot = path.resolve(options.libraryRoot || 'library');
|
||||
const sourceId = options.seriesUrl.split('/series/')[1]?.split('/')[0];
|
||||
if (!sourceId) throw new Error('Could not read WeebCentral series id');
|
||||
const workRoot = path.join(libraryRoot, `.mcscraper-work-${sourceId}`);
|
||||
const stateFile = path.join(workRoot, 'job.json');
|
||||
const logFile = path.join(workRoot, 'job.log');
|
||||
await fs.mkdir(workRoot, { recursive: true });
|
||||
|
||||
const resumed = await exists(stateFile);
|
||||
let state = { version: 2, seriesUrl: options.seriesUrl, publicationMode: 'incremental', completed: [], started: new Date().toISOString() };
|
||||
if (resumed) state = JSON.parse(await fs.readFile(stateFile, 'utf8'));
|
||||
const incremental = state.publicationMode === 'incremental';
|
||||
const completed = new Set(state.completed || []);
|
||||
const event = async (level, message, details = {}) => {
|
||||
const entry = { time: new Date().toISOString(), level, message, ...details };
|
||||
await fs.appendFile(logFile, JSON.stringify(entry) + '\n');
|
||||
state.lastEvent = entry;
|
||||
if (level === 'warning' || level === 'error') {
|
||||
state.lastProblem = entry;
|
||||
state.problemCount = Number(state.problemCount || 0) + 1;
|
||||
}
|
||||
state.updated = entry.time;
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
};
|
||||
const http = new SlowHttp({
|
||||
minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2,
|
||||
onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }),
|
||||
});
|
||||
|
||||
checkControl();
|
||||
console.log(`[archive] discovering catalog ${options.seriesUrl}`);
|
||||
const catalog = await discoverWeebCatalog(options.seriesUrl, http);
|
||||
if (options.onCatalog) await options.onCatalog(catalog);
|
||||
if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || 'WeebCentral lists no chapter links for this series.'), {
|
||||
failureKind: catalog.chapterLinks ? 'adapter' : 'unavailable',
|
||||
});
|
||||
const keys = new Map();
|
||||
for (const chapter of catalog.chapters) {
|
||||
const key = `${chapter.book?.number || 1}:${chapter.number}`;
|
||||
const prior = keys.get(key);
|
||||
if (prior) throw new Error(`Ambiguous mapped chapter ${key}: ${prior.url} and ${chapter.url}`);
|
||||
keys.set(key, chapter);
|
||||
}
|
||||
console.log(`[archive] ${catalog.title}: ${catalog.chapters.length} source chapters`);
|
||||
const metadata = options.metadataSearch ? await lookupAniList(options.metadataSearch) : {};
|
||||
const series = {
|
||||
title: catalog.title, ...require('../../reading-direction').fallbackDirection(),
|
||||
sources: [{ name: 'WeebCentral', url: options.seriesUrl }],
|
||||
metadata: { ...metadata, title: catalog.title },
|
||||
coverUrl: metadata['x-anilist']?.coverUrl,
|
||||
scraper: { ids: { weebcentral: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } },
|
||||
};
|
||||
|
||||
let inventory = await scanLibrary(libraryRoot);
|
||||
let manga = findManga(inventory, {
|
||||
source: 'weebcentral', seriesId: sourceId, sourceUrl: options.seriesUrl,
|
||||
anilist: metadata['x-anilist']?.id, libraryMangaId: options.libraryMangaId,
|
||||
});
|
||||
if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`);
|
||||
const eligible = incremental
|
||||
? catalog.chapters.filter(ref => shouldDownload(manga, ref, 'weebcentral', options.existingPolicy))
|
||||
: catalog.chapters;
|
||||
const refs = incremental ? selectByScope(eligible, options.scope) : eligible;
|
||||
state.status = 'running';
|
||||
state.catalogTotal = catalog.chapters.length;
|
||||
state.total = incremental ? refs.length : catalog.chapters.length;
|
||||
state.jobCompleted = 0;
|
||||
state.scope = options.scope || { mode: 'missing' };
|
||||
state.existingPolicy = options.existingPolicy || 'skip-any';
|
||||
state.rate = { minMs: options.minMs, maxMs: options.maxMs, chapterMinMs: options.chapterMinMs, chapterMaxMs: options.chapterMaxMs };
|
||||
state.lastError = null;
|
||||
await event('info', `Catalog ready: ${catalog.chapters.length} chapters; ${refs.length} selected`);
|
||||
|
||||
if (incremental && !refs.length) {
|
||||
state.status = 'complete'; state.current = null;
|
||||
await event('info', 'No eligible chapters to download');
|
||||
return { finalPath: manga?.path || null, publishedCount: 0, selectedCount: 0 };
|
||||
}
|
||||
|
||||
let publishedCount = 0;
|
||||
for (let index = 0; index < refs.length; index++) {
|
||||
checkControl();
|
||||
const ref = refs[index];
|
||||
if (!incremental && completed.has(ref.id)) {
|
||||
if (!incremental) console.log(`[archive] skip completed ${ref.number} (${index + 1}/${refs.length})`);
|
||||
continue;
|
||||
}
|
||||
if (incremental && !shouldDownload(manga, ref, 'weebcentral', options.existingPolicy)) {
|
||||
state.jobCompleted++;
|
||||
await event('info', `Skipped existing chapter ${ref.number}`, { chapter: ref.number });
|
||||
continue;
|
||||
}
|
||||
console.log(`[archive] inspect ${ref.number} (${index + 1}/${refs.length})`);
|
||||
let chapter;
|
||||
let result;
|
||||
try {
|
||||
state.status = 'inspecting';
|
||||
state.current = { number: ref.number, ...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: null };
|
||||
await event('info', `Inspecting chapter ${ref.number}`, { chapter: ref.number });
|
||||
chapter = await discoverWeebChapter(ref, http);
|
||||
state.status = 'downloading';
|
||||
state.current.expectedPages = chapter.pages.length;
|
||||
await event('info', `Downloading chapter ${ref.number}: ${chapter.pages.length} pages`, { chapter: ref.number });
|
||||
console.log(`[archive] download ${ref.number}: ${chapter.pages.length} pages`);
|
||||
result = await publishManifest({ version: 1, series, chapters: [chapter] }, {
|
||||
libraryRoot: incremental ? libraryRoot : workRoot,
|
||||
...(incremental && manga ? { mangaDir: manga.path } : {}),
|
||||
httpClient: http,
|
||||
onPage: async progress => {
|
||||
state.current.downloadedPages = progress.completed;
|
||||
state.updated = new Date().toISOString();
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
checkControl();
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ArchiveControlError) {
|
||||
state.status = error.reason;
|
||||
state.current = null;
|
||||
await event('info', `Archive ${error.reason}`, { chapter: ref.number });
|
||||
throw error;
|
||||
}
|
||||
state.status = 'error';
|
||||
state.lastError = { time: new Date().toISOString(), chapter: ref.number, message: error.message };
|
||||
await event('error', error.message, { chapter: ref.number });
|
||||
throw error;
|
||||
}
|
||||
completed.add(ref.id);
|
||||
state.completed = [...completed];
|
||||
state.jobCompleted = Number(state.jobCompleted || 0) + 1;
|
||||
state.lastChapter = ref.number;
|
||||
state.updated = new Date().toISOString();
|
||||
state.total = incremental ? refs.length : catalog.chapters.length;
|
||||
state.status = 'running';
|
||||
state.current = null;
|
||||
await writeJsonAtomic(stateFile, state);
|
||||
const published = result.results[0].status === 'published';
|
||||
if (published && incremental) {
|
||||
publishedCount++;
|
||||
if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] };
|
||||
manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number), sourceKey: sourceKey(chapter.source.url, chapter.source.name) });
|
||||
if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number });
|
||||
}
|
||||
await event('info', `Completed chapter ${ref.number} (${state.jobCompleted}/${state.total})`, { chapter: ref.number });
|
||||
console.log(`[archive] complete ${ref.number}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`);
|
||||
if (index < refs.length - 1) {
|
||||
checkControl();
|
||||
const pause = randomBetween(options.chapterMinMs, options.chapterMaxMs);
|
||||
console.log(`[archive] chapter pause ${Math.round(pause / 1000)}s`);
|
||||
await sleep(pause);
|
||||
checkControl();
|
||||
}
|
||||
}
|
||||
|
||||
if (incremental) {
|
||||
state.status = 'complete'; state.current = null;
|
||||
await event('info', `Published ${publishedCount} chapter${publishedCount === 1 ? '' : 's'} incrementally`);
|
||||
return { finalPath: manga?.path || null, publishedCount, selectedCount: refs.length };
|
||||
}
|
||||
|
||||
const stagedManga = path.join(workRoot, cleanPart(catalog.title));
|
||||
const finalManga = path.join(libraryRoot, cleanPart(catalog.title));
|
||||
if (await exists(finalManga)) throw new Error(`Final manga folder already exists: ${finalManga}`);
|
||||
await renameRetry(stagedManga, finalManga);
|
||||
state.status = 'complete';
|
||||
state.current = null;
|
||||
await event('info', `Published ${finalManga}`);
|
||||
await fs.rm(workRoot, { recursive: true, force: true });
|
||||
if (options.onPublished) await options.onPublished({ mangaDir: finalManga, completeManga: true });
|
||||
console.log(`[archive] PUBLISHED ${finalManga}`);
|
||||
return { finalPath: finalManga, publishedCount: 1, selectedCount: catalog.chapters.length };
|
||||
}
|
||||
|
||||
module.exports = { runArchive };
|
||||
Reference in new issue
Block a user