216 lines
11 KiB
JavaScript
216 lines
11 KiB
JavaScript
'use strict';
|
|
|
|
const fs = require('fs').promises;
|
|
const path = require('path');
|
|
const { SlowHttp } = require('../lib/http');
|
|
const { sleep } = require('../lib/delay');
|
|
const { ArchiveControlError } = require('../lib/control');
|
|
const { cleanPart } = require('../lib/names');
|
|
const { publishManifest } = require('../lib/publisher');
|
|
const { selectByScope } = require('../lib/scope');
|
|
const { scanLibrary, findManga, shouldDownload, sourceKey } = require('../lib/library-inventory');
|
|
const { lookupAniList, lookupAniListById } = require('../metadata/anilist');
|
|
const { mangaDexInfo, discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex');
|
|
|
|
const randomBetween = (min, max) => min + Math.floor(Math.random() * (max - min + 1));
|
|
const exists = async file => { try { await fs.access(file); return true; } catch { return false; } };
|
|
|
|
async function writeJsonAtomic(file, value) {
|
|
const temp = `${file}.tmp`;
|
|
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n');
|
|
await fs.rename(temp, file).catch(async error => {
|
|
if (!['EPERM', 'EEXIST'].includes(error.code)) throw error;
|
|
await fs.rm(file, { force: true });
|
|
await fs.rename(temp, file);
|
|
});
|
|
}
|
|
|
|
async function renameRetry(from, to) {
|
|
for (let attempt = 0; ; attempt++) {
|
|
try { await fs.rename(from, to); return; } catch (error) {
|
|
if (attempt >= 5 || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
|
await sleep(200 * (attempt + 1));
|
|
}
|
|
}
|
|
}
|
|
|
|
async function runMangaDexArchive(options) {
|
|
const info = mangaDexInfo(options.seriesUrl);
|
|
if (!info) throw new Error('Could not read MangaDex title id');
|
|
const checkControl = () => {
|
|
const reason = options.shouldStop && options.shouldStop();
|
|
if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason));
|
|
};
|
|
const libraryRoot = path.resolve(options.libraryRoot || 'library');
|
|
const workRoot = path.join(libraryRoot, `.mcscraper-work-${info.id}`);
|
|
const stateFile = path.join(workRoot, 'job.json');
|
|
const logFile = path.join(workRoot, 'job.log');
|
|
await fs.mkdir(workRoot, { recursive: true });
|
|
|
|
const resumed = await exists(stateFile);
|
|
let state = { version: 2, seriesUrl: options.seriesUrl, source: 'mangadex', publicationMode: 'incremental', completed: [], started: new Date().toISOString() };
|
|
if (resumed) state = JSON.parse(await fs.readFile(stateFile, 'utf8'));
|
|
const incremental = state.publicationMode === 'incremental';
|
|
const completed = new Set(state.completed || []);
|
|
const event = async (level, message, details = {}) => {
|
|
const entry = { time: new Date().toISOString(), level, message, ...details };
|
|
await fs.appendFile(logFile, JSON.stringify(entry) + '\n');
|
|
state.lastEvent = entry;
|
|
if (level === 'warning' || level === 'error') {
|
|
state.lastProblem = entry;
|
|
state.problemCount = Number(state.problemCount || 0) + 1;
|
|
}
|
|
state.updated = entry.time;
|
|
await writeJsonAtomic(stateFile, state);
|
|
};
|
|
|
|
const minMs = Number(options.minMs);
|
|
const maxMs = Number(options.maxMs);
|
|
const chapterMinMs = Number(options.chapterMinMs);
|
|
const chapterMaxMs = Number(options.chapterMaxMs);
|
|
const http = new SlowHttp({
|
|
minMs, maxMs, maxRetries: 2,
|
|
userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)',
|
|
onEvent: item => event(item.level, item.message, { status: item.status }),
|
|
});
|
|
|
|
checkControl();
|
|
console.log(`[mangadex] discovering catalog ${info.id}`);
|
|
const catalog = await discoverMangaDexCatalog(options.seriesUrl, http);
|
|
if (options.onCatalog) await options.onCatalog(catalog);
|
|
if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability.message), { failureKind: 'unavailable' });
|
|
const specialOnly = catalog.chapters.every(ref => ref.special);
|
|
const folderTitle = String(specialOnly ? catalog.title : options.title || catalog.title).trim() || catalog.title;
|
|
console.log(`[mangadex] ${folderTitle}: ${catalog.chapters.length} unique English chapters`);
|
|
const expectedAniList = catalog.links.al ? String(catalog.links.al) : null;
|
|
const metadata = expectedAniList
|
|
? await lookupAniListById(expectedAniList)
|
|
: options.metadataSearch && !specialOnly ? await lookupAniList(options.metadataSearch) : {};
|
|
const actualAniList = metadata['x-anilist']?.id ? String(metadata['x-anilist'].id) : null;
|
|
if (expectedAniList && actualAniList && expectedAniList !== actualAniList) {
|
|
throw new Error(`AniList identity mismatch: MangaDex ${expectedAniList}, lookup ${actualAniList}`);
|
|
}
|
|
const series = {
|
|
title: folderTitle, ...require('../../reading-direction').fallbackDirection(),
|
|
sources: [{ name: 'MangaDex', url: options.seriesUrl }],
|
|
metadata: { ...metadata, title: specialOnly ? catalog.title : metadata.title || folderTitle },
|
|
coverUrl: metadata['x-anilist']?.coverUrl,
|
|
scraper: { ids: { mangadex: info.id, ...(expectedAniList ? { anilist: expectedAniList } : {}) } },
|
|
};
|
|
|
|
let inventory = await scanLibrary(libraryRoot);
|
|
let manga = findManga(inventory, {
|
|
source: 'mangadex', seriesId: info.id, sourceUrl: options.seriesUrl,
|
|
anilist: specialOnly ? null : expectedAniList || actualAniList, libraryMangaId: options.libraryMangaId,
|
|
});
|
|
if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`);
|
|
const eligible = incremental
|
|
? catalog.chapters.filter(ref => shouldDownload(manga, ref, 'mangadex', options.existingPolicy))
|
|
: catalog.chapters;
|
|
const refs = incremental ? selectByScope(eligible, options.scope) : eligible;
|
|
state.status = 'running';
|
|
state.catalogTotal = catalog.chapters.length;
|
|
state.total = incremental ? refs.length : catalog.chapters.length;
|
|
state.jobCompleted = 0;
|
|
state.scope = options.scope || { mode: 'missing' };
|
|
state.existingPolicy = options.existingPolicy || 'skip-any';
|
|
state.rate = { minMs, maxMs, chapterMinMs, chapterMaxMs };
|
|
state.lastError = null;
|
|
await event('info', `Catalog ready: ${catalog.chapters.length} chapters; ${refs.length} selected`);
|
|
|
|
if (incremental && !refs.length) {
|
|
state.status = 'complete'; state.current = null;
|
|
await event('info', 'No eligible chapters to download');
|
|
return { finalPath: manga?.path || null, publishedCount: 0, selectedCount: 0 };
|
|
}
|
|
|
|
let publishedCount = 0;
|
|
for (let index = 0; index < refs.length; index++) {
|
|
checkControl();
|
|
const ref = refs[index];
|
|
if (!incremental && completed.has(ref.id)) {
|
|
if (!incremental) console.log(`[mangadex] skip completed ${ref.number} (${index + 1}/${refs.length})`);
|
|
continue;
|
|
}
|
|
if (incremental && !shouldDownload(manga, ref, 'mangadex', options.existingPolicy)) {
|
|
state.jobCompleted++;
|
|
await event('info', `Skipped existing chapter ${ref.number}`, { chapter: ref.number });
|
|
continue;
|
|
}
|
|
let chapter, result;
|
|
try {
|
|
state.status = 'inspecting';
|
|
state.current = { number: ref.number, label: ref.special ? ref.label : null, ...(ref.volume ? { volume: ref.volume } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: ref.pages || null };
|
|
await event('info', `Inspecting chapter ${ref.number}`, { chapter: ref.number });
|
|
chapter = await discoverMangaDexChapter(ref, http);
|
|
state.status = 'downloading';
|
|
state.current.expectedPages = chapter.pages.length;
|
|
await event('info', `Downloading chapter ${ref.number}: ${chapter.pages.length} pages`, { chapter: ref.number });
|
|
console.log(`[mangadex] download ${ref.number}: ${chapter.pages.length} pages`);
|
|
result = await publishManifest({ version: 1, series, chapters: [chapter] }, {
|
|
libraryRoot: incremental ? libraryRoot : workRoot,
|
|
...(incremental && manga ? { mangaDir: manga.path } : {}),
|
|
httpClient: http,
|
|
onPage: async progress => {
|
|
state.current.downloadedPages = progress.completed;
|
|
state.updated = new Date().toISOString();
|
|
await writeJsonAtomic(stateFile, state);
|
|
checkControl();
|
|
},
|
|
});
|
|
} catch (error) {
|
|
if (error instanceof ArchiveControlError) {
|
|
state.status = error.reason;
|
|
state.current = null;
|
|
await event('info', `Archive ${error.reason}`, { chapter: ref.number });
|
|
throw error;
|
|
}
|
|
state.status = 'error';
|
|
state.lastError = { time: new Date().toISOString(), chapter: ref.number, message: error.message };
|
|
await event('error', error.message, { chapter: ref.number });
|
|
throw error;
|
|
}
|
|
completed.add(ref.id);
|
|
state.completed = [...completed];
|
|
state.jobCompleted = Number(state.jobCompleted || 0) + 1;
|
|
state.lastChapter = ref.number;
|
|
state.status = 'running';
|
|
state.current = null;
|
|
await writeJsonAtomic(stateFile, state);
|
|
const published = result.results[0].status === 'published';
|
|
if (published && incremental) {
|
|
publishedCount++;
|
|
if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] };
|
|
manga.chapters.push({ book: '1', number: String(ref.number), sourceKey: sourceKey(chapter.source.url, chapter.source.name) });
|
|
if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number });
|
|
}
|
|
await event('info', `Completed chapter ${ref.number} (${state.jobCompleted}/${state.total})`, { chapter: ref.number });
|
|
console.log(`[mangadex] complete ${ref.number}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`);
|
|
if (index < refs.length - 1) {
|
|
checkControl();
|
|
await sleep(randomBetween(chapterMinMs, chapterMaxMs));
|
|
checkControl();
|
|
}
|
|
}
|
|
|
|
if (incremental) {
|
|
state.status = 'complete'; state.current = null;
|
|
await event('info', `Published ${publishedCount} chapter${publishedCount === 1 ? '' : 's'} incrementally`);
|
|
return { finalPath: manga?.path || null, publishedCount, selectedCount: refs.length };
|
|
}
|
|
|
|
const stagedManga = path.join(workRoot, cleanPart(folderTitle));
|
|
const finalManga = path.join(libraryRoot, cleanPart(folderTitle));
|
|
if (await exists(finalManga)) throw new Error(`Final manga folder already exists: ${finalManga}`);
|
|
await renameRetry(stagedManga, finalManga);
|
|
state.status = 'complete';
|
|
state.current = null;
|
|
await event('info', `Published ${finalManga}`);
|
|
await fs.rm(workRoot, { recursive: true, force: true });
|
|
if (options.onPublished) await options.onPublished({ mangaDir: finalManga, completeManga: true });
|
|
console.log(`[mangadex] PUBLISHED ${finalManga}`);
|
|
return { finalPath: finalManga, publishedCount: 1, selectedCount: catalog.chapters.length };
|
|
}
|
|
|
|
module.exports = { runMangaDexArchive };
|