'use strict'; const fs = require('fs').promises; const path = require('path'); const { SlowHttp } = require('../lib/http'); const { sleep } = require('../lib/delay'); const { ArchiveControlError } = require('../lib/control'); const { cleanPart } = require('../lib/names'); const { publishManifest } = require('../lib/publisher'); const { selectByScope } = require('../lib/scope'); const { scanLibrary, findManga, shouldDownload, sourceKey } = require('../lib/library-inventory'); const { lookupAniList, lookupAniListById } = require('../metadata/anilist'); const { mangaDexInfo, discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex'); const randomBetween = (min, max) => min + Math.floor(Math.random() * (max - min + 1)); const exists = async file => { try { await fs.access(file); return true; } catch { return false; } }; async function writeJsonAtomic(file, value) { const temp = `${file}.tmp`; await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n'); await fs.rename(temp, file).catch(async error => { if (!['EPERM', 'EEXIST'].includes(error.code)) throw error; await fs.rm(file, { force: true }); await fs.rename(temp, file); }); } async function renameRetry(from, to) { for (let attempt = 0; ; attempt++) { try { await fs.rename(from, to); return; } catch (error) { if (attempt >= 5 || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error; await sleep(200 * (attempt + 1)); } } } async function runMangaDexArchive(options) { const info = mangaDexInfo(options.seriesUrl); if (!info) throw new Error('Could not read MangaDex title id'); const checkControl = () => { const reason = options.shouldStop && options.shouldStop(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); }; const libraryRoot = path.resolve(options.libraryRoot || 'library'); const workRoot = path.join(libraryRoot, `.mcscraper-work-${info.id}`); const stateFile = path.join(workRoot, 'job.json'); const logFile = path.join(workRoot, 'job.log'); await fs.mkdir(workRoot, { recursive: true }); const resumed = await exists(stateFile); let state = { version: 2, seriesUrl: options.seriesUrl, source: 'mangadex', publicationMode: 'incremental', completed: [], started: new Date().toISOString() }; if (resumed) state = JSON.parse(await fs.readFile(stateFile, 'utf8')); const incremental = state.publicationMode === 'incremental'; const completed = new Set(state.completed || []); const event = async (level, message, details = {}) => { const entry = { time: new Date().toISOString(), level, message, ...details }; await fs.appendFile(logFile, JSON.stringify(entry) + '\n'); state.lastEvent = entry; if (level === 'warning' || level === 'error') { state.lastProblem = entry; state.problemCount = Number(state.problemCount || 0) + 1; } state.updated = entry.time; await writeJsonAtomic(stateFile, state); }; const minMs = Number(options.minMs); const maxMs = Number(options.maxMs); const chapterMinMs = Number(options.chapterMinMs); const chapterMaxMs = Number(options.chapterMaxMs); const http = new SlowHttp({ gateKey: 'mangadex', minMs, maxMs, maxRetries: 2, userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)', onEvent: item => event(item.level, item.message, { status: item.status }), }); checkControl(); console.log(`[mangadex] discovering catalog ${info.id}`); const catalog = await discoverMangaDexCatalog(options.seriesUrl, http); if (options.onCatalog) await options.onCatalog(catalog); if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability.message), { failureKind: 'unavailable' }); const specialOnly = catalog.chapters.every(ref => ref.special); const folderTitle = String(specialOnly ? catalog.title : options.title || catalog.title).trim() || catalog.title; console.log(`[mangadex] ${folderTitle}: ${catalog.chapters.length} unique English chapters`); const expectedAniList = catalog.links.al ? String(catalog.links.al) : null; const metadata = expectedAniList ? await lookupAniListById(expectedAniList) : options.metadataSearch && !specialOnly ? await lookupAniList(options.metadataSearch) : {}; const actualAniList = metadata['x-anilist']?.id ? String(metadata['x-anilist'].id) : null; if (expectedAniList && actualAniList && expectedAniList !== actualAniList) { throw new Error(`AniList identity mismatch: MangaDex ${expectedAniList}, lookup ${actualAniList}`); } const series = { title: folderTitle, ...require('../../reading-direction').fallbackDirection(), sources: [{ name: 'MangaDex', url: options.seriesUrl }], metadata: { ...metadata, title: specialOnly ? catalog.title : metadata.title || folderTitle }, coverUrl: metadata['x-anilist']?.coverUrl, scraper: { ids: { mangadex: info.id, ...(expectedAniList ? { anilist: expectedAniList } : {}) } }, }; let inventory = await scanLibrary(libraryRoot); let manga = findManga(inventory, { source: 'mangadex', seriesId: info.id, sourceUrl: options.seriesUrl, anilist: specialOnly ? null : expectedAniList || actualAniList, libraryMangaId: options.libraryMangaId, }); if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`); const eligible = incremental ? catalog.chapters.filter(ref => shouldDownload(manga, ref, 'mangadex', options.existingPolicy)) : catalog.chapters; const refs = incremental ? selectByScope(eligible, options.scope) : eligible; state.status = 'running'; state.catalogTotal = catalog.chapters.length; state.total = incremental ? refs.length : catalog.chapters.length; state.jobCompleted = 0; state.scope = options.scope || { mode: 'missing' }; state.existingPolicy = options.existingPolicy || 'skip-any'; state.rate = { minMs, maxMs, chapterMinMs, chapterMaxMs }; state.lastError = null; await event('info', `Catalog ready: ${catalog.chapters.length} chapters; ${refs.length} selected`); if (incremental && !refs.length) { state.status = 'complete'; state.current = null; await event('info', 'No eligible chapters to download'); return { finalPath: manga?.path || null, publishedCount: 0, selectedCount: 0 }; } let publishedCount = 0; for (let index = 0; index < refs.length; index++) { checkControl(); const ref = refs[index]; if (!incremental && completed.has(ref.id)) { if (!incremental) console.log(`[mangadex] skip completed ${ref.number} (${index + 1}/${refs.length})`); continue; } if (incremental && !shouldDownload(manga, ref, 'mangadex', options.existingPolicy)) { state.jobCompleted++; await event('info', `Skipped existing chapter ${ref.number}`, { chapter: ref.number }); continue; } let chapter, result; try { state.status = 'inspecting'; state.current = { number: ref.number, label: ref.special ? ref.label : null, ...(ref.volume ? { volume: ref.volume } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: ref.pages || null }; await event('info', `Inspecting chapter ${ref.number}`, { chapter: ref.number }); chapter = await discoverMangaDexChapter(ref, http); state.status = 'downloading'; state.current.expectedPages = chapter.pages.length; await event('info', `Downloading chapter ${ref.number}: ${chapter.pages.length} pages`, { chapter: ref.number }); console.log(`[mangadex] download ${ref.number}: ${chapter.pages.length} pages`); result = await publishManifest({ version: 1, series, chapters: [chapter] }, { libraryRoot: incremental ? libraryRoot : workRoot, ...(incremental && manga ? { mangaDir: manga.path } : {}), httpClient: http, onPage: async progress => { state.current.downloadedPages = progress.completed; state.updated = new Date().toISOString(); await writeJsonAtomic(stateFile, state); checkControl(); }, }); } catch (error) { if (error instanceof ArchiveControlError) { state.status = error.reason; state.current = null; await event('info', `Archive ${error.reason}`, { chapter: ref.number }); throw error; } state.status = 'error'; state.lastError = { time: new Date().toISOString(), chapter: ref.number, message: error.message }; await event('error', error.message, { chapter: ref.number }); throw error; } completed.add(ref.id); state.completed = [...completed]; state.jobCompleted = Number(state.jobCompleted || 0) + 1; state.lastChapter = ref.number; state.status = 'running'; state.current = null; await writeJsonAtomic(stateFile, state); const published = result.results[0].status === 'published'; if (published && incremental) { publishedCount++; if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] }; manga.chapters.push({ book: '1', number: String(ref.number), sourceKey: sourceKey(chapter.source.url, chapter.source.name) }); if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number }); } await event('info', `Completed chapter ${ref.number} (${state.jobCompleted}/${state.total})`, { chapter: ref.number }); console.log(`[mangadex] complete ${ref.number}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`); if (index < refs.length - 1) { checkControl(); await sleep(randomBetween(chapterMinMs, chapterMaxMs)); checkControl(); } } if (incremental) { state.status = 'complete'; state.current = null; await event('info', `Published ${publishedCount} chapter${publishedCount === 1 ? '' : 's'} incrementally`); return { finalPath: manga?.path || null, publishedCount, selectedCount: refs.length }; } const stagedManga = path.join(workRoot, cleanPart(folderTitle)); const finalManga = path.join(libraryRoot, cleanPart(folderTitle)); if (await exists(finalManga)) throw new Error(`Final manga folder already exists: ${finalManga}`); await renameRetry(stagedManga, finalManga); state.status = 'complete'; state.current = null; await event('info', `Published ${finalManga}`); await fs.rm(workRoot, { recursive: true, force: true }); if (options.onPublished) await options.onPublished({ mangaDir: finalManga, completeManga: true }); console.log(`[mangadex] PUBLISHED ${finalManga}`); return { finalPath: finalManga, publishedCount: 1, selectedCount: catalog.chapters.length }; } module.exports = { runMangaDexArchive };