'use strict'; const fs = require('fs').promises; const path = require('path'); const { createWeebHttp } = require('../plugins/broker'); const { sleep } = require('../lib/delay'); const { ArchiveControlError } = require('../lib/control'); const { cleanPart } = require('../lib/names'); const { publishManifest } = require('../lib/publisher'); const { selectByScope } = require('../lib/scope'); const { scanLibrary, findManga, shouldDownload, sourceKey } = require('../lib/library-inventory'); const { lookupAniList } = require('../metadata/anilist'); const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral'); function randomBetween(min, max) { return min + Math.floor(Math.random() * (max - min + 1)); } async function exists(file) { try { await fs.access(file); return true; } catch { return false; } } async function writeJsonAtomic(file, value) { const temp = `${file}.tmp`; await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n'); await fs.rename(temp, file).catch(async error => { if (!['EPERM', 'EEXIST'].includes(error.code)) throw error; await fs.rm(file, { force: true }); await fs.rename(temp, file); }); } async function renameRetry(from, to) { for (let attempt = 0; ; attempt++) { try { await fs.rename(from, to); return; } catch (error) { if (attempt >= 5 || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error; await sleep(200 * (attempt + 1)); } } } async function runArchive(options) { const source = options.source || 'weebcentral'; const sourceLabel = options.sourceLabel || 'WeebCentral'; const discoverCatalog = options.discoverCatalog || discoverWeebCatalog; const discoverChapter = options.discoverChapter || discoverWeebChapter; const checkControl = () => { const reason = options.shouldStop && options.shouldStop(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); }; const libraryRoot = path.resolve(options.libraryRoot || 'library'); const sourceId = options.sourceId || options.seriesUrl.split('/series/')[1]?.split('/')[0]; if (!sourceId) throw new Error(`Could not read ${sourceLabel} series id`); const workRoot = path.join(libraryRoot, source === 'weebcentral' ? `.mcscraper-work-${sourceId}` : `.mcscraper-work-${source}-${sourceId}`); const stateFile = path.join(workRoot, 'job.json'); const logFile = path.join(workRoot, 'job.log'); await fs.mkdir(workRoot, { recursive: true }); const resumed = await exists(stateFile); let state = { version: 2, seriesUrl: options.seriesUrl, publicationMode: 'incremental', completed: [], started: new Date().toISOString() }; if (resumed) state = JSON.parse(await fs.readFile(stateFile, 'utf8')); const incremental = state.publicationMode === 'incremental'; const completed = new Set(state.completed || []); const event = async (level, message, details = {}) => { const entry = { time: new Date().toISOString(), level, message, ...details }; await fs.appendFile(logFile, JSON.stringify(entry) + '\n'); state.lastEvent = entry; if (level === 'warning' || level === 'error') { state.lastProblem = entry; state.problemCount = Number(state.problemCount || 0) + 1; } state.updated = entry.time; await writeJsonAtomic(stateFile, state); }; const httpOptions = { minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2, onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }), }; const http = options.createHttp ? options.createHttp(httpOptions) : createWeebHttp(httpOptions); checkControl(); console.log(`[archive] discovering catalog ${options.seriesUrl}`); const catalog = await discoverCatalog(options.seriesUrl, http); if (options.onCatalog) await options.onCatalog(catalog); if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || `${sourceLabel} lists no chapter links for this series.`), { failureKind: catalog.chapterLinks ? 'adapter' : 'unavailable', }); const keys = new Map(); for (const chapter of catalog.chapters) { const key = `${chapter.book?.number || 1}:${chapter.number}`; const prior = keys.get(key); if (prior) throw new Error(`Ambiguous mapped chapter ${key}: ${prior.url} and ${chapter.url}`); keys.set(key, chapter); } console.log(`[archive] ${catalog.title}: ${catalog.chapters.length} source chapters`); const metadata = options.metadataSearch ? await lookupAniList(options.metadataSearch) : {}; const series = { title: catalog.title, ...require('../../reading-direction').fallbackDirection(), sources: [{ name: sourceLabel, url: options.seriesUrl }], metadata: { ...metadata, title: catalog.title }, coverUrl: metadata['x-anilist']?.coverUrl, scraper: { ids: { [source]: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } }, }; let inventory = await scanLibrary(libraryRoot); let manga = findManga(inventory, { source, seriesId: sourceId, sourceUrl: options.seriesUrl, anilist: metadata['x-anilist']?.id, libraryMangaId: options.libraryMangaId, }); if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`); const eligible = incremental ? catalog.chapters.filter(ref => shouldDownload(manga, ref, source, options.existingPolicy)) : catalog.chapters; const refs = incremental ? selectByScope(eligible, options.scope) : eligible; state.status = 'running'; state.catalogTotal = catalog.chapters.length; state.total = incremental ? refs.length : catalog.chapters.length; state.jobCompleted = 0; state.scope = options.scope || { mode: 'missing' }; state.existingPolicy = options.existingPolicy || 'skip-any'; state.rate = { minMs: options.minMs, maxMs: options.maxMs, chapterMinMs: options.chapterMinMs, chapterMaxMs: options.chapterMaxMs }; state.lastError = null; await event('info', `Catalog ready: ${catalog.chapters.length} chapters; ${refs.length} selected`); if (incremental && !refs.length) { state.status = 'complete'; state.current = null; await event('info', 'No eligible chapters to download'); return { finalPath: manga?.path || null, publishedCount: 0, selectedCount: 0 }; } let publishedCount = 0; for (let index = 0; index < refs.length; index++) { checkControl(); const ref = refs[index]; const display = ref.label || ref.number; if (!incremental && completed.has(ref.id)) { if (!incremental) console.log(`[archive] skip completed ${ref.number} (${index + 1}/${refs.length})`); continue; } if (incremental && !shouldDownload(manga, ref, source, options.existingPolicy)) { state.jobCompleted++; await event('info', `Skipped existing ${display}`, { chapter: ref.number }); continue; } console.log(`[archive] inspect ${display} (${index + 1}/${refs.length})`); let chapter; let result; try { state.status = 'inspecting'; state.current = { number: ref.number, ...(ref.label ? { label: ref.label } : {}), ...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: null }; await event('info', `Inspecting ${display}`, { chapter: ref.number }); chapter = await discoverChapter(ref, http); state.status = 'downloading'; state.current.expectedPages = chapter.pages.length; await event('info', `Downloading ${display}: ${chapter.pages.length} pages`, { chapter: ref.number }); console.log(`[archive] download ${display}: ${chapter.pages.length} pages`); result = await publishManifest({ version: 1, series, chapters: [chapter] }, { libraryRoot: incremental ? libraryRoot : workRoot, ...(incremental && manga ? { mangaDir: manga.path } : {}), httpClient: http, onPage: async progress => { state.current.downloadedPages = progress.completed; state.updated = new Date().toISOString(); await writeJsonAtomic(stateFile, state); checkControl(); }, }); } catch (error) { if (error instanceof ArchiveControlError) { state.status = error.reason; state.current = null; await event('info', `Archive ${error.reason}`, { chapter: ref.number }); throw error; } state.status = 'error'; state.lastError = { time: new Date().toISOString(), chapter: ref.number, message: error.message }; await event('error', error.message, { chapter: ref.number }); throw error; } completed.add(ref.id); state.completed = [...completed]; state.jobCompleted = Number(state.jobCompleted || 0) + 1; state.lastChapter = ref.number; state.updated = new Date().toISOString(); state.total = incremental ? refs.length : catalog.chapters.length; state.status = 'running'; state.current = null; await writeJsonAtomic(stateFile, state); const published = result.results[0].status === 'published'; if (published && incremental) { publishedCount++; if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] }; manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number), releaseKind: ref.releaseKind || null, sourceKey: sourceKey(chapter.source.url, chapter.source.name) }); if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number }); } await event('info', `Completed ${display} (${state.jobCompleted}/${state.total})`, { chapter: ref.number }); console.log(`[archive] complete ${display}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`); if (index < refs.length - 1) { checkControl(); const pause = randomBetween(options.chapterMinMs, options.chapterMaxMs); console.log(`[archive] chapter pause ${Math.round(pause / 1000)}s`); await sleep(pause); checkControl(); } } if (incremental) { state.status = 'complete'; state.current = null; await event('info', `Published ${publishedCount} chapter${publishedCount === 1 ? '' : 's'} incrementally`); return { finalPath: manga?.path || null, publishedCount, selectedCount: refs.length }; } const stagedManga = path.join(workRoot, cleanPart(catalog.title)); const finalManga = path.join(libraryRoot, cleanPart(catalog.title)); if (await exists(finalManga)) throw new Error(`Final manga folder already exists: ${finalManga}`); await renameRetry(stagedManga, finalManga); state.status = 'complete'; state.current = null; await event('info', `Published ${finalManga}`); await fs.rm(workRoot, { recursive: true, force: true }); if (options.onPublished) await options.onPublished({ mangaDir: finalManga, completeManga: true }); console.log(`[archive] PUBLISHED ${finalManga}`); return { finalPath: finalManga, publishedCount: 1, selectedCount: catalog.chapters.length }; } module.exports = { runArchive };