223 lines
11 KiB
JavaScript
223 lines
11 KiB
JavaScript
'use strict';
|
|
|
|
const fs = require('fs').promises;
|
|
const path = require('path');
|
|
const { SlowHttp } = require('../lib/http');
|
|
const { createWeebHttp } = require('../plugins/broker');
|
|
const { sleep } = require('../lib/delay');
|
|
const { ArchiveControlError } = require('../lib/control');
|
|
const { cleanPart } = require('../lib/names');
|
|
const { publishManifest } = require('../lib/publisher');
|
|
const { selectByScope } = require('../lib/scope');
|
|
const { scanLibrary, findManga, shouldDownload, sourceKey } = require('../lib/library-inventory');
|
|
const { lookupAniList } = require('../metadata/anilist');
|
|
const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral');
|
|
|
|
function randomBetween(min, max) {
|
|
return min + Math.floor(Math.random() * (max - min + 1));
|
|
}
|
|
|
|
async function exists(file) {
|
|
try { await fs.access(file); return true; } catch { return false; }
|
|
}
|
|
|
|
async function writeJsonAtomic(file, value) {
|
|
const temp = `${file}.tmp`;
|
|
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n');
|
|
await fs.rename(temp, file).catch(async error => {
|
|
if (!['EPERM', 'EEXIST'].includes(error.code)) throw error;
|
|
await fs.rm(file, { force: true });
|
|
await fs.rename(temp, file);
|
|
});
|
|
}
|
|
|
|
async function renameRetry(from, to) {
|
|
for (let attempt = 0; ; attempt++) {
|
|
try { await fs.rename(from, to); return; } catch (error) {
|
|
if (attempt >= 5 || !['EPERM', 'EBUSY', 'EACCES'].includes(error.code)) throw error;
|
|
await sleep(200 * (attempt + 1));
|
|
}
|
|
}
|
|
}
|
|
|
|
async function runArchive(options) {
|
|
const checkControl = () => {
|
|
const reason = options.shouldStop && options.shouldStop();
|
|
if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason));
|
|
};
|
|
const libraryRoot = path.resolve(options.libraryRoot || 'library');
|
|
const sourceId = options.seriesUrl.split('/series/')[1]?.split('/')[0];
|
|
if (!sourceId) throw new Error('Could not read WeebCentral series id');
|
|
const workRoot = path.join(libraryRoot, `.mcscraper-work-${sourceId}`);
|
|
const stateFile = path.join(workRoot, 'job.json');
|
|
const logFile = path.join(workRoot, 'job.log');
|
|
await fs.mkdir(workRoot, { recursive: true });
|
|
|
|
const resumed = await exists(stateFile);
|
|
let state = { version: 2, seriesUrl: options.seriesUrl, publicationMode: 'incremental', completed: [], started: new Date().toISOString() };
|
|
if (resumed) state = JSON.parse(await fs.readFile(stateFile, 'utf8'));
|
|
const incremental = state.publicationMode === 'incremental';
|
|
const completed = new Set(state.completed || []);
|
|
const event = async (level, message, details = {}) => {
|
|
const entry = { time: new Date().toISOString(), level, message, ...details };
|
|
await fs.appendFile(logFile, JSON.stringify(entry) + '\n');
|
|
state.lastEvent = entry;
|
|
if (level === 'warning' || level === 'error') {
|
|
state.lastProblem = entry;
|
|
state.problemCount = Number(state.problemCount || 0) + 1;
|
|
}
|
|
state.updated = entry.time;
|
|
await writeJsonAtomic(stateFile, state);
|
|
};
|
|
const httpOptions = {
|
|
minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2,
|
|
onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }),
|
|
};
|
|
const http = process.env.SOURCE_PLUGIN_URL ? createWeebHttp(httpOptions) : new SlowHttp(httpOptions);
|
|
|
|
checkControl();
|
|
console.log(`[archive] discovering catalog ${options.seriesUrl}`);
|
|
const catalog = await discoverWeebCatalog(options.seriesUrl, http);
|
|
if (options.onCatalog) await options.onCatalog(catalog);
|
|
if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || 'WeebCentral lists no chapter links for this series.'), {
|
|
failureKind: catalog.chapterLinks ? 'adapter' : 'unavailable',
|
|
});
|
|
const keys = new Map();
|
|
for (const chapter of catalog.chapters) {
|
|
const key = `${chapter.book?.number || 1}:${chapter.number}`;
|
|
const prior = keys.get(key);
|
|
if (prior) throw new Error(`Ambiguous mapped chapter ${key}: ${prior.url} and ${chapter.url}`);
|
|
keys.set(key, chapter);
|
|
}
|
|
console.log(`[archive] ${catalog.title}: ${catalog.chapters.length} source chapters`);
|
|
const metadata = options.metadataSearch ? await lookupAniList(options.metadataSearch) : {};
|
|
const series = {
|
|
title: catalog.title, ...require('../../reading-direction').fallbackDirection(),
|
|
sources: [{ name: 'WeebCentral', url: options.seriesUrl }],
|
|
metadata: { ...metadata, title: catalog.title },
|
|
coverUrl: metadata['x-anilist']?.coverUrl,
|
|
scraper: { ids: { weebcentral: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } },
|
|
};
|
|
|
|
let inventory = await scanLibrary(libraryRoot);
|
|
let manga = findManga(inventory, {
|
|
source: 'weebcentral', seriesId: sourceId, sourceUrl: options.seriesUrl,
|
|
anilist: metadata['x-anilist']?.id, libraryMangaId: options.libraryMangaId,
|
|
});
|
|
if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`);
|
|
const eligible = incremental
|
|
? catalog.chapters.filter(ref => shouldDownload(manga, ref, 'weebcentral', options.existingPolicy))
|
|
: catalog.chapters;
|
|
const refs = incremental ? selectByScope(eligible, options.scope) : eligible;
|
|
state.status = 'running';
|
|
state.catalogTotal = catalog.chapters.length;
|
|
state.total = incremental ? refs.length : catalog.chapters.length;
|
|
state.jobCompleted = 0;
|
|
state.scope = options.scope || { mode: 'missing' };
|
|
state.existingPolicy = options.existingPolicy || 'skip-any';
|
|
state.rate = { minMs: options.minMs, maxMs: options.maxMs, chapterMinMs: options.chapterMinMs, chapterMaxMs: options.chapterMaxMs };
|
|
state.lastError = null;
|
|
await event('info', `Catalog ready: ${catalog.chapters.length} chapters; ${refs.length} selected`);
|
|
|
|
if (incremental && !refs.length) {
|
|
state.status = 'complete'; state.current = null;
|
|
await event('info', 'No eligible chapters to download');
|
|
return { finalPath: manga?.path || null, publishedCount: 0, selectedCount: 0 };
|
|
}
|
|
|
|
let publishedCount = 0;
|
|
for (let index = 0; index < refs.length; index++) {
|
|
checkControl();
|
|
const ref = refs[index];
|
|
if (!incremental && completed.has(ref.id)) {
|
|
if (!incremental) console.log(`[archive] skip completed ${ref.number} (${index + 1}/${refs.length})`);
|
|
continue;
|
|
}
|
|
if (incremental && !shouldDownload(manga, ref, 'weebcentral', options.existingPolicy)) {
|
|
state.jobCompleted++;
|
|
await event('info', `Skipped existing chapter ${ref.number}`, { chapter: ref.number });
|
|
continue;
|
|
}
|
|
console.log(`[archive] inspect ${ref.number} (${index + 1}/${refs.length})`);
|
|
let chapter;
|
|
let result;
|
|
try {
|
|
state.status = 'inspecting';
|
|
state.current = { number: ref.number, ...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: null };
|
|
await event('info', `Inspecting chapter ${ref.number}`, { chapter: ref.number });
|
|
chapter = await discoverWeebChapter(ref, http);
|
|
state.status = 'downloading';
|
|
state.current.expectedPages = chapter.pages.length;
|
|
await event('info', `Downloading chapter ${ref.number}: ${chapter.pages.length} pages`, { chapter: ref.number });
|
|
console.log(`[archive] download ${ref.number}: ${chapter.pages.length} pages`);
|
|
result = await publishManifest({ version: 1, series, chapters: [chapter] }, {
|
|
libraryRoot: incremental ? libraryRoot : workRoot,
|
|
...(incremental && manga ? { mangaDir: manga.path } : {}),
|
|
httpClient: http,
|
|
onPage: async progress => {
|
|
state.current.downloadedPages = progress.completed;
|
|
state.updated = new Date().toISOString();
|
|
await writeJsonAtomic(stateFile, state);
|
|
checkControl();
|
|
},
|
|
});
|
|
} catch (error) {
|
|
if (error instanceof ArchiveControlError) {
|
|
state.status = error.reason;
|
|
state.current = null;
|
|
await event('info', `Archive ${error.reason}`, { chapter: ref.number });
|
|
throw error;
|
|
}
|
|
state.status = 'error';
|
|
state.lastError = { time: new Date().toISOString(), chapter: ref.number, message: error.message };
|
|
await event('error', error.message, { chapter: ref.number });
|
|
throw error;
|
|
}
|
|
completed.add(ref.id);
|
|
state.completed = [...completed];
|
|
state.jobCompleted = Number(state.jobCompleted || 0) + 1;
|
|
state.lastChapter = ref.number;
|
|
state.updated = new Date().toISOString();
|
|
state.total = incremental ? refs.length : catalog.chapters.length;
|
|
state.status = 'running';
|
|
state.current = null;
|
|
await writeJsonAtomic(stateFile, state);
|
|
const published = result.results[0].status === 'published';
|
|
if (published && incremental) {
|
|
publishedCount++;
|
|
if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] };
|
|
manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number), sourceKey: sourceKey(chapter.source.url, chapter.source.name) });
|
|
if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number });
|
|
}
|
|
await event('info', `Completed chapter ${ref.number} (${state.jobCompleted}/${state.total})`, { chapter: ref.number });
|
|
console.log(`[archive] complete ${ref.number}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`);
|
|
if (index < refs.length - 1) {
|
|
checkControl();
|
|
const pause = randomBetween(options.chapterMinMs, options.chapterMaxMs);
|
|
console.log(`[archive] chapter pause ${Math.round(pause / 1000)}s`);
|
|
await sleep(pause);
|
|
checkControl();
|
|
}
|
|
}
|
|
|
|
if (incremental) {
|
|
state.status = 'complete'; state.current = null;
|
|
await event('info', `Published ${publishedCount} chapter${publishedCount === 1 ? '' : 's'} incrementally`);
|
|
return { finalPath: manga?.path || null, publishedCount, selectedCount: refs.length };
|
|
}
|
|
|
|
const stagedManga = path.join(workRoot, cleanPart(catalog.title));
|
|
const finalManga = path.join(libraryRoot, cleanPart(catalog.title));
|
|
if (await exists(finalManga)) throw new Error(`Final manga folder already exists: ${finalManga}`);
|
|
await renameRetry(stagedManga, finalManga);
|
|
state.status = 'complete';
|
|
state.current = null;
|
|
await event('info', `Published ${finalManga}`);
|
|
await fs.rm(workRoot, { recursive: true, force: true });
|
|
if (options.onPublished) await options.onPublished({ mangaDir: finalManga, completeManga: true });
|
|
console.log(`[archive] PUBLISHED ${finalManga}`);
|
|
return { finalPath: finalManga, publishedCount: 1, selectedCount: catalog.chapters.length };
|
|
}
|
|
|
|
module.exports = { runArchive };
|