Add first-party MangaPill source and distinct volume releases

scraper/jobs/weebcentral-archive.js

@@ -40,14 +40,18 @@ async function renameRetry(from, to) {
 }
 
 async function runArchive(options) {
+  const source = options.source || 'weebcentral';
+  const sourceLabel = options.sourceLabel || 'WeebCentral';
+  const discoverCatalog = options.discoverCatalog || discoverWeebCatalog;
+  const discoverChapter = options.discoverChapter || discoverWeebChapter;
   const checkControl = () => {
     const reason = options.shouldStop && options.shouldStop();
     if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason));
   };
   const libraryRoot = path.resolve(options.libraryRoot || 'library');
-  const sourceId = options.seriesUrl.split('/series/')[1]?.split('/')[0];
-  if (!sourceId) throw new Error('Could not read WeebCentral series id');
-  const workRoot = path.join(libraryRoot, `.mcscraper-work-${sourceId}`);
+  const sourceId = options.sourceId || options.seriesUrl.split('/series/')[1]?.split('/')[0];
+  if (!sourceId) throw new Error(`Could not read ${sourceLabel} series id`);
+  const workRoot = path.join(libraryRoot, source === 'weebcentral' ? `.mcscraper-work-${sourceId}` : `.mcscraper-work-${source}-${sourceId}`);
   const stateFile = path.join(workRoot, 'job.json');
   const logFile = path.join(workRoot, 'job.log');
   await fs.mkdir(workRoot, { recursive: true });
@@ -72,13 +76,13 @@ async function runArchive(options) {
     minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2,
     onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }),
   };
-  const http = createWeebHttp(httpOptions);
+  const http = options.createHttp ? options.createHttp(httpOptions) : createWeebHttp(httpOptions);
 
   checkControl();
   console.log(`[archive] discovering catalog ${options.seriesUrl}`);
-  const catalog = await discoverWeebCatalog(options.seriesUrl, http);
+  const catalog = await discoverCatalog(options.seriesUrl, http);
   if (options.onCatalog) await options.onCatalog(catalog);
-  if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || 'WeebCentral lists no chapter links for this series.'), {
+  if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || `${sourceLabel} lists no chapter links for this series.`), {
     failureKind: catalog.chapterLinks ? 'adapter' : 'unavailable',
   });
   const keys = new Map();
@@ -92,20 +96,20 @@ async function runArchive(options) {
   const metadata = options.metadataSearch ? await lookupAniList(options.metadataSearch) : {};
   const series = {
     title: catalog.title, ...require('../../reading-direction').fallbackDirection(),
-    sources: [{ name: 'WeebCentral', url: options.seriesUrl }],
+    sources: [{ name: sourceLabel, url: options.seriesUrl }],
     metadata: { ...metadata, title: catalog.title },
     coverUrl: metadata['x-anilist']?.coverUrl,
-    scraper: { ids: { weebcentral: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } },
+    scraper: { ids: { [source]: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } },
   };
 
   let inventory = await scanLibrary(libraryRoot);
   let manga = findManga(inventory, {
-    source: 'weebcentral', seriesId: sourceId, sourceUrl: options.seriesUrl,
+    source, seriesId: sourceId, sourceUrl: options.seriesUrl,
     anilist: metadata['x-anilist']?.id, libraryMangaId: options.libraryMangaId,
   });
   if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`);
   const eligible = incremental
-    ? catalog.chapters.filter(ref => shouldDownload(manga, ref, 'weebcentral', options.existingPolicy))
+    ? catalog.chapters.filter(ref => shouldDownload(manga, ref, source, options.existingPolicy))
     : catalog.chapters;
   const refs = incremental ? selectByScope(eligible, options.scope) : eligible;
   state.status = 'running';
@@ -128,27 +132,30 @@ async function runArchive(options) {
   for (let index = 0; index < refs.length; index++) {
     checkControl();
     const ref = refs[index];
+    const display = ref.label || ref.number;
     if (!incremental && completed.has(ref.id)) {
       if (!incremental) console.log(`[archive] skip completed ${ref.number} (${index + 1}/${refs.length})`);
       continue;
     }
-    if (incremental && !shouldDownload(manga, ref, 'weebcentral', options.existingPolicy)) {
+    if (incremental && !shouldDownload(manga, ref, source, options.existingPolicy)) {
       state.jobCompleted++;
-      await event('info', `Skipped existing chapter ${ref.number}`, { chapter: ref.number });
+      await event('info', `Skipped existing ${display}`, { chapter: ref.number });
       continue;
     }
-    console.log(`[archive] inspect ${ref.number} (${index + 1}/${refs.length})`);
+    console.log(`[archive] inspect ${display} (${index + 1}/${refs.length})`);
     let chapter;
     let result;
     try {
       state.status = 'inspecting';
-      state.current = { number: ref.number, ...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: null };
-      await event('info', `Inspecting chapter ${ref.number}`, { chapter: ref.number });
-      chapter = await discoverWeebChapter(ref, http);
+      state.current = { number: ref.number, ...(ref.label ? { label: ref.label } : {}),
+        ...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1,
+        total: refs.length, downloadedPages: 0, expectedPages: null };
+      await event('info', `Inspecting ${display}`, { chapter: ref.number });
+      chapter = await discoverChapter(ref, http);
       state.status = 'downloading';
       state.current.expectedPages = chapter.pages.length;
-      await event('info', `Downloading chapter ${ref.number}: ${chapter.pages.length} pages`, { chapter: ref.number });
-      console.log(`[archive] download ${ref.number}: ${chapter.pages.length} pages`);
+      await event('info', `Downloading ${display}: ${chapter.pages.length} pages`, { chapter: ref.number });
+      console.log(`[archive] download ${display}: ${chapter.pages.length} pages`);
       result = await publishManifest({ version: 1, series, chapters: [chapter] }, {
         libraryRoot: incremental ? libraryRoot : workRoot,
         ...(incremental && manga ? { mangaDir: manga.path } : {}),
@@ -185,11 +192,12 @@ async function runArchive(options) {
     if (published && incremental) {
       publishedCount++;
       if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] };
-      manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number), sourceKey: sourceKey(chapter.source.url, chapter.source.name) });
+      manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number),
+        releaseKind: ref.releaseKind || null, sourceKey: sourceKey(chapter.source.url, chapter.source.name) });
       if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number });
     }
-    await event('info', `Completed chapter ${ref.number} (${state.jobCompleted}/${state.total})`, { chapter: ref.number });
-    console.log(`[archive] complete ${ref.number}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`);
+    await event('info', `Completed ${display} (${state.jobCompleted}/${state.total})`, { chapter: ref.number });
+    console.log(`[archive] complete ${display}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`);
     if (index < refs.length - 1) {
       checkControl();
       const pause = randomBetween(options.chapterMinMs, options.chapterMaxMs);