Keep source plugins in single scraper process

scraper/sources/common.js

@@ -0,0 +1,12 @@
+'use strict';
+
+const { ArchiveControlError } = require('../lib/control');
+
+function controlledHttp(http, shouldStop) {
+  return { async get(...args) {
+    const check = () => { const reason = shouldStop?.(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); };
+    check(); const response = await http.get(...args); check(); return response;
+  } };
+}
+
+module.exports = { controlledHttp };

scraper/sources/mangadex.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const { SlowHttp } = require('../lib/http');
+const { mangaDexInfo, discoverMangaDexCatalog } = require('../adapters/mangadex');
+const { runMangaDexArchive } = require('../jobs/mangadex-archive');
+const { controlledHttp } = require('./common');
+
+module.exports = {
+  id: 'mangadex', label: 'MangaDex', match: mangaDexInfo,
+  async discover(item, options) {
+    const http = new SlowHttp({ minMs: options.settings.minMs, maxMs: options.settings.maxMs,
+      maxRetries: 2, userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)',
+      onEvent: options.onEvent });
+    return discoverMangaDexCatalog(item.url, controlledHttp(http, options.shouldStop));
+  },
+  run(item, options) {
+    return runMangaDexArchive({
+      seriesUrl: item.url, title: item.title, libraryRoot: options.libraryRoot,
+      metadataSearch: item.metadataSearch || item.title,
+      minMs: options.settings.minMs, maxMs: options.settings.maxMs,
+      chapterMinMs: options.settings.chapterMinMs, chapterMaxMs: options.settings.chapterMaxMs,
+      shouldStop: options.shouldStop, scope: item.scope, existingPolicy: item.existingPolicy,
+      libraryMangaId: item.libraryMangaId, onPublished: options.onPublished, onCatalog: options.onCatalog,
+    });
+  },
+};

scraper/sources/weebcentral.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const { createWeebHttp } = require('../plugins/broker');
+const { discoverWeebCatalog, weebCentralInfo } = require('../adapters/weebcentral');
+const { runArchive } = require('../jobs/weebcentral-archive');
+const { controlledHttp } = require('./common');
+
+module.exports = {
+  id: 'weebcentral', label: 'WeebCentral', match: weebCentralInfo,
+  async discover(item, options) {
+    const http = createWeebHttp({ minMs: options.settings.minMs, maxMs: options.settings.maxMs,
+      maxRetries: 2, onEvent: options.onEvent });
+    return discoverWeebCatalog(item.url, controlledHttp(http, options.shouldStop));
+  },
+  run(item, options) {
+    return runArchive({
+      seriesUrl: item.url, libraryRoot: options.libraryRoot,
+      metadataSearch: item.metadataSearch || item.title,
+      minMs: options.settings.minMs, maxMs: options.settings.maxMs,
+      chapterMinMs: options.settings.chapterMinMs, chapterMaxMs: options.settings.chapterMaxMs,
+      shouldStop: options.shouldStop, scope: item.scope, existingPolicy: item.existingPolicy,
+      libraryMangaId: item.libraryMangaId, onPublished: options.onPublished, onCatalog: options.onCatalog,
+    });
+  },
+};