Keep source plugins in single scraper process

docker-compose.yml

@@ -80,9 +80,7 @@ services:
       SCRAPER_PORT: "8091"
       SCRAPER_TOKEN_FILE: /run/secrets/scraper_token
       PEER_DATA: /data/peering
-      SOURCE_PLUGIN_URL: ${SOURCE_PLUGIN_URL:-}
+    networks: [backend, egress]
-      MANGADEX_PLUGIN_URL: ${MANGADEX_PLUGIN_URL:-}
-    networks: [backend, egress, plugin_backend]
     healthcheck:
       test: [CMD, node, tools/healthcheck.js, scraper]
       interval: 30s
@@ -148,67 +146,9 @@ services:
       start_period: 30s
       retries: 3
 
-  source-plugin-weebcentral:
-    image: ${APP_IMAGE:-mangacircus:local}
-    build:
-      context: .
-      args:
-        VCS_REF: ${VCS_REF:-development}
-    profiles: [source-plugins]
-    command: [node, scraper/plugins/worker.js]
-    init: true
-    restart: unless-stopped
-    read_only: true
-    tmpfs: [/tmp]
-    security_opt: [no-new-privileges:true]
-    cap_drop: [ALL]
-    pids_limit: 64
-    mem_limit: 256m
-    environment:
-      SOURCE_PLUGIN_ID: weebcentral
-      SOURCE_PLUGIN_HOST: 0.0.0.0
-      SOURCE_PLUGIN_PORT: "8094"
-    networks: [plugin_backend]
-    healthcheck:
-      test: [CMD, node, tools/healthcheck-source-plugin.js]
-      interval: 30s
-      timeout: 5s
-      start_period: 15s
-      retries: 3
-
-  source-plugin-mangadex:
-    image: ${APP_IMAGE:-mangacircus:local}
-    build:
-      context: .
-      args:
-        VCS_REF: ${VCS_REF:-development}
-    profiles: [source-plugins]
-    command: [node, scraper/plugins/worker.js]
-    init: true
-    restart: unless-stopped
-    read_only: true
-    tmpfs: [/tmp]
-    security_opt: [no-new-privileges:true]
-    cap_drop: [ALL]
-    pids_limit: 64
-    mem_limit: 256m
-    environment:
-      SOURCE_PLUGIN_ID: mangadex
-      SOURCE_PLUGIN_HOST: 0.0.0.0
-      SOURCE_PLUGIN_PORT: "8094"
-    networks: [plugin_backend]
-    healthcheck:
-      test: [CMD, node, tools/healthcheck-source-plugin.js]
-      interval: 30s
-      timeout: 5s
-      start_period: 15s
-      retries: 3
-
 networks:
   backend:
     internal: true
-  plugin_backend:
-    internal: true
   egress: {}
 
 secrets:

docs/architecture.md

@@ -17,6 +17,7 @@ Related documents:
 - [settings-navigation.md](settings-navigation.md): User/Admin workspace, scraper tabs and deep links.
 - [peering.md](peering.md): optional private server-to-server catalog and transfer.
 - [discovery.md](discovery.md): reader-owned, opt-in deterministic discovery feed and its limits.
+- [source-modules.md](source-modules.md): adding and maintaining first-party hosters inside the single scraper process.
 
 ## 1. Principles
 

docs/scraper-service.md

@@ -5,31 +5,23 @@ It owns the durable queue and is the only process allowed to run a queued archiv
 The browser never contacts it directly: authenticated admin routes in `server.js`
 proxy commands to its private HTTP API.
 
-## First-party source parser pilot
+## In-process source modules
 
-The WeebCentral and MangaDex parsers have reviewed first-party modules in
+There is one scraper daemon/container and one durable queue. WeebCentral and
-`scraper/plugins/`. By default their existing in-process paths remain in use.
+MangaDex source modules are registered explicitly in `scraper/adapters/index.js`.
-To test process isolation in Compose, start the `source-plugins` profile and
+Their descriptors live in `scraper/sources/`; they call source-specific catalog
-set `SOURCE_PLUGIN_URL=http://source-plugin-weebcentral:8094` and/or
+and archive code in the same process. Pure parsing functions in
-`MANGADEX_PLUGIN_URL=http://source-plugin-mangadex:8094` for the scraper.
+`scraper/plugins/` handle source page/API shapes. No source parser has a separate
-Each URL selects its source independently. A selected worker that is unavailable
+container, HTTP endpoint, runtime download, or independent queue. Only reviewed
-causes that source's discovery to fail closed and retain its checkpoint. Remove
+first-party modules ship in the image; Uchiyomi packages are not executed.
-its URL to roll back. Do not restart a live scraper merely to try this option;
-use the normal checkpoint-safe deployment procedure.
 
-Each worker has no published port, egress network, library/data mount, or token.
+The scraper's paced HTTP client fetches source content. Parsing is an ordinary
-Only the existing rate-limited scraper HTTP client fetches remote HTML/images.
+function call. The registry checks source URLs and validates parsed output before
-The worker accepts bounded HTML for four parsing steps over its internal-only
+the scraper follows links. WeebCentral catalog requests use a pinned-DNS HTTPS
-network. It loads only modules listed in the static first-party registry; there
+transport that validates redirects and stops on access errors. Image hosts are
-is no runtime plugin download or execution of Uchiyomi packages. This is a
+not restricted to a reviewed CDN allowlist yet. Existing queue state, source
-parser-isolation pilot, **not yet** a complete generic source-plugin contract
+delays, chapter delays, and partial-download checkpoints are unchanged by this
-or migration of either archive runner. Parsed catalog and image URLs are checked
+code organization. See [adding a source](source-modules.md) for the full path.
-before use, and parser requests/responses are size-bounded. When the worker is
-enabled, WeebCentral requests use a pinned-DNS HTTPS transport that validates
-each redirect and stops on access errors; the ordinary in-process path is not
-changed. Public image hosts are not yet restricted to a reviewed CDN allowlist.
-Keep the profile off in production until a separate deployment validation is
-completed.
 
 ## Start and storage
 
@@ -218,7 +210,7 @@ the shared token.
 
 ## Adapter contract
 
-Each source plugin is registered in `scraper/adapters/index.js` with:
+Each first-party source module is registered in `scraper/adapters/index.js` with:
 
 ```js
 {

docs/source-modules.md

@@ -0,0 +1,42 @@
+# Adding a source to the single scraper
+
+MangaCircus uses **one scraper process**. A source module is reviewed first-party
+code bundled into that process, not a container or a downloaded extension.
+`scraper/adapters/index.js` is the explicit registration point. The library,
+queue, pacing settings, checkpointing, source-error policy, and publication
+remain shared infrastructure. Do not add another queue writer for a new hoster.
+
+## Source boundary
+
+| Concern | Location | New-source responsibility |
+| --- | --- | --- |
+| URL recognition and adapter contract | `scraper/sources/<hoster>.js` | `id`, `label`, `match`, `discover`, `run` |
+| Catalog and chapter discovery | `scraper/adapters/<hoster>.js` | Map source responses to stable chapter IDs, metadata and page URLs |
+| Source HTML/API parsing | `scraper/plugins/<hoster>.js` | Pure functions; no network or filesystem writes |
+| Parser operation/URL validation | `scraper/plugins/manifest.js`, `registry.js`, `validation.js` | Allow only known operations and expected hosts/links |
+| Resumable archive workflow | `scraper/jobs/<hoster>-archive.js` | Use the common publisher, inventory, pacing and checkpoints |
+| Registration | `scraper/adapters/index.js` | Add one explicit `require` to `adapters` |
+
+The module contract is shown in [scraper-service.md](scraper-service.md#adapter-contract).
+Start from the closest existing hoster. `match(url)` returns `null` for unrelated
+URLs and `{ source, id, title, url }` for an exact supported series URL. `discover`
+must be read-only and return a catalog. `run` must honor scope, existing-content
+policy, `shouldStop`, source-specific delays, and publication callbacks. Stable
+source series/chapter IDs are essential for retries and duplicate detection.
+
+## Before enabling a hoster
+
+1. Document its permitted access method, terms and source-specific pacing. Do
+   not work around 401, 403, 429, CAPTCHA, challenges or source protections.
+2. Add fixture-backed tests for URL matching, catalog, empty/partial catalogs,
+   chapter pages, unsafe links, and duplicate release handling. No test should
+   contact the live hoster by default.
+3. Test pause/resume from a partial chapter and an existing library target with
+   the common publisher; verify one source lane and no parallel image workers.
+4. Run `npm run test:scraper` and an isolated Compose smoke test. Review and deploy
+   the single scraper image with a checkpoint-safe procedure; never start another
+   daemon against the live queue.
+
+Removing a hoster from registration stops new matches; it does **not** delete its
+library chapters or queued/checkpointed work. Inspect affected queue items and
+preserve checkpoints before disabling a source in a release.

scraper/adapters/index.js

@@ -1,92 +1,13 @@
 'use strict';
 
-const { runArchive } = require('../jobs/weebcentral-archive');
+// First-party registration only; never load executable source code from a
-const { runMangaDexArchive } = require('../jobs/mangadex-archive');
+// remote catalog or scan a writable directory for modules.
-const { SlowHttp } = require('../lib/http');
+const adapters = Object.freeze([
-const { createWeebHttp } = require('../plugins/broker');
+  require('../sources/weebcentral'),
-const { discoverWeebCatalog } = require('./weebcentral');
+  require('../sources/mangadex'),
-const { mangaDexInfo, discoverMangaDexCatalog } = require('./mangadex');
+]);
-const { ArchiveControlError } = require('../lib/control');
 const { adapterForPeer } = require('../../peering/adapter');
 
-function controlledHttp(http, shouldStop) {
-  return { async get(...args) {
-    const check = () => { const reason = shouldStop?.(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); };
-    check(); const response = await http.get(...args); check(); return response;
-  } };
-}
-
-function weebCentralInfo(value) {
-  let url;
-  try { url = new URL(String(value)); } catch { return null; }
-  if (url.protocol !== 'https:' || url.hostname !== 'weebcentral.com') return null;
-  const match = /^\/series\/([A-Z0-9]+)(?:\/([^/?#]+))?\/?$/i.exec(url.pathname);
-  if (!match) return null;
-  const title = decodeURIComponent(match[2] || match[1]).replace(/[-_]+/g, ' ').trim();
-  return { source: 'weebcentral', id: match[1].toUpperCase(), title, url: `https://weebcentral.com/series/${match[1]}${match[2] ? `/${match[2]}` : ''}` };
-}
-
-const adapters = [{
-  id: 'weebcentral',
-  label: 'WeebCentral',
-  match: weebCentralInfo,
-  async discover(item, options) {
-    const httpOptions = {
-      minMs: options.settings.minMs, maxMs: options.settings.maxMs, maxRetries: 2,
-      onEvent: options.onEvent,
-    };
-    const http = process.env.SOURCE_PLUGIN_URL ? createWeebHttp(httpOptions) : new SlowHttp(httpOptions);
-    return discoverWeebCatalog(item.url, controlledHttp(http, options.shouldStop));
-  },
-  run(item, options) {
-    return runArchive({
-      seriesUrl: item.url,
-      libraryRoot: options.libraryRoot,
-      metadataSearch: item.metadataSearch || item.title,
-      minMs: options.settings.minMs,
-      maxMs: options.settings.maxMs,
-      chapterMinMs: options.settings.chapterMinMs,
-      chapterMaxMs: options.settings.chapterMaxMs,
-      shouldStop: options.shouldStop,
-      scope: item.scope,
-      existingPolicy: item.existingPolicy,
-      libraryMangaId: item.libraryMangaId,
-      onPublished: options.onPublished,
-      onCatalog: options.onCatalog,
-    });
-  },
-}, {
-  id: 'mangadex',
-  label: 'MangaDex',
-  match: mangaDexInfo,
-  async discover(item, options) {
-    const http = new SlowHttp({
-      minMs: options.settings.minMs, maxMs: options.settings.maxMs, maxRetries: 2,
-      userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)',
-      onEvent: options.onEvent,
-    });
-    return discoverMangaDexCatalog(item.url, controlledHttp(http, options.shouldStop));
-  },
-  run(item, options) {
-    return runMangaDexArchive({
-      seriesUrl: item.url,
-      title: item.title,
-      libraryRoot: options.libraryRoot,
-      metadataSearch: item.metadataSearch || item.title,
-      minMs: options.settings.minMs,
-      maxMs: options.settings.maxMs,
-      chapterMinMs: options.settings.chapterMinMs,
-      chapterMaxMs: options.settings.chapterMaxMs,
-      shouldStop: options.shouldStop,
-      scope: item.scope,
-      existingPolicy: item.existingPolicy,
-      libraryMangaId: item.libraryMangaId,
-      onPublished: options.onPublished,
-      onCatalog: options.onCatalog,
-    });
-  },
-}];
-
 function adapterFor(url) {
   const peer = adapterForPeer(url);
   if (peer) return peer;
@@ -97,4 +18,6 @@ function adapterFor(url) {
   return null;
 }
 
-module.exports = { adapters, adapterFor, weebCentralInfo, mangaDexInfo };
+module.exports = { adapters, adapterFor,
+  weebCentralInfo: require('./weebcentral').weebCentralInfo,
+  mangaDexInfo: require('./mangadex').mangaDexInfo };

scraper/adapters/mangadex.js

@@ -1,8 +1,8 @@
 'use strict';
 
 const API = 'https://api.mangadex.org';
-const { specialKey } = require('../../chapter-identity');
+const { parse } = require('../plugins/registry');
-const { parseSource } = require('../plugins/client');
+const { localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary, pageUrls } = require('../plugins/mangadex');
 
 function mangaDexInfo(value) {
   let url;
@@ -15,78 +15,6 @@ function mangaDexInfo(value) {
   return { source: 'mangadex', id, title, url: `https://mangadex.org/title/${id}${match[2] ? `/${match[2]}` : ''}` };
 }
 
-function localizedTitle(attributes = {}) {
-  const titles = attributes.title || {};
-  return titles.en || titles['ja-ro'] || titles.ja || Object.values(titles).find(Boolean) || 'Untitled';
-}
-
-function chapterNumber(value) {
-  const raw = String(value ?? '').trim();
-  if (!/^\d+(?:\.\d+)?$/.test(raw)) return null;
-  const [whole, decimal = ''] = raw.split('.');
-  const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0';
-  const normalizedDecimal = decimal.replace(/0+$/, '');
-  return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole;
-}
-
-function groupFor(resource) {
-  const relation = (resource.relationships || []).find(item => item.type === 'scanlation_group');
-  return relation ? { id: relation.id, name: relation.attributes?.name || 'Unknown group' } : null;
-}
-
-function releaseRank(resource) {
-  const group = groupFor(resource)?.name || '';
-  const credited = /official|license/i.test(group) ? 0 : group ? 1 : 2;
-  const date = Date.parse(resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt || '') || Number.MAX_SAFE_INTEGER;
-  return [credited, date, resource.id];
-}
-
-function compareRank(left, right) {
-  const a = releaseRank(left), b = releaseRank(right);
-  return a[0] - b[0] || a[1] - b[1] || String(a[2]).localeCompare(String(b[2]));
-}
-
-function selectChapterReleases(resources = [], specialLabel = 'Special') {
-  const selected = new Map();
-  for (const resource of resources) {
-    if (resource.attributes?.externalUrl || Number(resource.attributes?.pages || 0) < 1) continue;
-    const number = chapterNumber(resource.attributes?.chapter) || specialKey(resource.id);
-    const prior = selected.get(number);
-    if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource);
-  }
-  return [...selected.entries()].map(([number, resource]) => {
-    const group = groupFor(resource);
-    return {
-      id: resource.id, number, title: resource.attributes?.title || '',
-      ...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}),
-      ...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}),
-      pages: Number(resource.attributes?.pages || 0),
-      added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt,
-      group,
-      url: `https://mangadex.org/chapter/${resource.id}`,
-    };
-  }).sort((a, b) => Number(!!a.special) - Number(!!b.special) || (a.special ? a.number.localeCompare(b.number) : Number(a.number) - Number(b.number) || a.number.localeCompare(b.number)));
-}
-
-function availabilitySummary(attributes, releases, chapters, externalReleases = null) {
-  const numbered = chapters.filter(ref => !ref.special);
-  const expected = chapterNumber(attributes.lastChapter);
-  const maximum = numbered.length ? Math.max(...numbered.map(ref => Number(ref.number))) : null;
-  // Reaching the final chapter number is not full coverage when the catalog
-  // contains fewer distinct numbered releases than the expected chapter span.
-  const expectedCount = expected && /^\d+$/.test(expected) ? Number(expected) : null;
-  const sparse = expectedCount && numbered.length < expectedCount;
-  const reason = chapters.length ? null : externalReleases > 0 ? 'external-only'
-    : Array.isArray(attributes.availableTranslatedLanguages) && !attributes.availableTranslatedLanguages.includes('en') ? 'no-english' : 'no-hosted-pages';
-  const message = reason === 'external-only' ? `MangaDex lists ${externalReleases} English external-link releases, but no hosted English pages. External links are not downloaded.`
-    : reason === 'no-english' ? 'This MangaDex edition has no English releases listed.'
-      : reason ? 'This MangaDex edition has no hosted English releases with pages.' : null;
-  return { language: 'en', hostedReleases: releases.filter(r => !r.attributes?.externalUrl && Number(r.attributes?.pages) > 0).length,
-    selectedChapters: chapters.length, numberedChapters: numbered.length, specialReleases: chapters.length - numbered.length,
-    externalReleases, expectedLastChapter: expected, highestHostedChapter: maximum,
-    coverage: expected && (maximum === null || maximum < Number(expected) || sparse) ? 'partial' : 'unknown', reason, message };
-}
-
 async function jsonGet(http, url) {
   const response = await http.get(url, { accept: 'application/json' });
   const body = await response.json();
@@ -99,10 +27,7 @@ async function discoverMangaDexCatalog(seriesUrl, http) {
   if (!info) throw new Error('Invalid MangaDex title URL');
   const manga = (await jsonGet(http, `${API}/manga/${info.id}`)).data;
   if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
-  const isolated = !!process.env.MANGADEX_PLUGIN_URL;
+  const metadata = parse('mangadex', 'metadata', { data: manga, url: `${API}/manga/${info.id}` });
-  const pluginMetadata = isolated
-    ? await parseSource('mangadex', 'metadata', { html: JSON.stringify(manga), url: `${API}/manga/${info.id}` })
-    : null;
 
   const releases = [];
   let offset = 0;
@@ -112,67 +37,45 @@ async function discoverMangaDexCatalog(seriesUrl, http) {
     params.append('includes[]', 'scanlation_group');
     params.append('order[chapter]', 'asc');
     const page = await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`);
-    if (isolated) {
+    releases.push(...parse('mangadex', 'feed', { data: page.data || [], url: `${API}/manga/${info.id}/feed` }).releases);
-      const parsed = await parseSource('mangadex', 'feed', { html: JSON.stringify(page.data || []), url: `${API}/manga/${info.id}/feed` });
-      releases.push(...parsed.releases);
-    } else releases.push(...(page.data || []));
     offset += page.data?.length || 0;
     if (!page.data?.length || offset >= Number(page.total || 0)) break;
   }
 
-  const title = isolated ? pluginMetadata.title : localizedTitle(manga.attributes);
+  const { title, specialLabel } = metadata;
-  const specialLabel = isolated ? pluginMetadata.specialLabel : /pre.serialization|pilot/i.test(title) ? 'Pilot'
+  const parseCatalog = externalReleases => parse('mangadex', 'catalog', {
-    : (manga.attributes?.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
+    data: { attributes: metadata.availabilityAttributes, releases, specialLabel, externalReleases },
-  const pluginCatalog = async externalReleases => parseSource('mangadex', 'catalog', {
-    html: JSON.stringify({ attributes: pluginMetadata.availabilityAttributes, releases, specialLabel, externalReleases }),
     url: `${API}/manga/${info.id}`,
   });
-  let parsed = isolated ? await pluginCatalog(null) : null;
+  let parsed = parseCatalog(null);
-  const chapters = isolated ? parsed.chapters : selectChapterReleases(releases, specialLabel);
+  const chapters = parsed.chapters;
   // External-only is a separate feed mode, not a combined hosted+external catalog.
   let externalReleases = null;
   if (!chapters.length) {
     const params = new URLSearchParams({ limit: '1', includeFutureUpdates: '0', includeExternalUrl: '1' });
     params.append('translatedLanguage[]', 'en');
     externalReleases = Number((await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`)).total || 0);
-    if (isolated) parsed = await pluginCatalog(externalReleases);
+    parsed = parseCatalog(externalReleases);
   }
   return {
     id: info.id,
     title,
-    originalLanguage: isolated ? pluginMetadata.originalLanguage : manga.attributes?.originalLanguage || null,
+    originalLanguage: metadata.originalLanguage,
-    links: isolated ? pluginMetadata.links : manga.attributes?.links || {},
+    links: metadata.links,
-    availability: isolated ? parsed.availability : availabilitySummary(manga.attributes || {}, releases, chapters, externalReleases),
+    availability: parsed.availability,
     chapters,
   };
 }
 
-function pageUrls(atHome, chapterUrl, expectedPages) {
-  const base = new URL(atHome.baseUrl);
-  if (base.protocol !== 'https:' || !(base.hostname === 'uploads.mangadex.org' || base.hostname.endsWith('.mangadex.network'))) {
-    throw new Error(`Unexpected MangaDex@Home host: ${base.hostname}`);
-  }
-  const hash = String(atHome.chapter?.hash || '');
-  const files = atHome.chapter?.data;
-  if (!hash || !Array.isArray(files) || !files.length) throw new Error('MangaDex@Home returned no pages');
-  if (expectedPages && files.length !== expectedPages) throw new Error(`MangaDex page count changed: expected ${expectedPages}, received ${files.length}`);
-  return files.map(file => ({
-    url: new URL(`/data/${encodeURIComponent(hash)}/${encodeURIComponent(file)}`, base).href,
-    headers: { referer: chapterUrl },
-  }));
-}
-
 async function discoverMangaDexChapter(ref, http) {
   const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`);
   const groupName = ref.group?.name ? ` · ${ref.group.name}` : '';
-  const parsed = process.env.MANGADEX_PLUGIN_URL
+  const parsed = parse('mangadex', 'atHome', { data: { response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }, url: `${API}/at-home/server/${ref.id}` });
-    ? await parseSource('mangadex', 'atHome', { html: JSON.stringify({ response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }), url: `${API}/at-home/server/${ref.id}` })
-    : null;
   return {
     number: ref.number, title: ref.title || '', added: ref.added,
     ...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}),
     source: { name: `MangaDex${groupName}`, url: ref.url },
-    pages: parsed ? parsed.pages : pageUrls(atHome, ref.url, ref.pages),
+    pages: parsed.pages,
   };
 }
 

scraper/adapters/weebcentral.js

@@ -3,16 +3,26 @@
 const { SlowHttp } = require('../lib/http');
 const { fallbackDirection } = require('../../reading-direction');
 const parser = require('../plugins/weebcentral');
-const { parseSource } = require('../plugins/client');
+const { parse } = require('../plugins/registry');
+
+function weebCentralInfo(value) {
+  let url;
+  try { url = new URL(String(value)); } catch { return null; }
+  if (url.protocol !== 'https:' || url.hostname !== 'weebcentral.com') return null;
+  const match = /^\/series\/([A-Z0-9]+)(?:\/([^/?#]+))?\/?$/i.exec(url.pathname);
+  if (!match) return null;
+  const title = decodeURIComponent(match[2] || match[1]).replace(/[-_]+/g, ' ').trim();
+  return { source: 'weebcentral', id: match[1].toUpperCase(), title, url: `https://weebcentral.com/series/${match[1]}${match[2] ? `/${match[2]}` : ''}` };
+}
 
 const parseChapters = (html, baseUrl) => parser.catalog({ html, url: baseUrl }).chapters;
 const parsePageUrls = (html, baseUrl) => parser.pages({ html, url: baseUrl }).pages;
 
 async function discoverWeebCatalog(seriesUrl, http) {
   const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text();
-  const { title, listUrl } = await parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl });
+  const { title, listUrl } = parse('weebcentral', 'series', { html: seriesHtml, url: seriesUrl });
   const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html' })).text() : seriesHtml;
-  const parsed = await parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl });
+  const parsed = parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl });
   const chapterLinks = parsed.chapterLinks;
   const chapters = parsed.chapters.sort((a, b) =>
     Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number));
@@ -24,9 +34,9 @@ async function discoverWeebCatalog(seriesUrl, http) {
 
 async function discoverWeebChapter(chapter, http) {
   const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text();
-  const { manifestUrl, added } = await parseSource('weebcentral', 'chapter', { html: pageHtml, url: chapter.url });
+  const { manifestUrl, added } = parse('weebcentral', 'chapter', { html: pageHtml, url: chapter.url });
   const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip&current_page=1`, { accept: 'text/html' })).text();
-  const parsed = await parseSource('weebcentral', 'pages', { html: fragment, url: manifestUrl });
+  const parsed = parse('weebcentral', 'pages', { html: fragment, url: manifestUrl });
   const pages = parsed.pages.map(url => ({ url, headers: { referer: chapter.url } }));
   if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`);
   return {
@@ -55,4 +65,4 @@ async function discoverWeebCentral(seriesUrl, options = {}) {
   };
 }
 
-module.exports = { discoverWeebCentral, discoverWeebCatalog, discoverWeebChapter, parseChapters, parsePageUrls };
+module.exports = { weebCentralInfo, discoverWeebCentral, discoverWeebCatalog, discoverWeebChapter, parseChapters, parsePageUrls };

scraper/jobs/weebcentral-archive.js

@@ -2,7 +2,6 @@
 
 const fs = require('fs').promises;
 const path = require('path');
-const { SlowHttp } = require('../lib/http');
 const { createWeebHttp } = require('../plugins/broker');
 const { sleep } = require('../lib/delay');
 const { ArchiveControlError } = require('../lib/control');
@@ -73,7 +72,7 @@ async function runArchive(options) {
     minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2,
     onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }),
   };
-  const http = process.env.SOURCE_PLUGIN_URL ? createWeebHttp(httpOptions) : new SlowHttp(httpOptions);
+  const http = createWeebHttp(httpOptions);
 
   checkControl();
   console.log(`[archive] discovering catalog ${options.seriesUrl}`);

scraper/plugins/client.js

@@ -1,35 +0,0 @@
-'use strict';
-
-const { parse } = require('./registry');
-const { validate } = require('./validation');
-const MAX_MESSAGE_BYTES = 2 * 1024 * 1024;
-
-async function boundedJson(response) {
-  const chunks = [];
-  let bytes = 0;
-  for await (const chunk of response.body) {
-    bytes += chunk.length;
-    if (bytes > MAX_MESSAGE_BYTES) throw new Error('Source plugin response exceeds limit');
-    chunks.push(chunk);
-  }
-  return JSON.parse(Buffer.concat(chunks).toString('utf8'));
-}
-
-async function parseSource(plugin, operation, input) {
-  const endpoint = plugin === 'mangadex' ? process.env.MANGADEX_PLUGIN_URL : process.env.SOURCE_PLUGIN_URL;
-  if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input));
-  const url = new URL('/v1/parse', endpoint);
-  if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint');
-  const body = JSON.stringify({ plugin, operation, input });
-  if (Buffer.byteLength(body) > MAX_MESSAGE_BYTES) throw new Error('Source plugin request exceeds limit');
-  const response = await fetch(url, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json' },
-    body,
-    signal: AbortSignal.timeout(10000),
-  });
-  if (!response.ok) throw new Error(`Source parser unavailable (${response.status}); checkpoint preserved`);
-  return validate(plugin, operation, await boundedJson(response));
-}
-
-module.exports = { parseSource };

scraper/plugins/mangadex.js

@@ -87,8 +87,7 @@ function pageUrls(atHome, chapterUrl, expectedPages) {
   }));
 }
 
-function metadata({ html }) {
+function metadata({ data: manga }) {
-  const manga = JSON.parse(html);
   if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
   const attributes = manga.attributes || {};
   const title = localizedTitle(attributes);
@@ -101,8 +100,7 @@ function metadata({ html }) {
     } };
 }
 
-function feed({ html }) {
+function feed({ data: releases }) {
-  const releases = JSON.parse(html);
   if (!Array.isArray(releases)) throw new Error('Invalid MangaDex feed');
   return { releases: releases.map(resource => ({
     id: resource.id,
@@ -116,15 +114,15 @@ function feed({ html }) {
   })) };
 }
 
-function catalog({ html }) {
+function catalog({ data }) {
-  const { attributes, releases, specialLabel, externalReleases } = JSON.parse(html);
+  const { attributes, releases, specialLabel, externalReleases } = data;
   if (!attributes || !Array.isArray(releases)) throw new Error('Invalid MangaDex catalog input');
   const chapters = selectChapterReleases(releases, specialLabel);
   return { chapters, availability: availabilitySummary(attributes, releases, chapters, externalReleases) };
 }
 
-function atHome({ html }) {
+function atHome({ data }) {
-  const { response, chapterUrl, expectedPages } = JSON.parse(html);
+  const { response, chapterUrl, expectedPages } = data;
   return { pages: pageUrls(response, chapterUrl, expectedPages) };
 }
 

scraper/plugins/manifest.js

@@ -4,14 +4,14 @@
 // source may add executable modules at runtime.
 const manifests = Object.freeze({
   weebcentral: Object.freeze({
-    id: 'weebcentral', version: 1, label: 'WeebCentral', status: 'pilot',
+    id: 'weebcentral', version: 1, label: 'WeebCentral', status: 'in-process',
     sourceHosts: Object.freeze(['weebcentral.com']),
     implementation: 'first-party',
     provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied',
     operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']),
   }),
   mangadex: Object.freeze({
-    id: 'mangadex', version: 1, label: 'MangaDex', status: 'pilot',
+    id: 'mangadex', version: 1, label: 'MangaDex', status: 'in-process',
     sourceHosts: Object.freeze(['api.mangadex.org']),
     implementation: 'first-party',
     provenance: 'MangaCircus existing MangaDex adapter; no Uchiyomi code copied',

scraper/plugins/registry.js

@@ -2,6 +2,7 @@
 
 // Reviewed first-party modules only. No filesystem scanning or runtime installs.
 const { manifests } = require('./manifest');
+const { validate } = require('./validation');
 const plugins = Object.freeze({
   weebcentral: require('./weebcentral'),
   mangadex: require('./mangadex'),
@@ -11,13 +12,14 @@ function parse(plugin, operation, input) {
   const manifest = manifests[plugin];
   const handler = manifest?.operations.includes(operation) ? plugins[plugin]?.[operation] : null;
   if (typeof handler !== 'function') throw new Error('Unknown source plugin or operation');
-  if (!input || typeof input !== 'object' || typeof input.html !== 'string' || typeof input.url !== 'string') {
+  if (!input || typeof input !== 'object' || typeof input.url !== 'string'
+    || (typeof input.html !== 'string' && !(plugin === 'mangadex' && input.data && typeof input.data === 'object'))) {
     throw new Error('Invalid parser input');
   }
   const url = new URL(input.url);
   if (!['https:', 'http:'].includes(url.protocol)) throw new Error('Invalid source URL');
   if (!manifest.sourceHosts.includes(url.hostname)) throw new Error('Source URL does not belong to plugin');
-  return handler({ html: input.html, url: url.href });
+  return validate(plugin, operation, handler({ html: input.html, data: input.data, url: url.href }));
 }
 
 module.exports = { parse, manifests };

scraper/plugins/worker.js

@@ -1,49 +0,0 @@
-'use strict';
-
-const http = require('node:http');
-const { parse } = require('./registry');
-const MAX_BODY_BYTES = 2 * 1024 * 1024;
-
-function createServer(pluginId = process.env.SOURCE_PLUGIN_ID || null) {
-  return http.createServer((request, response) => {
-    if (request.method === 'GET' && request.url === '/health') {
-      response.writeHead(200, { 'content-type': 'application/json' });
-      response.end(JSON.stringify({ ok: true }));
-      return;
-    }
-    if (request.method !== 'POST' || request.url !== '/v1/parse') {
-      response.writeHead(404).end();
-      return;
-    }
-    let bytes = 0;
-    const chunks = [];
-    request.on('data', chunk => {
-      bytes += chunk.length;
-      if (bytes > MAX_BODY_BYTES) {
-        response.writeHead(413).end();
-        request.destroy();
-        return;
-      }
-      chunks.push(chunk);
-    });
-    request.on('end', () => {
-      if (bytes > MAX_BODY_BYTES) return;
-      try {
-        const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8'));
-        if (pluginId && plugin !== pluginId) throw new Error('Plugin worker does not serve this source');
-        const output = parse(plugin, operation, input);
-        response.writeHead(200, { 'content-type': 'application/json' });
-        response.end(JSON.stringify(output));
-      } catch (error) {
-        response.writeHead(400, { 'content-type': 'application/json' });
-        response.end(JSON.stringify({ error: error.message }));
-      }
-    });
-  });
-}
-
-if (require.main === module) {
-  createServer().listen(Number(process.env.SOURCE_PLUGIN_PORT || 8094), process.env.SOURCE_PLUGIN_HOST || '127.0.0.1');
-}
-
-module.exports = { createServer };

scraper/sources/common.js

@@ -0,0 +1,12 @@
+'use strict';
+
+const { ArchiveControlError } = require('../lib/control');
+
+function controlledHttp(http, shouldStop) {
+  return { async get(...args) {
+    const check = () => { const reason = shouldStop?.(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); };
+    check(); const response = await http.get(...args); check(); return response;
+  } };
+}
+
+module.exports = { controlledHttp };

scraper/sources/mangadex.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const { SlowHttp } = require('../lib/http');
+const { mangaDexInfo, discoverMangaDexCatalog } = require('../adapters/mangadex');
+const { runMangaDexArchive } = require('../jobs/mangadex-archive');
+const { controlledHttp } = require('./common');
+
+module.exports = {
+  id: 'mangadex', label: 'MangaDex', match: mangaDexInfo,
+  async discover(item, options) {
+    const http = new SlowHttp({ minMs: options.settings.minMs, maxMs: options.settings.maxMs,
+      maxRetries: 2, userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)',
+      onEvent: options.onEvent });
+    return discoverMangaDexCatalog(item.url, controlledHttp(http, options.shouldStop));
+  },
+  run(item, options) {
+    return runMangaDexArchive({
+      seriesUrl: item.url, title: item.title, libraryRoot: options.libraryRoot,
+      metadataSearch: item.metadataSearch || item.title,
+      minMs: options.settings.minMs, maxMs: options.settings.maxMs,
+      chapterMinMs: options.settings.chapterMinMs, chapterMaxMs: options.settings.chapterMaxMs,
+      shouldStop: options.shouldStop, scope: item.scope, existingPolicy: item.existingPolicy,
+      libraryMangaId: item.libraryMangaId, onPublished: options.onPublished, onCatalog: options.onCatalog,
+    });
+  },
+};

scraper/sources/weebcentral.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const { createWeebHttp } = require('../plugins/broker');
+const { discoverWeebCatalog, weebCentralInfo } = require('../adapters/weebcentral');
+const { runArchive } = require('../jobs/weebcentral-archive');
+const { controlledHttp } = require('./common');
+
+module.exports = {
+  id: 'weebcentral', label: 'WeebCentral', match: weebCentralInfo,
+  async discover(item, options) {
+    const http = createWeebHttp({ minMs: options.settings.minMs, maxMs: options.settings.maxMs,
+      maxRetries: 2, onEvent: options.onEvent });
+    return discoverWeebCatalog(item.url, controlledHttp(http, options.shouldStop));
+  },
+  run(item, options) {
+    return runArchive({
+      seriesUrl: item.url, libraryRoot: options.libraryRoot,
+      metadataSearch: item.metadataSearch || item.title,
+      minMs: options.settings.minMs, maxMs: options.settings.maxMs,
+      chapterMinMs: options.settings.chapterMinMs, chapterMaxMs: options.settings.chapterMaxMs,
+      shouldStop: options.shouldStop, scope: item.scope, existingPolicy: item.existingPolicy,
+      libraryMangaId: item.libraryMangaId, onPublished: options.onPublished, onCatalog: options.onCatalog,
+    });
+  },
+};

scraper/test/source-mangadex-plugin.test.js

@@ -2,8 +2,8 @@
 
 const test = require('node:test');
 const assert = require('node:assert/strict');
-const { createServer } = require('../plugins/worker');
 const { discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex');
+const { parse } = require('../plugins/registry');
 
 const mangaId = '123e4567-e89b-42d3-a456-426614174000';
 const chapterIds = [
@@ -29,27 +29,12 @@ function fakeHttp() {
   } };
 }
 
-test('MangaDex isolated parser matches the existing catalog and page behavior', async () => {
+test('MangaDex parses a catalog and pages in the scraper process', async () => {
-  const server = createServer('mangadex');
+  const catalog = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
-  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  const chapter = await discoverMangaDexChapter(catalog.chapters[0], fakeHttp());
-  const previous = process.env.MANGADEX_PLUGIN_URL;
+  assert.equal(catalog.title, 'Fixture');
-  delete process.env.MANGADEX_PLUGIN_URL;
+  assert.deepEqual(catalog.chapters.map(ref => ref.number), ['1', '2']);
-  try {
+  assert.equal(chapter.pages.length, 2);
-    const baseline = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
+  assert.match(chapter.pages[0].url, /^https:\/\/uploads\.mangadex\.org\/data\//);
-    const baselineChapter = await discoverMangaDexChapter(baseline.chapters[0], fakeHttp());
+  assert.throws(() => parse('mangadex', 'metadata', { data: manga.data, url: 'https://other.example/manga/x' }), /does not belong/);
-    process.env.MANGADEX_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
-    const isolated = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
-    const isolatedChapter = await discoverMangaDexChapter(isolated.chapters[0], fakeHttp());
-    assert.deepEqual(isolated, baseline);
-    assert.deepEqual(isolatedChapter, baselineChapter);
-    const rejected = await fetch(`${process.env.MANGADEX_PLUGIN_URL}/v1/parse`, {
-      method: 'POST', headers: { 'content-type': 'application/json' },
-      body: JSON.stringify({ plugin: 'weebcentral', operation: 'series', input: { html: '<h1>X</h1>', url: 'https://weebcentral.com/series/X' } }),
-    });
-    assert.equal(rejected.status, 400);
-  } finally {
-    if (previous === undefined) delete process.env.MANGADEX_PLUGIN_URL;
-    else process.env.MANGADEX_PLUGIN_URL = previous;
-    await new Promise(resolve => server.close(resolve));
-  }
 });

scraper/test/source-plugin.test.js

@@ -2,12 +2,10 @@
 
 const test = require('node:test');
 const assert = require('node:assert/strict');
-const httpModule = require('node:http');
-const { createServer } = require('../plugins/worker');
 const { parse } = require('../plugins/registry');
-const { parseSource } = require('../plugins/client');
 const { validate } = require('../plugins/validation');
 const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral');
+const { adapters, adapterFor } = require('../adapters');
 
 const seriesUrl = 'https://weebcentral.com/series/ABC/title';
 const chapterUrl = 'https://weebcentral.com/chapters/XYZ';
@@ -32,11 +30,7 @@ test('parser output cannot turn catalog and image links into local or unrelated
   assert.doesNotThrow(() => validate('weebcentral', 'pages', { pages: ['https://img.example/1.jpg'] }));
 });
 
-test('isolated parser preserves WeebCentral catalog and page discovery', async () => {
+test('in-process parser preserves WeebCentral catalog and page discovery', async () => {
-  const server = createServer();
-  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
-  const previous = process.env.SOURCE_PLUGIN_URL;
-  process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
   const requested = [];
   const http = { async get(url) {
     requested.push(url);
@@ -46,50 +40,22 @@ test('isolated parser preserves WeebCentral catalog and page discovery', async (
           : imagesHtml;
     return { text: async () => body };
   } };
-  try {
+  const catalog = await discoverWeebCatalog(seriesUrl, http);
-    const catalog = await discoverWeebCatalog(seriesUrl, http);
+  assert.equal(catalog.title, 'Test Manga');
-    assert.equal(catalog.title, 'Test Manga');
+  assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']);
-    assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']);
+  const chapter = await discoverWeebChapter(catalog.chapters[0], http);
-    const chapter = await discoverWeebChapter(catalog.chapters[0], http);
+  assert.equal(chapter.pages.length, 2);
-    assert.equal(chapter.pages.length, 2);
+  assert.equal(chapter.added, '2026-01-01');
-    assert.equal(chapter.added, '2026-01-01');
+  assert.equal(requested.length, 4);
-    assert.equal(requested.length, 4);
-    const response = await fetch(`${process.env.SOURCE_PLUGIN_URL}/v1/parse`, {
-      method: 'POST', headers: { 'content-type': 'application/json' },
-      body: JSON.stringify({ plugin: 'unapproved', operation: 'catalog', input: { html: listHtml, url: seriesUrl } }),
-    });
-    assert.equal(response.status, 400);
-  } finally {
-    if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL;
-    else process.env.SOURCE_PLUGIN_URL = previous;
-    await new Promise(resolve => server.close(resolve));
-  }
 });
 
-test('configured worker failure does not fall back to local parsing', async () => {
+test('a new source is an explicit adapter, not another container or remote module', () => {
-  const previous = process.env.SOURCE_PLUGIN_URL;
+  assert.deepEqual(adapters.map(adapter => adapter.id), ['weebcentral', 'mangadex']);
-  process.env.SOURCE_PLUGIN_URL = 'http://127.0.0.1:1';
+  for (const adapter of adapters) {
-  try {
+    assert.equal(typeof adapter.match, 'function');
-    await assert.rejects(parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }));
+    assert.equal(typeof adapter.discover, 'function');
-  } finally {
+    assert.equal(typeof adapter.run, 'function');
-    if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL;
-    else process.env.SOURCE_PLUGIN_URL = previous;
-  }
-});
-
-test('scraper rejects unsafe responses from a configured worker', async () => {
-  const server = httpModule.createServer((_request, response) => {
-    response.writeHead(200, { 'content-type': 'application/json' });
-    response.end(JSON.stringify({ title: 'X', listUrl: 'http://127.0.0.1:8091/api/state' }));
-  });
-  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
-  const previous = process.env.SOURCE_PLUGIN_URL;
-  process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
-  try {
-    await assert.rejects(parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl }), /unexpected/);
-  } finally {
-    if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL;
-    else process.env.SOURCE_PLUGIN_URL = previous;
-    await new Promise(resolve => server.close(resolve));
   }
+  assert.equal(adapterFor(seriesUrl)?.adapter.id, 'weebcentral');
+  assert.equal(adapterFor('https://unknown.example/series/ABC'), null);
 });

tools/healthcheck-source-plugin.js

@@ -1,5 +0,0 @@
-'use strict';
-
-fetch('http://127.0.0.1:8094/health', { signal: AbortSignal.timeout(3000) })
-  .then(response => { if (!response.ok) process.exitCode = 1; })
-  .catch(() => { process.exitCode = 1; });