Keep source plugins in single scraper process

scraper/plugins/client.js

@@ -1,35 +0,0 @@
-'use strict';
-
-const { parse } = require('./registry');
-const { validate } = require('./validation');
-const MAX_MESSAGE_BYTES = 2 * 1024 * 1024;
-
-async function boundedJson(response) {
-  const chunks = [];
-  let bytes = 0;
-  for await (const chunk of response.body) {
-    bytes += chunk.length;
-    if (bytes > MAX_MESSAGE_BYTES) throw new Error('Source plugin response exceeds limit');
-    chunks.push(chunk);
-  }
-  return JSON.parse(Buffer.concat(chunks).toString('utf8'));
-}
-
-async function parseSource(plugin, operation, input) {
-  const endpoint = plugin === 'mangadex' ? process.env.MANGADEX_PLUGIN_URL : process.env.SOURCE_PLUGIN_URL;
-  if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input));
-  const url = new URL('/v1/parse', endpoint);
-  if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint');
-  const body = JSON.stringify({ plugin, operation, input });
-  if (Buffer.byteLength(body) > MAX_MESSAGE_BYTES) throw new Error('Source plugin request exceeds limit');
-  const response = await fetch(url, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json' },
-    body,
-    signal: AbortSignal.timeout(10000),
-  });
-  if (!response.ok) throw new Error(`Source parser unavailable (${response.status}); checkpoint preserved`);
-  return validate(plugin, operation, await boundedJson(response));
-}
-
-module.exports = { parseSource };

scraper/plugins/mangadex.js

@@ -87,8 +87,7 @@ function pageUrls(atHome, chapterUrl, expectedPages) {
   }));
 }
 
-function metadata({ html }) {
-  const manga = JSON.parse(html);
+function metadata({ data: manga }) {
   if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
   const attributes = manga.attributes || {};
   const title = localizedTitle(attributes);
@@ -101,8 +100,7 @@ function metadata({ html }) {
     } };
 }
 
-function feed({ html }) {
-  const releases = JSON.parse(html);
+function feed({ data: releases }) {
   if (!Array.isArray(releases)) throw new Error('Invalid MangaDex feed');
   return { releases: releases.map(resource => ({
     id: resource.id,
@@ -116,15 +114,15 @@ function feed({ html }) {
   })) };
 }
 
-function catalog({ html }) {
-  const { attributes, releases, specialLabel, externalReleases } = JSON.parse(html);
+function catalog({ data }) {
+  const { attributes, releases, specialLabel, externalReleases } = data;
   if (!attributes || !Array.isArray(releases)) throw new Error('Invalid MangaDex catalog input');
   const chapters = selectChapterReleases(releases, specialLabel);
   return { chapters, availability: availabilitySummary(attributes, releases, chapters, externalReleases) };
 }
 
-function atHome({ html }) {
-  const { response, chapterUrl, expectedPages } = JSON.parse(html);
+function atHome({ data }) {
+  const { response, chapterUrl, expectedPages } = data;
   return { pages: pageUrls(response, chapterUrl, expectedPages) };
 }
 

scraper/plugins/manifest.js

@@ -4,14 +4,14 @@
 // source may add executable modules at runtime.
 const manifests = Object.freeze({
   weebcentral: Object.freeze({
-    id: 'weebcentral', version: 1, label: 'WeebCentral', status: 'pilot',
+    id: 'weebcentral', version: 1, label: 'WeebCentral', status: 'in-process',
     sourceHosts: Object.freeze(['weebcentral.com']),
     implementation: 'first-party',
     provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied',
     operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']),
   }),
   mangadex: Object.freeze({
-    id: 'mangadex', version: 1, label: 'MangaDex', status: 'pilot',
+    id: 'mangadex', version: 1, label: 'MangaDex', status: 'in-process',
     sourceHosts: Object.freeze(['api.mangadex.org']),
     implementation: 'first-party',
     provenance: 'MangaCircus existing MangaDex adapter; no Uchiyomi code copied',

scraper/plugins/registry.js

@@ -2,6 +2,7 @@
 
 // Reviewed first-party modules only. No filesystem scanning or runtime installs.
 const { manifests } = require('./manifest');
+const { validate } = require('./validation');
 const plugins = Object.freeze({
   weebcentral: require('./weebcentral'),
   mangadex: require('./mangadex'),
@@ -11,13 +12,14 @@ function parse(plugin, operation, input) {
   const manifest = manifests[plugin];
   const handler = manifest?.operations.includes(operation) ? plugins[plugin]?.[operation] : null;
   if (typeof handler !== 'function') throw new Error('Unknown source plugin or operation');
-  if (!input || typeof input !== 'object' || typeof input.html !== 'string' || typeof input.url !== 'string') {
+  if (!input || typeof input !== 'object' || typeof input.url !== 'string'
+    || (typeof input.html !== 'string' && !(plugin === 'mangadex' && input.data && typeof input.data === 'object'))) {
     throw new Error('Invalid parser input');
   }
   const url = new URL(input.url);
   if (!['https:', 'http:'].includes(url.protocol)) throw new Error('Invalid source URL');
   if (!manifest.sourceHosts.includes(url.hostname)) throw new Error('Source URL does not belong to plugin');
-  return handler({ html: input.html, url: url.href });
+  return validate(plugin, operation, handler({ html: input.html, data: input.data, url: url.href }));
 }
 
 module.exports = { parse, manifests };

scraper/plugins/worker.js

@@ -1,49 +0,0 @@
-'use strict';
-
-const http = require('node:http');
-const { parse } = require('./registry');
-const MAX_BODY_BYTES = 2 * 1024 * 1024;
-
-function createServer(pluginId = process.env.SOURCE_PLUGIN_ID || null) {
-  return http.createServer((request, response) => {
-    if (request.method === 'GET' && request.url === '/health') {
-      response.writeHead(200, { 'content-type': 'application/json' });
-      response.end(JSON.stringify({ ok: true }));
-      return;
-    }
-    if (request.method !== 'POST' || request.url !== '/v1/parse') {
-      response.writeHead(404).end();
-      return;
-    }
-    let bytes = 0;
-    const chunks = [];
-    request.on('data', chunk => {
-      bytes += chunk.length;
-      if (bytes > MAX_BODY_BYTES) {
-        response.writeHead(413).end();
-        request.destroy();
-        return;
-      }
-      chunks.push(chunk);
-    });
-    request.on('end', () => {
-      if (bytes > MAX_BODY_BYTES) return;
-      try {
-        const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8'));
-        if (pluginId && plugin !== pluginId) throw new Error('Plugin worker does not serve this source');
-        const output = parse(plugin, operation, input);
-        response.writeHead(200, { 'content-type': 'application/json' });
-        response.end(JSON.stringify(output));
-      } catch (error) {
-        response.writeHead(400, { 'content-type': 'application/json' });
-        response.end(JSON.stringify({ error: error.message }));
-      }
-    });
-  });
-}
-
-if (require.main === module) {
-  createServer().listen(Number(process.env.SOURCE_PLUGIN_PORT || 8094), process.env.SOURCE_PLUGIN_HOST || '127.0.0.1');
-}
-
-module.exports = { createServer };