Validate first-party parser boundaries

scraper/plugins/client.js

@@ -1,20 +1,35 @@
 'use strict';
 
 const { parse } = require('./registry');
+const { validate } = require('./validation');
+const MAX_MESSAGE_BYTES = 2 * 1024 * 1024;
+
+async function boundedJson(response) {
+  const chunks = [];
+  let bytes = 0;
+  for await (const chunk of response.body) {
+    bytes += chunk.length;
+    if (bytes > MAX_MESSAGE_BYTES) throw new Error('Source plugin response exceeds limit');
+    chunks.push(chunk);
+  }
+  return JSON.parse(Buffer.concat(chunks).toString('utf8'));
+}
 
 async function parseSource(plugin, operation, input) {
   const endpoint = process.env.SOURCE_PLUGIN_URL;
-  if (!endpoint) return parse(plugin, operation, input);
+  if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input));
   const url = new URL('/v1/parse', endpoint);
   if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint');
+  const body = JSON.stringify({ plugin, operation, input });
+  if (Buffer.byteLength(body) > MAX_MESSAGE_BYTES) throw new Error('Source plugin request exceeds limit');
   const response = await fetch(url, {
     method: 'POST',
     headers: { 'content-type': 'application/json' },
-    body: JSON.stringify({ plugin, operation, input }),
+    body,
     signal: AbortSignal.timeout(10000),
   });
   if (!response.ok) throw new Error(`Source parser unavailable (${response.status}); checkpoint preserved`);
-  return response.json();
+  return validate(plugin, operation, await boundedJson(response));
 }
 
 module.exports = { parseSource };

scraper/plugins/validation.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const net = require('node:net');
+
+function sourceUrl(value, prefix) {
+  const url = new URL(value);
+  if (url.protocol !== 'https:' || url.hostname !== 'weebcentral.com' || url.username || url.password || !url.pathname.startsWith(prefix)) {
+    throw new Error('Source plugin returned an unexpected WeebCentral URL');
+  }
+  return url.href;
+}
+
+function imageUrl(value) {
+  const url = new URL(value);
+  const hostname = url.hostname.toLowerCase();
+  if (url.protocol !== 'https:' || url.username || url.password || net.isIP(hostname)
+    || hostname === 'localhost' || hostname.endsWith('.localhost') || hostname.endsWith('.local')
+    || !hostname.includes('.')) {
+    throw new Error('Source plugin returned an unsafe image URL');
+  }
+  return url.href;
+}
+
+function validate(plugin, operation, output) {
+  if (plugin !== 'weebcentral' || !output || typeof output !== 'object' || Array.isArray(output)) {
+    throw new Error('Invalid source plugin output');
+  }
+  if (operation === 'series') {
+    if (typeof output.title !== 'string' || typeof output.listUrl !== 'string' && output.listUrl !== null) throw new Error('Invalid series parse');
+    if (output.listUrl !== null) sourceUrl(output.listUrl, '/series/');
+  } else if (operation === 'catalog') {
+    if (!Array.isArray(output.chapters) || !Number.isInteger(output.chapterLinks) || output.chapterLinks < 0) throw new Error('Invalid catalog parse');
+    for (const chapter of output.chapters) {
+      if (!chapter || typeof chapter.id !== 'string' || typeof chapter.number !== 'string') throw new Error('Invalid chapter reference');
+      sourceUrl(chapter.url, '/chapters/');
+    }
+  } else if (operation === 'chapter') {
+    sourceUrl(output.manifestUrl, '/chapters/');
+    if (!new URL(output.manifestUrl).pathname.endsWith('/images')) throw new Error('Invalid chapter image manifest');
+  } else if (operation === 'pages') {
+    if (!Array.isArray(output.pages)) throw new Error('Invalid page parse');
+    for (const page of output.pages) imageUrl(page);
+  } else throw new Error('Unknown source plugin operation');
+  return output;
+}
+
+module.exports = { validate };

scraper/test/source-plugin.test.js

@@ -2,9 +2,11 @@
 
 const test = require('node:test');
 const assert = require('node:assert/strict');
+const httpModule = require('node:http');
 const { createServer } = require('../plugins/worker');
 const { parse } = require('../plugins/registry');
 const { parseSource } = require('../plugins/client');
+const { validate } = require('../plugins/validation');
 const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral');
 
 const seriesUrl = 'https://weebcentral.com/series/ABC/title';
@@ -21,6 +23,15 @@ test('first-party registry rejects unknown plugins and malformed input', () => {
   assert.equal(parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }).chapters.length, 2);
 });
 
+test('parser output cannot turn catalog and image links into local or unrelated requests', () => {
+  assert.throws(() => validate('weebcentral', 'series', { title: 'X', listUrl: 'http://127.0.0.1:8091/api/state' }), /unexpected/);
+  assert.throws(() => validate('weebcentral', 'catalog', { chapterLinks: 1, chapters: [{ id: 'X', number: '1', url: 'https://other.example/chapters/X' }] }), /unexpected/);
+  assert.throws(() => validate('weebcentral', 'chapter', { manifestUrl: 'https://weebcentral.com/chapters/X/not-images' }), /manifest/);
+  assert.throws(() => validate('weebcentral', 'pages', { pages: ['http://127.0.0.1:8080/private'] }), /unsafe/);
+  assert.throws(() => validate('weebcentral', 'pages', { pages: ['https://localhost/private'] }), /unsafe/);
+  assert.doesNotThrow(() => validate('weebcentral', 'pages', { pages: ['https://img.example/1.jpg'] }));
+});
+
 test('isolated parser preserves WeebCentral catalog and page discovery', async () => {
   const server = createServer();
   await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
@@ -65,3 +76,20 @@ test('configured worker failure does not fall back to local parsing', async () =
     else process.env.SOURCE_PLUGIN_URL = previous;
   }
 });
+
+test('scraper rejects unsafe responses from a configured worker', async () => {
+  const server = httpModule.createServer((_request, response) => {
+    response.writeHead(200, { 'content-type': 'application/json' });
+    response.end(JSON.stringify({ title: 'X', listUrl: 'http://127.0.0.1:8091/api/state' }));
+  });
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  const previous = process.env.SOURCE_PLUGIN_URL;
+  process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
+  try {
+    await assert.rejects(parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl }), /unexpected/);
+  } finally {
+    if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL;
+    else process.env.SOURCE_PLUGIN_URL = previous;
+    await new Promise(resolve => server.close(resolve));
+  }
+});

vision/first-party-source-plugins.md

@@ -6,6 +6,8 @@ Compose worker. No external plugin was installed, no host was probed, and no
 live queue, delay, or container was changed. The broker, generic runner, new
 hoster pilot, admin controls, and full deployment validation below remain work
 to do; the diagram describes the target rather than the current implementation.
+Parser-result URL validation and bounded worker messages were added next; full
+redirect/DNS enforcement and source-specific CDN allowlists are still pending.
 
 ## Decision