From 6599d0a24474798bfc5462afe822749d64ae20ed Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 13 Oct 2026 21:47:18 +0200 Subject: [PATCH] Validate first-party parser boundaries --- scraper/plugins/client.js | 21 +++++++++++-- scraper/plugins/validation.js | 47 ++++++++++++++++++++++++++++ scraper/test/source-plugin.test.js | 28 +++++++++++++++++ vision/first-party-source-plugins.md | 2 ++ 4 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 scraper/plugins/validation.js diff --git a/scraper/plugins/client.js b/scraper/plugins/client.js index 9d3e966..a8c0aee 100644 --- a/scraper/plugins/client.js +++ b/scraper/plugins/client.js @@ -1,20 +1,35 @@ 'use strict'; const { parse } = require('./registry'); +const { validate } = require('./validation'); +const MAX_MESSAGE_BYTES = 2 * 1024 * 1024; + +async function boundedJson(response) { + const chunks = []; + let bytes = 0; + for await (const chunk of response.body) { + bytes += chunk.length; + if (bytes > MAX_MESSAGE_BYTES) throw new Error('Source plugin response exceeds limit'); + chunks.push(chunk); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8')); +} async function parseSource(plugin, operation, input) { const endpoint = process.env.SOURCE_PLUGIN_URL; - if (!endpoint) return parse(plugin, operation, input); + if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input)); const url = new URL('/v1/parse', endpoint); if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint'); + const body = JSON.stringify({ plugin, operation, input }); + if (Buffer.byteLength(body) > MAX_MESSAGE_BYTES) throw new Error('Source plugin request exceeds limit'); const response = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ plugin, operation, input }), + body, signal: AbortSignal.timeout(10000), }); if (!response.ok) throw new Error(`Source parser unavailable (${response.status}); checkpoint preserved`); - return response.json(); + return validate(plugin, operation, await boundedJson(response)); } module.exports = { parseSource }; diff --git a/scraper/plugins/validation.js b/scraper/plugins/validation.js new file mode 100644 index 0000000..7af082c --- /dev/null +++ b/scraper/plugins/validation.js @@ -0,0 +1,47 @@ +'use strict'; + +const net = require('node:net'); + +function sourceUrl(value, prefix) { + const url = new URL(value); + if (url.protocol !== 'https:' || url.hostname !== 'weebcentral.com' || url.username || url.password || !url.pathname.startsWith(prefix)) { + throw new Error('Source plugin returned an unexpected WeebCentral URL'); + } + return url.href; +} + +function imageUrl(value) { + const url = new URL(value); + const hostname = url.hostname.toLowerCase(); + if (url.protocol !== 'https:' || url.username || url.password || net.isIP(hostname) + || hostname === 'localhost' || hostname.endsWith('.localhost') || hostname.endsWith('.local') + || !hostname.includes('.')) { + throw new Error('Source plugin returned an unsafe image URL'); + } + return url.href; +} + +function validate(plugin, operation, output) { + if (plugin !== 'weebcentral' || !output || typeof output !== 'object' || Array.isArray(output)) { + throw new Error('Invalid source plugin output'); + } + if (operation === 'series') { + if (typeof output.title !== 'string' || typeof output.listUrl !== 'string' && output.listUrl !== null) throw new Error('Invalid series parse'); + if (output.listUrl !== null) sourceUrl(output.listUrl, '/series/'); + } else if (operation === 'catalog') { + if (!Array.isArray(output.chapters) || !Number.isInteger(output.chapterLinks) || output.chapterLinks < 0) throw new Error('Invalid catalog parse'); + for (const chapter of output.chapters) { + if (!chapter || typeof chapter.id !== 'string' || typeof chapter.number !== 'string') throw new Error('Invalid chapter reference'); + sourceUrl(chapter.url, '/chapters/'); + } + } else if (operation === 'chapter') { + sourceUrl(output.manifestUrl, '/chapters/'); + if (!new URL(output.manifestUrl).pathname.endsWith('/images')) throw new Error('Invalid chapter image manifest'); + } else if (operation === 'pages') { + if (!Array.isArray(output.pages)) throw new Error('Invalid page parse'); + for (const page of output.pages) imageUrl(page); + } else throw new Error('Unknown source plugin operation'); + return output; +} + +module.exports = { validate }; diff --git a/scraper/test/source-plugin.test.js b/scraper/test/source-plugin.test.js index c38af47..84cf34b 100644 --- a/scraper/test/source-plugin.test.js +++ b/scraper/test/source-plugin.test.js @@ -2,9 +2,11 @@ const test = require('node:test'); const assert = require('node:assert/strict'); +const httpModule = require('node:http'); const { createServer } = require('../plugins/worker'); const { parse } = require('../plugins/registry'); const { parseSource } = require('../plugins/client'); +const { validate } = require('../plugins/validation'); const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral'); const seriesUrl = 'https://weebcentral.com/series/ABC/title'; @@ -21,6 +23,15 @@ test('first-party registry rejects unknown plugins and malformed input', () => { assert.equal(parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }).chapters.length, 2); }); +test('parser output cannot turn catalog and image links into local or unrelated requests', () => { + assert.throws(() => validate('weebcentral', 'series', { title: 'X', listUrl: 'http://127.0.0.1:8091/api/state' }), /unexpected/); + assert.throws(() => validate('weebcentral', 'catalog', { chapterLinks: 1, chapters: [{ id: 'X', number: '1', url: 'https://other.example/chapters/X' }] }), /unexpected/); + assert.throws(() => validate('weebcentral', 'chapter', { manifestUrl: 'https://weebcentral.com/chapters/X/not-images' }), /manifest/); + assert.throws(() => validate('weebcentral', 'pages', { pages: ['http://127.0.0.1:8080/private'] }), /unsafe/); + assert.throws(() => validate('weebcentral', 'pages', { pages: ['https://localhost/private'] }), /unsafe/); + assert.doesNotThrow(() => validate('weebcentral', 'pages', { pages: ['https://img.example/1.jpg'] })); +}); + test('isolated parser preserves WeebCentral catalog and page discovery', async () => { const server = createServer(); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); @@ -65,3 +76,20 @@ test('configured worker failure does not fall back to local parsing', async () = else process.env.SOURCE_PLUGIN_URL = previous; } }); + +test('scraper rejects unsafe responses from a configured worker', async () => { + const server = httpModule.createServer((_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ title: 'X', listUrl: 'http://127.0.0.1:8091/api/state' })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const previous = process.env.SOURCE_PLUGIN_URL; + process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`; + try { + await assert.rejects(parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl }), /unexpected/); + } finally { + if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL; + else process.env.SOURCE_PLUGIN_URL = previous; + await new Promise(resolve => server.close(resolve)); + } +}); diff --git a/vision/first-party-source-plugins.md b/vision/first-party-source-plugins.md index bae36ed..8ddc3da 100644 --- a/vision/first-party-source-plugins.md +++ b/vision/first-party-source-plugins.md @@ -6,6 +6,8 @@ Compose worker. No external plugin was installed, no host was probed, and no live queue, delay, or container was changed. The broker, generic runner, new hoster pilot, admin controls, and full deployment validation below remain work to do; the diagram describes the target rather than the current implementation. +Parser-result URL validation and bounded worker messages were added next; full +redirect/DNS enforcement and source-specific CDN allowlists are still pending. ## Decision