Checkpoint guarded source request broker

scraper/test/source-broker.test.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { SlowHttp, StopScrapeError } = require('../lib/http');
+const { publicAddress, validateWeebRequest } = require('../plugins/broker');
+
+test('plugin broker rejects local, private, non-TLS, and cross-source URLs', () => {
+  for (const address of ['127.0.0.1', '10.0.0.5', '192.168.1.1', '169.254.169.254', '::1', '::ffff:127.0.0.1']) {
+    assert.equal(publicAddress(address), false, address);
+  }
+  assert.equal(publicAddress('1.1.1.1'), true);
+  assert.equal(publicAddress('2606:4700:4700::1111'), true);
+  assert.throws(() => validateWeebRequest('http://weebcentral.com/series/X', { accept: 'text/html' }), /public HTTPS/);
+  assert.throws(() => validateWeebRequest('https://localhost/secret'), /public HTTPS/);
+  assert.throws(() => validateWeebRequest('https://other.example/series/X', { accept: 'text/html' }), /source host/);
+  assert.doesNotThrow(() => validateWeebRequest('https://img.example/page.jpg'));
+});
+
+test('redirect hops are validated and paced before another request', async () => {
+  const calls = [];
+  const http = new SlowHttp({
+    minMs: 0, maxMs: 0, maxRetries: 0, validateUrl: validateWeebRequest,
+    transport: async url => {
+      calls.push(url);
+      return calls.length === 1
+        ? { status: 302, headers: { get: () => 'https://localhost/private' } }
+        : { status: 200, ok: true };
+    },
+  });
+  await assert.rejects(http.get('https://weebcentral.com/series/X', { accept: 'text/html' }), /public HTTPS/);
+  assert.deepEqual(calls, ['https://weebcentral.com/series/X']);
+});
+
+test('a guarded 403 stops immediately without redirect or retry', async () => {
+  let calls = 0;
+  const http = new SlowHttp({
+    minMs: 0, maxMs: 0, maxRetries: 2, validateUrl: validateWeebRequest,
+    transport: async () => { calls++; return { status: 403 }; },
+  });
+  await assert.rejects(http.get('https://weebcentral.com/series/X', { accept: 'text/html' }), StopScrapeError);
+  assert.equal(calls, 1);
+});