Checkpoint guarded source request broker

scraper/plugins/broker.js

@@ -0,0 +1,85 @@
+'use strict';
+
+const dns = require('node:dns').promises;
+const https = require('node:https');
+const { BlockList, isIP } = require('node:net');
+const { SlowHttp } = require('../lib/http');
+
+const blocked = new BlockList();
+for (const [address, bits] of [
+  ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8],
+  ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24],
+  ['192.0.2.0', 24], ['192.168.0.0', 16], ['198.18.0.0', 15],
+  ['198.51.100.0', 24], ['203.0.113.0', 24], ['224.0.0.0', 4],
+  ['240.0.0.0', 4], ['::', 128], ['::1', 128], ['fc00::', 7],
+  ['fe80::', 10], ['ff00::', 8], ['2001:db8::', 32],
+]) blocked.addSubnet(address, bits, isIP(address) === 6 ? 'ipv6' : 'ipv4');
+
+function publicAddress(address) {
+  const family = isIP(address);
+  return !!family && !address.toLowerCase().startsWith('::ffff:')
+    && !blocked.check(address, family === 4 ? 'ipv4' : 'ipv6');
+}
+
+function validateWeebRequest(value, options = {}) {
+  const url = new URL(value);
+  const host = url.hostname.toLowerCase();
+  if (url.protocol !== 'https:' || url.username || url.password || url.port
+    || isIP(host) || host === 'localhost' || host.endsWith('.localhost')
+    || host.endsWith('.local') || !host.includes('.')) {
+    throw new Error('Source request URL is not public HTTPS');
+  }
+  if (options.accept === 'text/html' && host !== 'weebcentral.com') {
+    throw new Error('WeebCentral catalog request left its source host');
+  }
+  return url;
+}
+
+async function pinnedFetch(value, options = {}) {
+  const url = new URL(value);
+  if (url.protocol !== 'https:') throw new Error('Pinned transport requires HTTPS');
+  const answers = await dns.lookup(url.hostname, { all: true });
+  if (!answers.length || answers.some(answer => !publicAddress(answer.address))) {
+    throw Object.assign(new Error('Source hostname resolved to a non-public address'), { nonRetryable: true });
+  }
+  const selected = answers[0];
+  const maxBytes = options.headers?.accept === 'text/html' ? 2 * 1024 * 1024 : 50 * 1024 * 1024;
+  return new Promise((resolve, reject) => {
+    const request = https.request(url, {
+      method: 'GET', headers: options.headers, signal: options.signal,
+      agent: false,
+      lookup: (_hostname, _lookupOptions, callback) => callback(null, selected.address, selected.family),
+    }, response => {
+      let bytes = 0;
+      const chunks = [];
+      response.on('data', chunk => {
+        bytes += chunk.length;
+        if (bytes > maxBytes) {
+          response.destroy(Object.assign(new Error('Source response exceeds size limit'), { nonRetryable: true }));
+          return;
+        }
+        chunks.push(chunk);
+      });
+      response.on('error', reject);
+      response.on('end', () => {
+        const body = Buffer.concat(chunks);
+        const status = response.statusCode || 0;
+        resolve({
+          status, ok: status >= 200 && status < 300,
+          headers: { get: name => response.headers[String(name).toLowerCase()] || null },
+          text: async () => body.toString('utf8'),
+          json: async () => JSON.parse(body.toString('utf8')),
+          arrayBuffer: async () => body,
+        });
+      });
+    });
+    request.on('error', reject);
+    request.end();
+  });
+}
+
+function createWeebHttp(options = {}) {
+  return new SlowHttp({ ...options, transport: pinnedFetch, validateUrl: validateWeebRequest });
+}
+
+module.exports = { publicAddress, validateWeebRequest, pinnedFetch, createWeebHttp };