diff --git a/scraper/plugins/broker.js b/scraper/plugins/broker.js index df9979e..af5e653 100644 --- a/scraper/plugins/broker.js +++ b/scraper/plugins/broker.js @@ -35,6 +35,12 @@ function validateWeebRequest(value, options = {}) { return url; } +function pinnedLookup(selected) { + return (_hostname, lookupOptions, callback) => lookupOptions.all + ? callback(null, [{ address: selected.address, family: selected.family }]) + : callback(null, selected.address, selected.family); +} + async function pinnedFetch(value, options = {}) { const url = new URL(value); if (url.protocol !== 'https:') throw new Error('Pinned transport requires HTTPS'); @@ -48,7 +54,7 @@ async function pinnedFetch(value, options = {}) { const request = https.request(url, { method: 'GET', headers: options.headers, signal: options.signal, agent: false, - lookup: (_hostname, _lookupOptions, callback) => callback(null, selected.address, selected.family), + lookup: pinnedLookup(selected), }, response => { let bytes = 0; const chunks = []; @@ -82,4 +88,4 @@ function createWeebHttp(options = {}) { return new SlowHttp({ ...options, transport: pinnedFetch, validateUrl: validateWeebRequest }); } -module.exports = { publicAddress, validateWeebRequest, pinnedFetch, createWeebHttp }; +module.exports = { publicAddress, validateWeebRequest, pinnedLookup, pinnedFetch, createWeebHttp }; diff --git a/scraper/test/source-broker.test.js b/scraper/test/source-broker.test.js index 32bdb16..36084ae 100644 --- a/scraper/test/source-broker.test.js +++ b/scraper/test/source-broker.test.js @@ -3,7 +3,7 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const { SlowHttp, StopScrapeError } = require('../lib/http'); -const { publicAddress, validateWeebRequest } = require('../plugins/broker'); +const { publicAddress, validateWeebRequest, pinnedLookup } = require('../plugins/broker'); test('plugin broker rejects local, private, non-TLS, and cross-source URLs', () => { for (const address of ['127.0.0.1', '10.0.0.5', '192.168.1.1', '169.254.169.254', '::1', '::ffff:127.0.0.1']) { @@ -17,6 +17,19 @@ test('plugin broker rejects local, private, non-TLS, and cross-source URLs', () assert.doesNotThrow(() => validateWeebRequest('https://img.example/page.jpg')); }); +test('pinned DNS lookup supports both Node single-address and all-address requests', () => { + const lookup = pinnedLookup({ address: '1.1.1.1', family: 4 }); + lookup('example.com', { all: false }, (error, address, family) => { + assert.equal(error, null); + assert.equal(address, '1.1.1.1'); + assert.equal(family, 4); + }); + lookup('example.com', { all: true }, (error, addresses) => { + assert.equal(error, null); + assert.deepEqual(addresses, [{ address: '1.1.1.1', family: 4 }]); + }); +}); + test('redirect hops are validated and paced before another request', async () => { const calls = []; const http = new SlowHttp({