'use strict'; const test = require('node:test'); const assert = require('node:assert/strict'); const { SlowHttp, StopScrapeError } = require('../lib/http'); const { publicAddress, validateWeebRequest, pinnedLookup, responseByteLimit } = require('../plugins/broker'); test('only explicit HTML catalogs get the larger bounded response allowance', async () => { const html = { headers: { accept: 'text/html' } }; assert.equal(responseByteLimit(html), 2 * 1024 * 1024); assert.equal(responseByteLimit({ ...html, catalog: true }), 16 * 1024 * 1024); assert.equal(responseByteLimit({ headers: { accept: 'image/jpeg' }, catalog: true }), 50 * 1024 * 1024); const calls = []; const http = new SlowHttp({ minMs: 0, maxMs: 0, transport: async (_url, options) => { calls.push(options.catalog); return { status: 200, ok: true }; } }); await http.get('https://weebcentral.com/series/X', { accept: 'text/html' }); await http.get('https://weebcentral.com/series/X/full-chapter-list', { accept: 'text/html', catalog: true }); assert.deepEqual(calls, [false, true]); }); test('plugin broker rejects local, private, non-TLS, and cross-source URLs', () => { for (const address of ['127.0.0.1', '10.0.0.5', '192.168.1.1', '169.254.169.254', '::1', '::ffff:127.0.0.1']) { assert.equal(publicAddress(address), false, address); } assert.equal(publicAddress('1.1.1.1'), true); assert.equal(publicAddress('2606:4700:4700::1111'), true); assert.throws(() => validateWeebRequest('http://weebcentral.com/series/X', { accept: 'text/html' }), /public HTTPS/); assert.throws(() => validateWeebRequest('https://localhost/secret'), /public HTTPS/); assert.throws(() => validateWeebRequest('https://other.example/series/X', { accept: 'text/html' }), /source host/); assert.doesNotThrow(() => validateWeebRequest('https://img.example/page.jpg')); }); test('pinned DNS lookup supports both Node single-address and all-address requests', () => { const lookup = pinnedLookup({ address: '1.1.1.1', family: 4 }); lookup('example.com', { all: false }, (error, address, family) => { assert.equal(error, null); assert.equal(address, '1.1.1.1'); assert.equal(family, 4); }); lookup('example.com', { all: true }, (error, addresses) => { assert.equal(error, null); assert.deepEqual(addresses, [{ address: '1.1.1.1', family: 4 }]); }); }); test('redirect hops are validated and paced before another request', async () => { const calls = []; const http = new SlowHttp({ minMs: 0, maxMs: 0, maxRetries: 0, validateUrl: validateWeebRequest, transport: async url => { calls.push(url); return calls.length === 1 ? { status: 302, headers: { get: () => 'https://localhost/private' } } : { status: 200, ok: true }; }, }); await assert.rejects(http.get('https://weebcentral.com/series/X', { accept: 'text/html' }), /public HTTPS/); assert.deepEqual(calls, ['https://weebcentral.com/series/X']); }); test('a guarded 403 stops immediately without redirect or retry', async () => { let calls = 0; const http = new SlowHttp({ minMs: 0, maxMs: 0, maxRetries: 2, validateUrl: validateWeebRequest, transport: async () => { calls++; return { status: 403 }; }, }); await assert.rejects(http.get('https://weebcentral.com/series/X', { accept: 'text/html' }), StopScrapeError); assert.equal(calls, 1); }); test('separate jobs on one source retain the configured request gap', async () => { const gateKey = `fixture-${Date.now()}-${Math.random()}`; const transport = async () => ({ status: 200, ok: true }); const first = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport }); await first.get('https://example.org/one'); const second = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport }); assert.equal(second.gate, first.gate); const started = Date.now(); await second.get('https://example.org/two'); assert.ok(Date.now() - started >= 30, 'a new job must not reset its source delay'); });