83 lines
3.9 KiB
JavaScript
83 lines
3.9 KiB
JavaScript
'use strict';
|
|
|
|
const test = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const { SlowHttp, StopScrapeError } = require('../lib/http');
|
|
const { publicAddress, validateWeebRequest, pinnedLookup, responseByteLimit } = require('../plugins/broker');
|
|
|
|
test('only explicit HTML catalogs get the larger bounded response allowance', async () => {
|
|
const html = { headers: { accept: 'text/html' } };
|
|
assert.equal(responseByteLimit(html), 2 * 1024 * 1024);
|
|
assert.equal(responseByteLimit({ ...html, catalog: true }), 16 * 1024 * 1024);
|
|
assert.equal(responseByteLimit({ headers: { accept: 'image/jpeg' }, catalog: true }), 50 * 1024 * 1024);
|
|
const calls = [];
|
|
const http = new SlowHttp({ minMs: 0, maxMs: 0, transport: async (_url, options) => {
|
|
calls.push(options.catalog); return { status: 200, ok: true };
|
|
} });
|
|
await http.get('https://weebcentral.com/series/X', { accept: 'text/html' });
|
|
await http.get('https://weebcentral.com/series/X/full-chapter-list', { accept: 'text/html', catalog: true });
|
|
assert.deepEqual(calls, [false, true]);
|
|
});
|
|
|
|
test('plugin broker rejects local, private, non-TLS, and cross-source URLs', () => {
|
|
for (const address of ['127.0.0.1', '10.0.0.5', '192.168.1.1', '169.254.169.254', '::1', '::ffff:127.0.0.1']) {
|
|
assert.equal(publicAddress(address), false, address);
|
|
}
|
|
assert.equal(publicAddress('1.1.1.1'), true);
|
|
assert.equal(publicAddress('2606:4700:4700::1111'), true);
|
|
assert.throws(() => validateWeebRequest('http://weebcentral.com/series/X', { accept: 'text/html' }), /public HTTPS/);
|
|
assert.throws(() => validateWeebRequest('https://localhost/secret'), /public HTTPS/);
|
|
assert.throws(() => validateWeebRequest('https://other.example/series/X', { accept: 'text/html' }), /source host/);
|
|
assert.doesNotThrow(() => validateWeebRequest('https://img.example/page.jpg'));
|
|
});
|
|
|
|
test('pinned DNS lookup supports both Node single-address and all-address requests', () => {
|
|
const lookup = pinnedLookup({ address: '1.1.1.1', family: 4 });
|
|
lookup('example.com', { all: false }, (error, address, family) => {
|
|
assert.equal(error, null);
|
|
assert.equal(address, '1.1.1.1');
|
|
assert.equal(family, 4);
|
|
});
|
|
lookup('example.com', { all: true }, (error, addresses) => {
|
|
assert.equal(error, null);
|
|
assert.deepEqual(addresses, [{ address: '1.1.1.1', family: 4 }]);
|
|
});
|
|
});
|
|
|
|
test('redirect hops are validated and paced before another request', async () => {
|
|
const calls = [];
|
|
const http = new SlowHttp({
|
|
minMs: 0, maxMs: 0, maxRetries: 0, validateUrl: validateWeebRequest,
|
|
transport: async url => {
|
|
calls.push(url);
|
|
return calls.length === 1
|
|
? { status: 302, headers: { get: () => 'https://localhost/private' } }
|
|
: { status: 200, ok: true };
|
|
},
|
|
});
|
|
await assert.rejects(http.get('https://weebcentral.com/series/X', { accept: 'text/html' }), /public HTTPS/);
|
|
assert.deepEqual(calls, ['https://weebcentral.com/series/X']);
|
|
});
|
|
|
|
test('a guarded 403 stops immediately without redirect or retry', async () => {
|
|
let calls = 0;
|
|
const http = new SlowHttp({
|
|
minMs: 0, maxMs: 0, maxRetries: 2, validateUrl: validateWeebRequest,
|
|
transport: async () => { calls++; return { status: 403 }; },
|
|
});
|
|
await assert.rejects(http.get('https://weebcentral.com/series/X', { accept: 'text/html' }), StopScrapeError);
|
|
assert.equal(calls, 1);
|
|
});
|
|
|
|
test('separate jobs on one source retain the configured request gap', async () => {
|
|
const gateKey = `fixture-${Date.now()}-${Math.random()}`;
|
|
const transport = async () => ({ status: 200, ok: true });
|
|
const first = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport });
|
|
await first.get('https://example.org/one');
|
|
const second = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport });
|
|
assert.equal(second.gate, first.gate);
|
|
const started = Date.now();
|
|
await second.get('https://example.org/two');
|
|
assert.ok(Date.now() - started >= 30, 'a new job must not reset its source delay');
|
|
});
|