Preserve per-source pacing across queue jobs and add source pause
This commit is contained in:
1 parent
a08e847932
commit
a22913129a
8 files changed
+76
-7
No files matched your search
@@ -54,3 +54,15 @@ test('a guarded 403 stops immediately without redirect or retry', async () => {
|
||||
await assert.rejects(http.get('https://weebcentral.com/series/X', { accept: 'text/html' }), StopScrapeError);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test('separate jobs on one source retain the configured request gap', async () => {
|
||||
const gateKey = `fixture-${Date.now()}-${Math.random()}`;
|
||||
const transport = async () => ({ status: 200, ok: true });
|
||||
const first = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport });
|
||||
await first.get('https://example.org/one');
|
||||
const second = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport });
|
||||
assert.equal(second.gate, first.gate);
|
||||
const started = Date.now();
|
||||
await second.get('https://example.org/two');
|
||||
assert.ok(Date.now() - started >= 30, 'a new job must not reset its source delay');
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { ScraperService } = require('../service');
|
||||
|
||||
test('an operator can pause one source while other lanes continue', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-source-pause-'));
|
||||
const service = await new ScraperService({ libraryRoot: path.join(root, 'library'),
|
||||
dataRoot: path.join(root, 'data') }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
await assert.rejects(service.control('pause-source', 'unknown'), /Unknown source/);
|
||||
await service.control('pause-source', 'weebcentral');
|
||||
assert.equal(service.state.sourcePaused.weebcentral, true);
|
||||
await service.control('resume');
|
||||
assert.equal(service.state.paused, false);
|
||||
assert.equal(service.state.sourcePaused.weebcentral, true);
|
||||
service.state.queue.push({ source: 'weebcentral', status: 'error', failure: { kind: 'access' } });
|
||||
await assert.rejects(service.control('resume-source', 'weebcentral'), /access block/);
|
||||
service.state.queue.pop();
|
||||
await service.control('resume-source', 'weebcentral');
|
||||
assert.equal(service.state.sourcePaused.weebcentral, undefined);
|
||||
});
|
||||
Reference in new issue
Block a user