Preserve per-source pacing across queue jobs and add source pause

This commit is contained in:
Codex committed 2026-10-14 09:23:04 +02:00
1 parent a08e847932
commit a22913129a
8 files changed
+76 -7

No files matched your search

+12
View File
@@ -54,3 +54,15 @@ test('a guarded 403 stops immediately without redirect or retry', async () => {
await assert.rejects(http.get('https://weebcentral.com/series/X', { accept: 'text/html' }), StopScrapeError);
assert.equal(calls, 1);
});
test('separate jobs on one source retain the configured request gap', async () => {
const gateKey = `fixture-${Date.now()}-${Math.random()}`;
const transport = async () => ({ status: 200, ok: true });
const first = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport });
await first.get('https://example.org/one');
const second = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport });
assert.equal(second.gate, first.gate);
const started = Date.now();
await second.get('https://example.org/two');
assert.ok(Date.now() - started >= 30, 'a new job must not reset its source delay');
});
+26
View File
@@ -0,0 +1,26 @@
'use strict';
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const { ScraperService } = require('../service');
test('an operator can pause one source while other lanes continue', async t => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-source-pause-'));
const service = await new ScraperService({ libraryRoot: path.join(root, 'library'),
dataRoot: path.join(root, 'data') }).init();
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
await assert.rejects(service.control('pause-source', 'unknown'), /Unknown source/);
await service.control('pause-source', 'weebcentral');
assert.equal(service.state.sourcePaused.weebcentral, true);
await service.control('resume');
assert.equal(service.state.paused, false);
assert.equal(service.state.sourcePaused.weebcentral, true);
service.state.queue.push({ source: 'weebcentral', status: 'error', failure: { kind: 'access' } });
await assert.rejects(service.control('resume-source', 'weebcentral'), /access block/);
service.state.queue.pop();
await service.control('resume-source', 'weebcentral');
assert.equal(service.state.sourcePaused.weebcentral, undefined);
});