Preserve per-source pacing across queue jobs and add source pause

This commit is contained in:
Codex committed 2026-10-14 09:23:04 +02:00
1 parent a08e847932
commit a22913129a
8 files changed
+76 -7

No files matched your search

+15 -1
View File
@@ -3,10 +3,24 @@
const { PoliteGate, sleep } = require('./delay');
class StopScrapeError extends Error {}
const sourceGates = new Map();
function gateFor(options) {
const next = new PoliteGate(options);
if (!options.gateKey) return next;
const key = String(options.gateKey);
const existing = sourceGates.get(key);
if (!existing) { sourceGates.set(key, next); return next; }
// A new job may use updated host settings, but it must not forget when the
// previous job last requested this same source.
existing.minMs = next.minMs;
existing.maxMs = next.maxMs;
return existing;
}
class SlowHttp {
constructor(options = {}) {
this.gate = new PoliteGate(options);
this.gate = gateFor(options);
this.userAgent = options.userAgent || 'MangaCircus-Scraper/1.0 (personal archival; sequential requests)';
this.maxRetries = options.maxRetries ?? 2;
this.onEvent = options.onEvent;