Preserve per-source pacing across queue jobs and add source pause
This commit is contained in:
1 parent
a08e847932
commit
a22913129a
8 files changed
+76
-7
No files matched your search
+15
-1
@@ -3,10 +3,24 @@
|
||||
const { PoliteGate, sleep } = require('./delay');
|
||||
|
||||
class StopScrapeError extends Error {}
|
||||
const sourceGates = new Map();
|
||||
|
||||
function gateFor(options) {
|
||||
const next = new PoliteGate(options);
|
||||
if (!options.gateKey) return next;
|
||||
const key = String(options.gateKey);
|
||||
const existing = sourceGates.get(key);
|
||||
if (!existing) { sourceGates.set(key, next); return next; }
|
||||
// A new job may use updated host settings, but it must not forget when the
|
||||
// previous job last requested this same source.
|
||||
existing.minMs = next.minMs;
|
||||
existing.maxMs = next.maxMs;
|
||||
return existing;
|
||||
}
|
||||
|
||||
class SlowHttp {
|
||||
constructor(options = {}) {
|
||||
this.gate = new PoliteGate(options);
|
||||
this.gate = gateFor(options);
|
||||
this.userAgent = options.userAgent || 'MangaCircus-Scraper/1.0 (personal archival; sequential requests)';
|
||||
this.maxRetries = options.maxRetries ?? 2;
|
||||
this.onEvent = options.onEvent;
|
||||
|
||||
Reference in new issue
Block a user