84 lines
3.3 KiB
JavaScript
84 lines
3.3 KiB
JavaScript
'use strict';
|
|
|
|
const { PoliteGate, sleep } = require('./delay');
|
|
|
|
class StopScrapeError extends Error {}
|
|
const sourceGates = new Map();
|
|
|
|
function gateFor(options) {
|
|
const next = new PoliteGate(options);
|
|
if (!options.gateKey) return next;
|
|
const key = String(options.gateKey);
|
|
const existing = sourceGates.get(key);
|
|
if (!existing) { sourceGates.set(key, next); return next; }
|
|
// A new job may use updated host settings, but it must not forget when the
|
|
// previous job last requested this same source.
|
|
existing.minMs = next.minMs;
|
|
existing.maxMs = next.maxMs;
|
|
return existing;
|
|
}
|
|
|
|
class SlowHttp {
|
|
constructor(options = {}) {
|
|
this.gate = gateFor(options);
|
|
this.userAgent = options.userAgent || 'MangaCircus-Scraper/1.0 (personal archival; sequential requests)';
|
|
this.maxRetries = options.maxRetries ?? 2;
|
|
this.onEvent = options.onEvent;
|
|
this.timeoutMs = options.timeoutMs || 60000;
|
|
this.transport = options.transport || fetch;
|
|
this.validateUrl = options.validateUrl || null;
|
|
}
|
|
|
|
async get(url, options = {}) {
|
|
let redirects = 0;
|
|
let currentUrl = url;
|
|
for (let attempt = 0; ; attempt++) {
|
|
if (this.validateUrl) this.validateUrl(currentUrl, options);
|
|
await this.gate.wait();
|
|
let response;
|
|
try {
|
|
response = await this.transport(currentUrl, {
|
|
redirect: this.validateUrl ? 'manual' : 'follow',
|
|
signal: AbortSignal.timeout(this.timeoutMs),
|
|
catalog: options.catalog === true,
|
|
headers: { 'user-agent': this.userAgent, accept: options.accept || '*/*', ...(options.headers || {}) },
|
|
});
|
|
this.gate.mark();
|
|
} catch (error) {
|
|
this.gate.mark();
|
|
if (error.nonRetryable) throw error;
|
|
if (attempt >= this.maxRetries) throw new Error(`Network request failed after ${attempt + 1} attempts: ${currentUrl}`, { cause: error });
|
|
const backoff = 30000 * (2 ** attempt);
|
|
const message = `Network failure; retrying in ${backoff / 1000}s (${attempt + 1}/${this.maxRetries})`;
|
|
console.warn(`[http] ${message}`);
|
|
if (this.onEvent) await this.onEvent({ level: 'warning', message, url: currentUrl });
|
|
await sleep(backoff);
|
|
continue;
|
|
}
|
|
|
|
if (this.validateUrl && response.status >= 300 && response.status < 400) {
|
|
const location = response.headers.get('location');
|
|
if (!location || ++redirects > 5) throw new Error('Source redirect limit or missing location');
|
|
currentUrl = new URL(location, currentUrl).href;
|
|
this.validateUrl(currentUrl, options);
|
|
await response.body?.cancel?.();
|
|
attempt = -1;
|
|
continue;
|
|
}
|
|
|
|
if ([401, 403, 429].includes(response.status)) {
|
|
if (this.onEvent) await this.onEvent({ level: 'error', message: `Host returned ${response.status}; stopping`, url: currentUrl, status: response.status });
|
|
throw Object.assign(new StopScrapeError(`Host returned ${response.status}; stop and require human review: ${currentUrl}`), { status: response.status });
|
|
}
|
|
if (response.status >= 500 && attempt < this.maxRetries) {
|
|
await sleep(30000 * (2 ** attempt));
|
|
continue;
|
|
}
|
|
if (!response.ok) throw Object.assign(new Error(`HTTP ${response.status}: ${currentUrl}`), { status: response.status });
|
|
return response;
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = { SlowHttp, StopScrapeError };
|