53 lines
2.0 KiB
JavaScript
53 lines
2.0 KiB
JavaScript
'use strict';
|
|
|
|
const { PoliteGate, sleep } = require('./delay');
|
|
|
|
class StopScrapeError extends Error {}
|
|
|
|
class SlowHttp {
|
|
constructor(options = {}) {
|
|
this.gate = new PoliteGate(options);
|
|
this.userAgent = options.userAgent || 'MangaCircus-Scraper/1.0 (personal archival; sequential requests)';
|
|
this.maxRetries = options.maxRetries ?? 2;
|
|
this.onEvent = options.onEvent;
|
|
this.timeoutMs = options.timeoutMs || 60000;
|
|
}
|
|
|
|
async get(url, options = {}) {
|
|
for (let attempt = 0; ; attempt++) {
|
|
await this.gate.wait();
|
|
let response;
|
|
try {
|
|
response = await fetch(url, {
|
|
redirect: 'follow',
|
|
signal: AbortSignal.timeout(this.timeoutMs),
|
|
headers: { 'user-agent': this.userAgent, accept: options.accept || '*/*', ...(options.headers || {}) },
|
|
});
|
|
this.gate.mark();
|
|
} catch (error) {
|
|
this.gate.mark();
|
|
if (attempt >= this.maxRetries) throw new Error(`Network request failed after ${attempt + 1} attempts: ${url}`, { cause: error });
|
|
const backoff = 30000 * (2 ** attempt);
|
|
const message = `Network failure; retrying in ${backoff / 1000}s (${attempt + 1}/${this.maxRetries})`;
|
|
console.warn(`[http] ${message}`);
|
|
if (this.onEvent) await this.onEvent({ level: 'warning', message, url });
|
|
await sleep(backoff);
|
|
continue;
|
|
}
|
|
|
|
if ([401, 403, 429].includes(response.status)) {
|
|
if (this.onEvent) await this.onEvent({ level: 'error', message: `Host returned ${response.status}; stopping`, url, status: response.status });
|
|
throw Object.assign(new StopScrapeError(`Host returned ${response.status}; stop and require human review: ${url}`), { status: response.status });
|
|
}
|
|
if (response.status >= 500 && attempt < this.maxRetries) {
|
|
await sleep(30000 * (2 ** attempt));
|
|
continue;
|
|
}
|
|
if (!response.ok) throw Object.assign(new Error(`HTTP ${response.status}: ${url}`), { status: response.status });
|
|
return response;
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = { SlowHttp, StopScrapeError };
|