Consolidate MangaCircus app, scraper, and Compose baseline

scraper/lib/http.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const { PoliteGate, sleep } = require('./delay');
+
+class StopScrapeError extends Error {}
+
+class SlowHttp {
+  constructor(options = {}) {
+    this.gate = new PoliteGate(options);
+    this.userAgent = options.userAgent || 'MangaCircus-Scraper/1.0 (personal archival; sequential requests)';
+    this.maxRetries = options.maxRetries ?? 2;
+    this.onEvent = options.onEvent;
+    this.timeoutMs = options.timeoutMs || 60000;
+  }
+
+  async get(url, options = {}) {
+    for (let attempt = 0; ; attempt++) {
+      await this.gate.wait();
+      let response;
+      try {
+        response = await fetch(url, {
+          redirect: 'follow',
+          signal: AbortSignal.timeout(this.timeoutMs),
+          headers: { 'user-agent': this.userAgent, accept: options.accept || '*/*', ...(options.headers || {}) },
+        });
+        this.gate.mark();
+      } catch (error) {
+        this.gate.mark();
+        if (attempt >= this.maxRetries) throw new Error(`Network request failed after ${attempt + 1} attempts: ${url}`, { cause: error });
+        const backoff = 30000 * (2 ** attempt);
+        const message = `Network failure; retrying in ${backoff / 1000}s (${attempt + 1}/${this.maxRetries})`;
+        console.warn(`[http] ${message}`);
+        if (this.onEvent) await this.onEvent({ level: 'warning', message, url });
+        await sleep(backoff);
+        continue;
+      }
+
+      if ([401, 403, 429].includes(response.status)) {
+        if (this.onEvent) await this.onEvent({ level: 'error', message: `Host returned ${response.status}; stopping`, url, status: response.status });
+        throw Object.assign(new StopScrapeError(`Host returned ${response.status}; stop and require human review: ${url}`), { status: response.status });
+      }
+      if (response.status >= 500 && attempt < this.maxRetries) {
+        await sleep(30000 * (2 ** attempt));
+        continue;
+      }
+      if (!response.ok) throw Object.assign(new Error(`HTTP ${response.status}: ${url}`), { status: response.status });
+      return response;
+    }
+  }
+}
+
+module.exports = { SlowHttp, StopScrapeError };