Checkpoint guarded source request broker

scraper/lib/http.js

@@ -11,39 +11,55 @@ class SlowHttp {
     this.maxRetries = options.maxRetries ?? 2;
     this.onEvent = options.onEvent;
     this.timeoutMs = options.timeoutMs || 60000;
+    this.transport = options.transport || fetch;
+    this.validateUrl = options.validateUrl || null;
   }
 
   async get(url, options = {}) {
+    let redirects = 0;
+    let currentUrl = url;
     for (let attempt = 0; ; attempt++) {
+      if (this.validateUrl) this.validateUrl(currentUrl, options);
       await this.gate.wait();
       let response;
       try {
-        response = await fetch(url, {
-          redirect: 'follow',
+        response = await this.transport(currentUrl, {
+          redirect: this.validateUrl ? 'manual' : 'follow',
           signal: AbortSignal.timeout(this.timeoutMs),
           headers: { 'user-agent': this.userAgent, accept: options.accept || '*/*', ...(options.headers || {}) },
         });
         this.gate.mark();
       } catch (error) {
         this.gate.mark();
-        if (attempt >= this.maxRetries) throw new Error(`Network request failed after ${attempt + 1} attempts: ${url}`, { cause: error });
+        if (error.nonRetryable) throw error;
+        if (attempt >= this.maxRetries) throw new Error(`Network request failed after ${attempt + 1} attempts: ${currentUrl}`, { cause: error });
         const backoff = 30000 * (2 ** attempt);
         const message = `Network failure; retrying in ${backoff / 1000}s (${attempt + 1}/${this.maxRetries})`;
         console.warn(`[http] ${message}`);
-        if (this.onEvent) await this.onEvent({ level: 'warning', message, url });
+        if (this.onEvent) await this.onEvent({ level: 'warning', message, url: currentUrl });
         await sleep(backoff);
         continue;
       }
 
+      if (this.validateUrl && response.status >= 300 && response.status < 400) {
+        const location = response.headers.get('location');
+        if (!location || ++redirects > 5) throw new Error('Source redirect limit or missing location');
+        currentUrl = new URL(location, currentUrl).href;
+        this.validateUrl(currentUrl, options);
+        await response.body?.cancel?.();
+        attempt = -1;
+        continue;
+      }
+
       if ([401, 403, 429].includes(response.status)) {
-        if (this.onEvent) await this.onEvent({ level: 'error', message: `Host returned ${response.status}; stopping`, url, status: response.status });
-        throw Object.assign(new StopScrapeError(`Host returned ${response.status}; stop and require human review: ${url}`), { status: response.status });
+        if (this.onEvent) await this.onEvent({ level: 'error', message: `Host returned ${response.status}; stopping`, url: currentUrl, status: response.status });
+        throw Object.assign(new StopScrapeError(`Host returned ${response.status}; stop and require human review: ${currentUrl}`), { status: response.status });
       }
       if (response.status >= 500 && attempt < this.maxRetries) {
         await sleep(30000 * (2 ** attempt));
         continue;
       }
-      if (!response.ok) throw Object.assign(new Error(`HTTP ${response.status}: ${url}`), { status: response.status });
+      if (!response.ok) throw Object.assign(new Error(`HTTP ${response.status}: ${currentUrl}`), { status: response.status });
       return response;
     }
   }