Checkpoint guarded source request broker

scraper/jobs/weebcentral-archive.js

@@ -3,6 +3,7 @@
 const fs = require('fs').promises;
 const path = require('path');
 const { SlowHttp } = require('../lib/http');
+const { createWeebHttp } = require('../plugins/broker');
 const { sleep } = require('../lib/delay');
 const { ArchiveControlError } = require('../lib/control');
 const { cleanPart } = require('../lib/names');
@@ -68,10 +69,11 @@ async function runArchive(options) {
     state.updated = entry.time;
     await writeJsonAtomic(stateFile, state);
   };
-  const http = new SlowHttp({
+  const httpOptions = {
     minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2,
     onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }),
-  });
+  };
+  const http = process.env.SOURCE_PLUGIN_URL ? createWeebHttp(httpOptions) : new SlowHttp(httpOptions);
 
   checkControl();
   console.log(`[archive] discovering catalog ${options.seriesUrl}`);