Keep source plugins in single scraper process
This commit is contained in:
1 parent
0fc1a7b52a
commit
6d713e0a8c
19 files changed
+202
-467
No files matched your search
@@ -2,7 +2,6 @@
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { SlowHttp } = require('../lib/http');
|
||||
const { createWeebHttp } = require('../plugins/broker');
|
||||
const { sleep } = require('../lib/delay');
|
||||
const { ArchiveControlError } = require('../lib/control');
|
||||
@@ -73,7 +72,7 @@ async function runArchive(options) {
|
||||
minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2,
|
||||
onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }),
|
||||
};
|
||||
const http = process.env.SOURCE_PLUGIN_URL ? createWeebHttp(httpOptions) : new SlowHttp(httpOptions);
|
||||
const http = createWeebHttp(httpOptions);
|
||||
|
||||
checkControl();
|
||||
console.log(`[archive] discovering catalog ${options.seriesUrl}`);
|
||||
|
||||
Reference in new issue
Block a user