diff --git a/docs/scraper-service.md b/docs/scraper-service.md index 0685db8..d4123e4 100644 --- a/docs/scraper-service.md +++ b/docs/scraper-service.md @@ -83,7 +83,7 @@ abandoned hidden partial stages, atomically exposes the accumulated manga folder changes its checkpoint to incremental mode. The resumed job then derives its remaining work from the visible library and publishes every later chapter immediately. -A source failure pauses only that source, allowing unrelated source workers to continue. Manual pause remains global. Request and chapter delays are configured independently for every installed host in **Settings → Scraper**. Both current adapters default to the fast profile: 1–4 seconds between sequential requests and 5–15 seconds between chapters. +A source failure pauses only that source, allowing unrelated source workers to continue. Manual pause remains global. Request and chapter delays are configured independently for every installed host in **Settings → Scraper**. The current adapters default to the fast profile: 1–4 seconds between sequential requests and 5–15 seconds between chapters. The in-process request gate is shared across successive jobs on the same source, so changing titles does not reset the request gap. The reader polls `publishedRevision` every 15 seconds and rescans after it increases. A source error pauses only that source lane for human review. Pause and shutdown requests @@ -202,7 +202,7 @@ All responses are JSON. When `SCRAPER_TOKEN` is set, send | `POST` | `/api/queue/remove` | `{id}`; active downloads cannot be removed | | `POST` | `/api/queue/clear-completed` | `{source?}`; remove inactive successful download job rows only; preserve library, history, checkpoints and other statuses | | `POST` | `/api/queue/reorder` | `{ids}` containing every queue id exactly once | -| `POST` | `/api/control` | `{action:"pause"|"resume"|"retry"|"inspect"|"confirm"|"skip",id?}` | +| `POST` | `/api/control` | `{action:"pause"|"resume"|"retry"|"inspect"|"confirm"|"skip",id?}`; `pause-source`/`resume-source` use `id` as the source name and do not change the global pause | | `POST` | `/api/settings` | `sourceRates.{adapter}.{minMs,maxMs,chapterMinMs,chapterMaxMs}` plus `autoUpdateEnabled` and local `autoUpdateTime` (`HH:MM`) | | `POST` | `/api/discovery/run` | Queue an immediate update check for every enabled manga | | `POST` | `/api/library/auto-update` | `{id,enabled}` updates `manga.json` atomically | diff --git a/scraper/jobs/mangadex-archive.js b/scraper/jobs/mangadex-archive.js index 8305f6a..c0bb1db 100644 --- a/scraper/jobs/mangadex-archive.js +++ b/scraper/jobs/mangadex-archive.js @@ -69,7 +69,7 @@ async function runMangaDexArchive(options) { const chapterMinMs = Number(options.chapterMinMs); const chapterMaxMs = Number(options.chapterMaxMs); const http = new SlowHttp({ - minMs, maxMs, maxRetries: 2, + gateKey: 'mangadex', minMs, maxMs, maxRetries: 2, userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)', onEvent: item => event(item.level, item.message, { status: item.status }), }); diff --git a/scraper/lib/http.js b/scraper/lib/http.js index 0279170..6788cec 100644 --- a/scraper/lib/http.js +++ b/scraper/lib/http.js @@ -3,10 +3,24 @@ const { PoliteGate, sleep } = require('./delay'); class StopScrapeError extends Error {} +const sourceGates = new Map(); + +function gateFor(options) { + const next = new PoliteGate(options); + if (!options.gateKey) return next; + const key = String(options.gateKey); + const existing = sourceGates.get(key); + if (!existing) { sourceGates.set(key, next); return next; } + // A new job may use updated host settings, but it must not forget when the + // previous job last requested this same source. + existing.minMs = next.minMs; + existing.maxMs = next.maxMs; + return existing; +} class SlowHttp { constructor(options = {}) { - this.gate = new PoliteGate(options); + this.gate = gateFor(options); this.userAgent = options.userAgent || 'MangaCircus-Scraper/1.0 (personal archival; sequential requests)'; this.maxRetries = options.maxRetries ?? 2; this.onEvent = options.onEvent; diff --git a/scraper/plugins/broker.js b/scraper/plugins/broker.js index 18be820..c210f44 100644 --- a/scraper/plugins/broker.js +++ b/scraper/plugins/broker.js @@ -89,11 +89,11 @@ async function pinnedFetch(value, options = {}) { } function createWeebHttp(options = {}) { - return new SlowHttp({ ...options, transport: pinnedFetch, validateUrl: validateWeebRequest }); + return new SlowHttp({ ...options, gateKey: 'weebcentral', transport: pinnedFetch, validateUrl: validateWeebRequest }); } function createHostHttp(sourceHost, options = {}) { - return new SlowHttp({ ...options, transport: pinnedFetch, + return new SlowHttp({ ...options, gateKey: sourceHost, transport: pinnedFetch, validateUrl: (value, requestOptions) => validateHostRequest(value, sourceHost, requestOptions) }); } diff --git a/scraper/service.js b/scraper/service.js index 057e58b..94905d0 100644 --- a/scraper/service.js +++ b/scraper/service.js @@ -634,6 +634,23 @@ class ScraperService { } async control(action, id) { + if (action === 'pause-source' || action === 'resume-source') { + const source = String(id || ''); + if (!this.adapterList.some(adapter => adapter.id === source)) throw new Error('Unknown source'); + if (action === 'resume-source' && this.state.queue.some(item => item.source === source + && item.status === 'error' && item.failure?.kind === 'access')) { + throw new Error('Review the source access block before resuming'); + } + if (action === 'pause-source') { + this.state.sourcePaused[source] = true; + const activeId = this.state.currentIds[source]; + const activeItem = this.state.queue.find(item => item.id === activeId); + if (activeItem) activeItem.stopReason = 'paused'; + } else delete this.state.sourcePaused[source]; + this.recomputeStatus(); + await this.event('info', `${source} source ${action === 'pause-source' ? 'paused' : 'resumed'} by operator`, { source }); + return; + } if (action === 'inspect' || action === 'confirm') { const item = this.state.queue.find(entry => entry.id === id); if (!item || Object.values(this.state.currentIds || {}).includes(item.id)) throw new Error('Job not found or already active'); diff --git a/scraper/sources/mangadex.js b/scraper/sources/mangadex.js index c49a27a..12236c9 100644 --- a/scraper/sources/mangadex.js +++ b/scraper/sources/mangadex.js @@ -8,7 +8,7 @@ const { controlledHttp } = require('./common'); module.exports = { id: 'mangadex', label: 'MangaDex', match: mangaDexInfo, async discover(item, options) { - const http = new SlowHttp({ minMs: options.settings.minMs, maxMs: options.settings.maxMs, + const http = new SlowHttp({ gateKey: 'mangadex', minMs: options.settings.minMs, maxMs: options.settings.maxMs, maxRetries: 2, userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)', onEvent: options.onEvent }); return discoverMangaDexCatalog(item.url, controlledHttp(http, options.shouldStop)); diff --git a/scraper/test/source-broker.test.js b/scraper/test/source-broker.test.js index 36084ae..2602b3a 100644 --- a/scraper/test/source-broker.test.js +++ b/scraper/test/source-broker.test.js @@ -54,3 +54,15 @@ test('a guarded 403 stops immediately without redirect or retry', async () => { await assert.rejects(http.get('https://weebcentral.com/series/X', { accept: 'text/html' }), StopScrapeError); assert.equal(calls, 1); }); + +test('separate jobs on one source retain the configured request gap', async () => { + const gateKey = `fixture-${Date.now()}-${Math.random()}`; + const transport = async () => ({ status: 200, ok: true }); + const first = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport }); + await first.get('https://example.org/one'); + const second = new SlowHttp({ gateKey, minMs: 40, maxMs: 40, transport }); + assert.equal(second.gate, first.gate); + const started = Date.now(); + await second.get('https://example.org/two'); + assert.ok(Date.now() - started >= 30, 'a new job must not reset its source delay'); +}); diff --git a/scraper/test/source-pause.test.js b/scraper/test/source-pause.test.js new file mode 100644 index 0000000..6513971 --- /dev/null +++ b/scraper/test/source-pause.test.js @@ -0,0 +1,26 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { ScraperService } = require('../service'); + +test('an operator can pause one source while other lanes continue', async t => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-source-pause-')); + const service = await new ScraperService({ libraryRoot: path.join(root, 'library'), + dataRoot: path.join(root, 'data') }).init(); + t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); }); + await assert.rejects(service.control('pause-source', 'unknown'), /Unknown source/); + await service.control('pause-source', 'weebcentral'); + assert.equal(service.state.sourcePaused.weebcentral, true); + await service.control('resume'); + assert.equal(service.state.paused, false); + assert.equal(service.state.sourcePaused.weebcentral, true); + service.state.queue.push({ source: 'weebcentral', status: 'error', failure: { kind: 'access' } }); + await assert.rejects(service.control('resume-source', 'weebcentral'), /access block/); + service.state.queue.pop(); + await service.control('resume-source', 'weebcentral'); + assert.equal(service.state.sourcePaused.weebcentral, undefined); +});