diff --git a/docs/resilience.md b/docs/resilience.md
index 4eced29..0705348 100644
--- a/docs/resilience.md
+++ b/docs/resilience.md
@@ -30,6 +30,12 @@ The recurring Codex check is an additional backstop, not the runtime watchdog.
| HTTP 401/403/429 or access challenge | Pause affected source lane | No automatic retry/bypass |
| Active job with no progress for 15 minutes | Sentinel requests cooperative checkpoint-safe retry | One stall recovery; next stall quarantined |
+Sentinel only treats an active job as stalled when its checkpoint can be read.
+The scraper resolves source-specific work folders (including MangaPill's
+`mangapill-` prefix) before reporting progress; a missing checkpoint is not
+evidence that downloads stopped. An operator retry resets the job's stall
+recovery count without deleting its partial download.
+
The daemon remains authoritative: a recovery request includes the observed progress
fingerprint and is rejected if the job has moved, paused, or is no longer stale.
Sentinel also observes completed/page counts; a deliberately paused queue is not a
diff --git a/scraper/adapters/weebcentral.js b/scraper/adapters/weebcentral.js
index 53af60d..d0e9bbc 100644
--- a/scraper/adapters/weebcentral.js
+++ b/scraper/adapters/weebcentral.js
@@ -21,7 +21,7 @@ const parsePageUrls = (html, baseUrl) => parser.pages({ html, url: baseUrl }).pa
async function discoverWeebCatalog(seriesUrl, http) {
const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text();
const { title, listUrl } = parse('weebcentral', 'series', { html: seriesHtml, url: seriesUrl });
- const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html' })).text() : seriesHtml;
+ const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html', catalog: true })).text() : seriesHtml;
const parsed = parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl });
const chapterLinks = parsed.chapterLinks;
const chapters = parsed.chapters.sort((a, b) =>
diff --git a/scraper/lib/http.js b/scraper/lib/http.js
index 6788cec..a44756c 100644
--- a/scraper/lib/http.js
+++ b/scraper/lib/http.js
@@ -40,6 +40,7 @@ class SlowHttp {
response = await this.transport(currentUrl, {
redirect: this.validateUrl ? 'manual' : 'follow',
signal: AbortSignal.timeout(this.timeoutMs),
+ catalog: options.catalog === true,
headers: { 'user-agent': this.userAgent, accept: options.accept || '*/*', ...(options.headers || {}) },
});
this.gate.mark();
diff --git a/scraper/plugins/broker.js b/scraper/plugins/broker.js
index c210f44..a40d3ae 100644
--- a/scraper/plugins/broker.js
+++ b/scraper/plugins/broker.js
@@ -53,7 +53,7 @@ async function pinnedFetch(value, options = {}) {
throw Object.assign(new Error('Source hostname resolved to a non-public address'), { nonRetryable: true });
}
const selected = answers[0];
- const maxBytes = options.headers?.accept === 'text/html' ? 2 * 1024 * 1024 : 50 * 1024 * 1024;
+ const maxBytes = responseByteLimit(options);
return new Promise((resolve, reject) => {
const request = https.request(url, {
method: 'GET', headers: options.headers, signal: options.signal,
@@ -88,6 +88,13 @@ async function pinnedFetch(value, options = {}) {
});
}
+function responseByteLimit(options = {}) {
+ // Chapter lists can be much larger than series/chapter HTML. Only an
+ // explicit first-party catalog request gets this bounded allowance.
+ if (options.headers?.accept === 'text/html') return (options.catalog ? 16 : 2) * 1024 * 1024;
+ return 50 * 1024 * 1024;
+}
+
function createWeebHttp(options = {}) {
return new SlowHttp({ ...options, gateKey: 'weebcentral', transport: pinnedFetch, validateUrl: validateWeebRequest });
}
@@ -97,4 +104,4 @@ function createHostHttp(sourceHost, options = {}) {
validateUrl: (value, requestOptions) => validateHostRequest(value, sourceHost, requestOptions) });
}
-module.exports = { publicAddress, validateWeebRequest, validateHostRequest, pinnedLookup, pinnedFetch, createWeebHttp, createHostHttp };
+module.exports = { publicAddress, validateWeebRequest, validateHostRequest, pinnedLookup, pinnedFetch, responseByteLimit, createWeebHttp, createHostHttp };
diff --git a/scraper/service.js b/scraper/service.js
index 274e5cb..bd59657 100644
--- a/scraper/service.js
+++ b/scraper/service.js
@@ -515,7 +515,11 @@ class ScraperService {
async jobState(item) {
if (!item) return null;
if (item.kind === 'inspection') return { status: 'discovering', updated: item.started, completed: 0, total: null, current: null };
- const file = path.join(this.libraryRoot, `.mcscraper-work-${item.seriesId || item.id}`, 'job.json');
+ const sourceId = item.seriesId || item.id;
+ const workName = item.source === 'mangapill'
+ ? `.mcscraper-work-mangapill-${sourceId}`
+ : `.mcscraper-work-${sourceId}`;
+ const file = path.join(this.libraryRoot, workName, 'job.json');
try {
const job = JSON.parse(await fs.readFile(file, 'utf8'));
return {
@@ -716,6 +720,7 @@ class ScraperService {
}
item.status = 'queued'; delete item.error; delete item.failure; delete item.nextAttemptAt;
item.recoveryAttempts = 0;
+ item.stallRecoveries = 0;
delete this.state.sourcePaused[source]; delete this.state.circuits[source];
for (const incident of this.state.incidents) if (incident.jobId === item.id && incident.state === 'open') { incident.state = 'resolved'; incident.resolution = 'operator-retry'; }
this.state.lastError = null; this.state.paused = false; this.recomputeStatus();
@@ -739,6 +744,7 @@ class ScraperService {
// Compare observations so a stale watcher cannot disturb a progressing worker.
if (Object.values(this.state.currentIds).includes(item.id)) {
const job = await this.jobState(item);
+ if (!job) return { changed: false };
if (progressKey(job) !== input.progressKey || this.state.paused || item.stopReason) return { changed: false };
if (Date.now() - Date.parse(job?.updated || item.started) < 900000) return { changed: false };
if ((item.stallRecoveries || 0) >= 1) {
diff --git a/scraper/test/checkpoint-location.test.js b/scraper/test/checkpoint-location.test.js
new file mode 100644
index 0000000..5315253
--- /dev/null
+++ b/scraper/test/checkpoint-location.test.js
@@ -0,0 +1,40 @@
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const { ScraperService } = require('../service');
+
+test('status and watchdog read the checkpoint for each source layout', async t => {
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-checkpoint-location-'));
+ t.after(() => fs.rm(root, { recursive: true, force: true }));
+ for (const [source, id, folder] of [
+ ['weebcentral', 'W1', '.mcscraper-work-W1'],
+ ['mangadex', 'D1', '.mcscraper-work-D1'],
+ ['mangapill', 'P1', '.mcscraper-work-mangapill-P1'],
+ ]) {
+ const dir = path.join(root, folder);
+ await fs.mkdir(dir);
+ await fs.writeFile(path.join(dir, 'job.json'), JSON.stringify({
+ status: 'downloading', publicationMode: 'incremental', jobCompleted: 3,
+ total: 10, updated: '2026-10-14T12:00:00Z', current: { downloadedPages: 4 },
+ }));
+ const service = Object.create(ScraperService.prototype);
+ service.libraryRoot = root;
+ const job = await service.jobState({ source, seriesId: id, kind: 'download' });
+ assert.equal(job.completed, 3, source);
+ assert.equal(job.current.downloadedPages, 4, source);
+ }
+});
+
+test('daemon refuses recovery when the checkpoint cannot be read', async () => {
+ const service = Object.create(ScraperService.prototype);
+ service.state = { paused: false, currentIds: { mangapill: 'job' },
+ queue: [{ id: 'job', source: 'mangapill', status: 'downloading' }] };
+ service.jobState = async () => null;
+ const result = await service.recover({ id: 'job', progressKey: '[0,0,"",0,""]' });
+ assert.deepEqual(result, { changed: false });
+ assert.equal(service.state.queue[0].stopReason, undefined);
+});
diff --git a/scraper/test/queue-inspection.test.js b/scraper/test/queue-inspection.test.js
index e11379f..fb8e507 100644
--- a/scraper/test/queue-inspection.test.js
+++ b/scraper/test/queue-inspection.test.js
@@ -51,9 +51,10 @@ test('retry repairs a legacy URL-ID lookup before requeueing it', async () => {
const item = await retry({ id: 'job', seriesId: '01J76XYFHVMSQVDJPQ42TN2XYN',
title: '01J76XYFHVMSQVDJPQ42TN2XYN', metadataSearch: '01J76XYFHVMSQVDJPQ42TN2XYN',
sourceTitle: 'The Exiled Heavy Knight Knows How to Game the System', source: 'weebcentral',
- status: 'unavailable', error: 'AniList HTTP 404' });
+ status: 'unavailable', error: 'AniList HTTP 404', stallRecoveries: 1 });
assert.equal(item.metadataSearch, item.sourceTitle);
assert.equal(item.title, item.sourceTitle);
assert.equal(item.status, 'queued');
assert.equal(item.error, undefined);
+ assert.equal(item.stallRecoveries, 0);
});
diff --git a/scraper/test/scraper.test.js b/scraper/test/scraper.test.js
index eae2bcb..efcb392 100644
--- a/scraper/test/scraper.test.js
+++ b/scraper/test/scraper.test.js
@@ -254,8 +254,13 @@ test('WeebCentral fragments preserve chapter and page order', () => {
test('WeebCentral reports unrecognized chapter labels separately from an empty catalog', async () => {
const series = 'Example
';
- const httpFor = list => ({ get: async url => ({ text: async () => url.endsWith('full-chapter-list') ? list : series }) });
+ const requests = [];
+ const httpFor = list => ({ get: async (url, options) => { requests.push({ url, options }); return {
+ text: async () => url.endsWith('full-chapter-list') ? list : series,
+ }; } });
const unknown = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('Part One'));
+ assert.equal(requests[0].options.catalog, undefined);
+ assert.equal(requests[1].options.catalog, true);
assert.equal(unknown.chapterLinks, 1);
assert.equal(unknown.chapters.length, 0);
assert.equal(unknown.availability.reason, 'unrecognized-labels');
diff --git a/scraper/test/source-broker.test.js b/scraper/test/source-broker.test.js
index 2602b3a..25d204b 100644
--- a/scraper/test/source-broker.test.js
+++ b/scraper/test/source-broker.test.js
@@ -3,7 +3,21 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { SlowHttp, StopScrapeError } = require('../lib/http');
-const { publicAddress, validateWeebRequest, pinnedLookup } = require('../plugins/broker');
+const { publicAddress, validateWeebRequest, pinnedLookup, responseByteLimit } = require('../plugins/broker');
+
+test('only explicit HTML catalogs get the larger bounded response allowance', async () => {
+ const html = { headers: { accept: 'text/html' } };
+ assert.equal(responseByteLimit(html), 2 * 1024 * 1024);
+ assert.equal(responseByteLimit({ ...html, catalog: true }), 16 * 1024 * 1024);
+ assert.equal(responseByteLimit({ headers: { accept: 'image/jpeg' }, catalog: true }), 50 * 1024 * 1024);
+ const calls = [];
+ const http = new SlowHttp({ minMs: 0, maxMs: 0, transport: async (_url, options) => {
+ calls.push(options.catalog); return { status: 200, ok: true };
+ } });
+ await http.get('https://weebcentral.com/series/X', { accept: 'text/html' });
+ await http.get('https://weebcentral.com/series/X/full-chapter-list', { accept: 'text/html', catalog: true });
+ assert.deepEqual(calls, [false, true]);
+});
test('plugin broker rejects local, private, non-TLS, and cross-source URLs', () => {
for (const address of ['127.0.0.1', '10.0.0.5', '192.168.1.1', '169.254.169.254', '::1', '::ffff:127.0.0.1']) {
diff --git a/sentinel/monitor.js b/sentinel/monitor.js
index 865eb43..e4e8a5e 100644
--- a/sentinel/monitor.js
+++ b/sentinel/monitor.js
@@ -36,6 +36,9 @@ class Monitor {
const ids = new Set();
for (const entry of scraper.active || []) {
const id = entry.current.id; ids.add(id);
+ // Without a checkpoint, a null progress key stays constant forever and
+ // cannot prove a stall. Do not quarantine a live worker on that basis.
+ if (!entry.job) { this.observations.delete(id); this.resolve(`stall:${id}`); continue; }
const key = progressKey(entry.job), prior = this.observations.get(id);
if (!prior || prior.key !== key) { this.observations.set(id, { key, at: now }); this.resolve(`stall:${id}`); continue; }
if (scraper.paused || scraper.sourcePaused?.[entry.source]) continue;
diff --git a/sentinel/test/monitor.test.js b/sentinel/test/monitor.test.js
index 4e7947e..f10e861 100644
--- a/sentinel/test/monitor.test.js
+++ b/sentinel/test/monitor.test.js
@@ -24,3 +24,15 @@ test('Sentinel reports service failures without restart loops and resolves recov
await monitor.observe({ ok: true }, { active: [], queue: [], paused: false });
assert.equal(monitor.state.status, 'healthy'); assert.ok(monitor.incidents.every(i => i.state === 'resolved'));
});
+
+test('missing checkpoint does not trigger a false stall recovery', async () => {
+ let now = 0, recoveries = 0;
+ const monitor = new Monitor({ now: () => now, stallMs: 100 });
+ const state = { paused: false, active: [{ source: 'mangapill', current: { id: 'job', title: 'Manga' }, job: null }], queue: [] };
+ for (const advance of [0, 1000000, 2000000]) {
+ now = advance;
+ await monitor.observe({ ok: true }, state, async () => { recoveries++; return { changed: true }; });
+ }
+ assert.equal(recoveries, 0);
+ assert.equal(monitor.incidents.some(item => item.key === 'stall:job'), false);
+});