Handle large WeebCentral catalogs and correct MangaPill checkpoint monitoring

docs/resilience.md

@@ -30,6 +30,12 @@ The recurring Codex check is an additional backstop, not the runtime watchdog.
 | HTTP 401/403/429 or access challenge | Pause affected source lane | No automatic retry/bypass |
 | Active job with no progress for 15 minutes | Sentinel requests cooperative checkpoint-safe retry | One stall recovery; next stall quarantined |
 
+Sentinel only treats an active job as stalled when its checkpoint can be read.
+The scraper resolves source-specific work folders (including MangaPill's
+`mangapill-` prefix) before reporting progress; a missing checkpoint is not
+evidence that downloads stopped. An operator retry resets the job's stall
+recovery count without deleting its partial download.
+
 The daemon remains authoritative: a recovery request includes the observed progress
 fingerprint and is rejected if the job has moved, paused, or is no longer stale.
 Sentinel also observes completed/page counts; a deliberately paused queue is not a

scraper/adapters/weebcentral.js

@@ -21,7 +21,7 @@ const parsePageUrls = (html, baseUrl) => parser.pages({ html, url: baseUrl }).pa
 async function discoverWeebCatalog(seriesUrl, http) {
   const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text();
   const { title, listUrl } = parse('weebcentral', 'series', { html: seriesHtml, url: seriesUrl });
-  const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html' })).text() : seriesHtml;
+  const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html', catalog: true })).text() : seriesHtml;
   const parsed = parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl });
   const chapterLinks = parsed.chapterLinks;
   const chapters = parsed.chapters.sort((a, b) =>

scraper/lib/http.js

@@ -40,6 +40,7 @@ class SlowHttp {
         response = await this.transport(currentUrl, {
           redirect: this.validateUrl ? 'manual' : 'follow',
           signal: AbortSignal.timeout(this.timeoutMs),
+          catalog: options.catalog === true,
           headers: { 'user-agent': this.userAgent, accept: options.accept || '*/*', ...(options.headers || {}) },
         });
         this.gate.mark();

scraper/plugins/broker.js

@@ -53,7 +53,7 @@ async function pinnedFetch(value, options = {}) {
     throw Object.assign(new Error('Source hostname resolved to a non-public address'), { nonRetryable: true });
   }
   const selected = answers[0];
-  const maxBytes = options.headers?.accept === 'text/html' ? 2 * 1024 * 1024 : 50 * 1024 * 1024;
+  const maxBytes = responseByteLimit(options);
   return new Promise((resolve, reject) => {
     const request = https.request(url, {
       method: 'GET', headers: options.headers, signal: options.signal,
@@ -88,6 +88,13 @@ async function pinnedFetch(value, options = {}) {
   });
 }
 
+function responseByteLimit(options = {}) {
+  // Chapter lists can be much larger than series/chapter HTML. Only an
+  // explicit first-party catalog request gets this bounded allowance.
+  if (options.headers?.accept === 'text/html') return (options.catalog ? 16 : 2) * 1024 * 1024;
+  return 50 * 1024 * 1024;
+}
+
 function createWeebHttp(options = {}) {
   return new SlowHttp({ ...options, gateKey: 'weebcentral', transport: pinnedFetch, validateUrl: validateWeebRequest });
 }
@@ -97,4 +104,4 @@ function createHostHttp(sourceHost, options = {}) {
     validateUrl: (value, requestOptions) => validateHostRequest(value, sourceHost, requestOptions) });
 }
 
-module.exports = { publicAddress, validateWeebRequest, validateHostRequest, pinnedLookup, pinnedFetch, createWeebHttp, createHostHttp };
+module.exports = { publicAddress, validateWeebRequest, validateHostRequest, pinnedLookup, pinnedFetch, responseByteLimit, createWeebHttp, createHostHttp };

scraper/service.js

@@ -515,7 +515,11 @@ class ScraperService {
   async jobState(item) {
     if (!item) return null;
     if (item.kind === 'inspection') return { status: 'discovering', updated: item.started, completed: 0, total: null, current: null };
-    const file = path.join(this.libraryRoot, `.mcscraper-work-${item.seriesId || item.id}`, 'job.json');
+    const sourceId = item.seriesId || item.id;
+    const workName = item.source === 'mangapill'
+      ? `.mcscraper-work-mangapill-${sourceId}`
+      : `.mcscraper-work-${sourceId}`;
+    const file = path.join(this.libraryRoot, workName, 'job.json');
     try {
       const job = JSON.parse(await fs.readFile(file, 'utf8'));
       return {
@@ -716,6 +720,7 @@ class ScraperService {
       }
       item.status = 'queued'; delete item.error; delete item.failure; delete item.nextAttemptAt;
       item.recoveryAttempts = 0;
+      item.stallRecoveries = 0;
       delete this.state.sourcePaused[source]; delete this.state.circuits[source];
       for (const incident of this.state.incidents) if (incident.jobId === item.id && incident.state === 'open') { incident.state = 'resolved'; incident.resolution = 'operator-retry'; }
       this.state.lastError = null; this.state.paused = false; this.recomputeStatus();
@@ -739,6 +744,7 @@ class ScraperService {
     // Compare observations so a stale watcher cannot disturb a progressing worker.
     if (Object.values(this.state.currentIds).includes(item.id)) {
       const job = await this.jobState(item);
+      if (!job) return { changed: false };
       if (progressKey(job) !== input.progressKey || this.state.paused || item.stopReason) return { changed: false };
       if (Date.now() - Date.parse(job?.updated || item.started) < 900000) return { changed: false };
       if ((item.stallRecoveries || 0) >= 1) {

scraper/test/checkpoint-location.test.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const { ScraperService } = require('../service');
+
+test('status and watchdog read the checkpoint for each source layout', async t => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-checkpoint-location-'));
+  t.after(() => fs.rm(root, { recursive: true, force: true }));
+  for (const [source, id, folder] of [
+    ['weebcentral', 'W1', '.mcscraper-work-W1'],
+    ['mangadex', 'D1', '.mcscraper-work-D1'],
+    ['mangapill', 'P1', '.mcscraper-work-mangapill-P1'],
+  ]) {
+    const dir = path.join(root, folder);
+    await fs.mkdir(dir);
+    await fs.writeFile(path.join(dir, 'job.json'), JSON.stringify({
+      status: 'downloading', publicationMode: 'incremental', jobCompleted: 3,
+      total: 10, updated: '2026-10-14T12:00:00Z', current: { downloadedPages: 4 },
+    }));
+    const service = Object.create(ScraperService.prototype);
+    service.libraryRoot = root;
+    const job = await service.jobState({ source, seriesId: id, kind: 'download' });
+    assert.equal(job.completed, 3, source);
+    assert.equal(job.current.downloadedPages, 4, source);
+  }
+});
+
+test('daemon refuses recovery when the checkpoint cannot be read', async () => {
+  const service = Object.create(ScraperService.prototype);
+  service.state = { paused: false, currentIds: { mangapill: 'job' },
+    queue: [{ id: 'job', source: 'mangapill', status: 'downloading' }] };
+  service.jobState = async () => null;
+  const result = await service.recover({ id: 'job', progressKey: '[0,0,"",0,""]' });
+  assert.deepEqual(result, { changed: false });
+  assert.equal(service.state.queue[0].stopReason, undefined);
+});

scraper/test/queue-inspection.test.js

@@ -51,9 +51,10 @@ test('retry repairs a legacy URL-ID lookup before requeueing it', async () => {
   const item = await retry({ id: 'job', seriesId: '01J76XYFHVMSQVDJPQ42TN2XYN',
     title: '01J76XYFHVMSQVDJPQ42TN2XYN', metadataSearch: '01J76XYFHVMSQVDJPQ42TN2XYN',
     sourceTitle: 'The Exiled Heavy Knight Knows How to Game the System', source: 'weebcentral',
-    status: 'unavailable', error: 'AniList HTTP 404' });
+    status: 'unavailable', error: 'AniList HTTP 404', stallRecoveries: 1 });
   assert.equal(item.metadataSearch, item.sourceTitle);
   assert.equal(item.title, item.sourceTitle);
   assert.equal(item.status, 'queued');
   assert.equal(item.error, undefined);
+  assert.equal(item.stallRecoveries, 0);
 });

scraper/test/scraper.test.js

@@ -254,8 +254,13 @@ test('WeebCentral fragments preserve chapter and page order', () => {
 
 test('WeebCentral reports unrecognized chapter labels separately from an empty catalog', async () => {
   const series = '<h1>Example</h1><div hx-get="/series/X/full-chapter-list"></div>';
-  const httpFor = list => ({ get: async url => ({ text: async () => url.endsWith('full-chapter-list') ? list : series }) });
+  const requests = [];
+  const httpFor = list => ({ get: async (url, options) => { requests.push({ url, options }); return {
+    text: async () => url.endsWith('full-chapter-list') ? list : series,
+  }; } });
   const unknown = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<a href="/chapters/ABC">Part One</a>'));
+  assert.equal(requests[0].options.catalog, undefined);
+  assert.equal(requests[1].options.catalog, true);
   assert.equal(unknown.chapterLinks, 1);
   assert.equal(unknown.chapters.length, 0);
   assert.equal(unknown.availability.reason, 'unrecognized-labels');

scraper/test/source-broker.test.js

@@ -3,7 +3,21 @@
 const test = require('node:test');
 const assert = require('node:assert/strict');
 const { SlowHttp, StopScrapeError } = require('../lib/http');
-const { publicAddress, validateWeebRequest, pinnedLookup } = require('../plugins/broker');
+const { publicAddress, validateWeebRequest, pinnedLookup, responseByteLimit } = require('../plugins/broker');
+
+test('only explicit HTML catalogs get the larger bounded response allowance', async () => {
+  const html = { headers: { accept: 'text/html' } };
+  assert.equal(responseByteLimit(html), 2 * 1024 * 1024);
+  assert.equal(responseByteLimit({ ...html, catalog: true }), 16 * 1024 * 1024);
+  assert.equal(responseByteLimit({ headers: { accept: 'image/jpeg' }, catalog: true }), 50 * 1024 * 1024);
+  const calls = [];
+  const http = new SlowHttp({ minMs: 0, maxMs: 0, transport: async (_url, options) => {
+    calls.push(options.catalog); return { status: 200, ok: true };
+  } });
+  await http.get('https://weebcentral.com/series/X', { accept: 'text/html' });
+  await http.get('https://weebcentral.com/series/X/full-chapter-list', { accept: 'text/html', catalog: true });
+  assert.deepEqual(calls, [false, true]);
+});
 
 test('plugin broker rejects local, private, non-TLS, and cross-source URLs', () => {
   for (const address of ['127.0.0.1', '10.0.0.5', '192.168.1.1', '169.254.169.254', '::1', '::ffff:127.0.0.1']) {

sentinel/monitor.js

@@ -36,6 +36,9 @@ class Monitor {
     const ids = new Set();
     for (const entry of scraper.active || []) {
       const id = entry.current.id; ids.add(id);
+      // Without a checkpoint, a null progress key stays constant forever and
+      // cannot prove a stall. Do not quarantine a live worker on that basis.
+      if (!entry.job) { this.observations.delete(id); this.resolve(`stall:${id}`); continue; }
       const key = progressKey(entry.job), prior = this.observations.get(id);
       if (!prior || prior.key !== key) { this.observations.set(id, { key, at: now }); this.resolve(`stall:${id}`); continue; }
       if (scraper.paused || scraper.sourcePaused?.[entry.source]) continue;

sentinel/test/monitor.test.js

@@ -24,3 +24,15 @@ test('Sentinel reports service failures without restart loops and resolves recov
   await monitor.observe({ ok: true }, { active: [], queue: [], paused: false });
   assert.equal(monitor.state.status, 'healthy'); assert.ok(monitor.incidents.every(i => i.state === 'resolved'));
 });
+
+test('missing checkpoint does not trigger a false stall recovery', async () => {
+  let now = 0, recoveries = 0;
+  const monitor = new Monitor({ now: () => now, stallMs: 100 });
+  const state = { paused: false, active: [{ source: 'mangapill', current: { id: 'job', title: 'Manga' }, job: null }], queue: [] };
+  for (const advance of [0, 1000000, 2000000]) {
+    now = advance;
+    await monitor.observe({ ok: true }, state, async () => { recoveries++; return { changed: true }; });
+  }
+  assert.equal(recoveries, 0);
+  assert.equal(monitor.incidents.some(item => item.key === 'stall:job'), false);
+});