Handle large WeebCentral catalogs and correct MangaPill checkpoint monitoring

scraper/test/checkpoint-location.test.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const { ScraperService } = require('../service');
+
+test('status and watchdog read the checkpoint for each source layout', async t => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-checkpoint-location-'));
+  t.after(() => fs.rm(root, { recursive: true, force: true }));
+  for (const [source, id, folder] of [
+    ['weebcentral', 'W1', '.mcscraper-work-W1'],
+    ['mangadex', 'D1', '.mcscraper-work-D1'],
+    ['mangapill', 'P1', '.mcscraper-work-mangapill-P1'],
+  ]) {
+    const dir = path.join(root, folder);
+    await fs.mkdir(dir);
+    await fs.writeFile(path.join(dir, 'job.json'), JSON.stringify({
+      status: 'downloading', publicationMode: 'incremental', jobCompleted: 3,
+      total: 10, updated: '2026-10-14T12:00:00Z', current: { downloadedPages: 4 },
+    }));
+    const service = Object.create(ScraperService.prototype);
+    service.libraryRoot = root;
+    const job = await service.jobState({ source, seriesId: id, kind: 'download' });
+    assert.equal(job.completed, 3, source);
+    assert.equal(job.current.downloadedPages, 4, source);
+  }
+});
+
+test('daemon refuses recovery when the checkpoint cannot be read', async () => {
+  const service = Object.create(ScraperService.prototype);
+  service.state = { paused: false, currentIds: { mangapill: 'job' },
+    queue: [{ id: 'job', source: 'mangapill', status: 'downloading' }] };
+  service.jobState = async () => null;
+  const result = await service.recover({ id: 'job', progressKey: '[0,0,"",0,""]' });
+  assert.deepEqual(result, { changed: false });
+  assert.equal(service.state.queue[0].stopReason, undefined);
+});

scraper/test/queue-inspection.test.js

@@ -51,9 +51,10 @@ test('retry repairs a legacy URL-ID lookup before requeueing it', async () => {
   const item = await retry({ id: 'job', seriesId: '01J76XYFHVMSQVDJPQ42TN2XYN',
     title: '01J76XYFHVMSQVDJPQ42TN2XYN', metadataSearch: '01J76XYFHVMSQVDJPQ42TN2XYN',
     sourceTitle: 'The Exiled Heavy Knight Knows How to Game the System', source: 'weebcentral',
-    status: 'unavailable', error: 'AniList HTTP 404' });
+    status: 'unavailable', error: 'AniList HTTP 404', stallRecoveries: 1 });
   assert.equal(item.metadataSearch, item.sourceTitle);
   assert.equal(item.title, item.sourceTitle);
   assert.equal(item.status, 'queued');
   assert.equal(item.error, undefined);
+  assert.equal(item.stallRecoveries, 0);
 });

scraper/test/scraper.test.js

@@ -254,8 +254,13 @@ test('WeebCentral fragments preserve chapter and page order', () => {
 
 test('WeebCentral reports unrecognized chapter labels separately from an empty catalog', async () => {
   const series = '<h1>Example</h1><div hx-get="/series/X/full-chapter-list"></div>';
-  const httpFor = list => ({ get: async url => ({ text: async () => url.endsWith('full-chapter-list') ? list : series }) });
+  const requests = [];
+  const httpFor = list => ({ get: async (url, options) => { requests.push({ url, options }); return {
+    text: async () => url.endsWith('full-chapter-list') ? list : series,
+  }; } });
   const unknown = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<a href="/chapters/ABC">Part One</a>'));
+  assert.equal(requests[0].options.catalog, undefined);
+  assert.equal(requests[1].options.catalog, true);
   assert.equal(unknown.chapterLinks, 1);
   assert.equal(unknown.chapters.length, 0);
   assert.equal(unknown.availability.reason, 'unrecognized-labels');

scraper/test/source-broker.test.js

@@ -3,7 +3,21 @@
 const test = require('node:test');
 const assert = require('node:assert/strict');
 const { SlowHttp, StopScrapeError } = require('../lib/http');
-const { publicAddress, validateWeebRequest, pinnedLookup } = require('../plugins/broker');
+const { publicAddress, validateWeebRequest, pinnedLookup, responseByteLimit } = require('../plugins/broker');
+
+test('only explicit HTML catalogs get the larger bounded response allowance', async () => {
+  const html = { headers: { accept: 'text/html' } };
+  assert.equal(responseByteLimit(html), 2 * 1024 * 1024);
+  assert.equal(responseByteLimit({ ...html, catalog: true }), 16 * 1024 * 1024);
+  assert.equal(responseByteLimit({ headers: { accept: 'image/jpeg' }, catalog: true }), 50 * 1024 * 1024);
+  const calls = [];
+  const http = new SlowHttp({ minMs: 0, maxMs: 0, transport: async (_url, options) => {
+    calls.push(options.catalog); return { status: 200, ok: true };
+  } });
+  await http.get('https://weebcentral.com/series/X', { accept: 'text/html' });
+  await http.get('https://weebcentral.com/series/X/full-chapter-list', { accept: 'text/html', catalog: true });
+  assert.deepEqual(calls, [false, true]);
+});
 
 test('plugin broker rejects local, private, non-TLS, and cross-source URLs', () => {
   for (const address of ['127.0.0.1', '10.0.0.5', '192.168.1.1', '169.254.169.254', '::1', '::ffff:127.0.0.1']) {