Handle large WeebCentral catalogs and correct MangaPill checkpoint monitoring

scraper/plugins/broker.js

@@ -53,7 +53,7 @@ async function pinnedFetch(value, options = {}) {
     throw Object.assign(new Error('Source hostname resolved to a non-public address'), { nonRetryable: true });
   }
   const selected = answers[0];
-  const maxBytes = options.headers?.accept === 'text/html' ? 2 * 1024 * 1024 : 50 * 1024 * 1024;
+  const maxBytes = responseByteLimit(options);
   return new Promise((resolve, reject) => {
     const request = https.request(url, {
       method: 'GET', headers: options.headers, signal: options.signal,
@@ -88,6 +88,13 @@ async function pinnedFetch(value, options = {}) {
   });
 }
 
+function responseByteLimit(options = {}) {
+  // Chapter lists can be much larger than series/chapter HTML. Only an
+  // explicit first-party catalog request gets this bounded allowance.
+  if (options.headers?.accept === 'text/html') return (options.catalog ? 16 : 2) * 1024 * 1024;
+  return 50 * 1024 * 1024;
+}
+
 function createWeebHttp(options = {}) {
   return new SlowHttp({ ...options, gateKey: 'weebcentral', transport: pinnedFetch, validateUrl: validateWeebRequest });
 }
@@ -97,4 +104,4 @@ function createHostHttp(sourceHost, options = {}) {
     validateUrl: (value, requestOptions) => validateHostRequest(value, sourceHost, requestOptions) });
 }
 
-module.exports = { publicAddress, validateWeebRequest, validateHostRequest, pinnedLookup, pinnedFetch, createWeebHttp, createHostHttp };
+module.exports = { publicAddress, validateWeebRequest, validateHostRequest, pinnedLookup, pinnedFetch, responseByteLimit, createWeebHttp, createHostHttp };