Scraper: stop walking the whole library once a minute for status polls

The reader polled /api/state every 15 s; each call rebuilt the scraper's library
summary when it was over 60 s old, reading every chapter.json (on the NAS about
5,300 opens / 16,000 NFS calls and 1.3 s CPU per walk, once a minute, plus 115 KB
per poll). The reader now polls GET /api/revision (falls back to /api/state on an
older scraper), and the summary is rebuilt after a library change or after 15
minutes. Release-date writes invalidate it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/scraper-service.md

@@ -199,7 +199,8 @@ All responses are JSON. When `SCRAPER_TOKEN` is set, send
 | Method | Path | Body / result |
 |---|---|---|
 | `GET` | `/health` | Basic liveness |
-| `GET` | `/api/state` | Queue, all active source checkpoints, settings, errors, history, adapters |
+| `GET` | `/api/state` | Queue, all active source checkpoints, settings, errors, history, adapters. The library summary in it is rebuilt in the background after a publish, or when it is 15 minutes old (the walk reads every `chapter.json`) |
+| `GET` | `/api/revision` | `{ publishedRevision, status, paused }` only; the reader polls this every 15 s |
 | `POST` | `/api/queue` | `{url,title?,metadataSearch?,scope?,existingPolicy?,libraryMangaId?,inspect?}` |
 | `POST` | `/api/queue/remove` | `{id}`; active downloads cannot be removed |
 | `POST` | `/api/queue/clear-completed` | `{source?}`; remove inactive successful download job rows only; preserve library, history, checkpoints and other statuses |

scraper/daemon.js

@@ -35,6 +35,7 @@ async function body(req) {
       if (url.pathname === '/health') return json(res, 200, { ok: true, status: service.state.status });
       if (TOKEN && req.headers.authorization !== `Bearer ${TOKEN}`) return json(res, 401, { error: 'Unauthorized' });
       if (req.method === 'GET' && url.pathname === '/api/state') return json(res, 200, await service.publicState());
+      if (req.method === 'GET' && url.pathname === '/api/revision') return json(res, 200, service.revision());
       if (req.method !== 'POST') return json(res, 404, { error: 'Not found' });
       const input = await body(req);
       if (url.pathname === '/api/queue') return json(res, 200, { item: await service.enqueue(input), state: await service.publicState() });

scraper/service.js

@@ -16,6 +16,7 @@ const { writeJsonAtomic } = require('./lib/json-file');
 const { ReleaseDateSources, refreshMangaReleaseDates } = require('./metadata/release-dates');
 
 const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs'];
+const INVENTORY_SUMMARY_MS = 15 * 60000; // status summary age limit; decisions still scan (libraryInventory)
 const FAST_RATE_SETTINGS = {
   minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000,
 };
@@ -401,7 +402,7 @@ class ScraperService {
       } catch (error) { await this.event('warning', `Release dates for ${manga.title} failed: ${error.message}`, { mangaId: manga.id }); }
     }
     // The reader reconciles fully when the revision rises without a visible folder change.
-    if (written) { this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
+    if (written) { this.invalidateInventory(); this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
     await this.event('info', `Release dates: ${written} chapter${written === 1 ? '' : 's'} dated in ${checked} ongoing series`, { reason, checked, written });
     return { checked, written };
   }
@@ -583,6 +584,15 @@ class ScraperService {
     } catch { return null; }
   }
 
+  inventoryStale() {
+    return this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= INVENTORY_SUMMARY_MS;
+  }
+
+  // What the reader polls every few seconds: no library walk, no event log read.
+  revision() {
+    return { publishedRevision: this.state.publishedRevision || 0, status: this.state.status, paused: this.state.paused };
+  }
+
   async publicState() {
     const activeItems = this.state.queue.filter(item => Object.values(this.state.currentIds || {}).includes(item.id));
     const active = await Promise.all(activeItems.map(async current => ({
@@ -593,8 +603,9 @@ class ScraperService {
     const job = active[0]?.job || null;
     // Status and watchdog probes must not wait for thousands of bind-mount reads.
     // Share one background scan, serve the previous summary, and surface its age/error.
-    if (!this.inventoryRefresh && (this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000)
-        && Date.now() - this.inventoryAttemptedAt >= 10000) {
+    // The scraper is the library's only writer and invalidates the summary on every publish, so the age
+    // limit only catches outside edits. A walk reads every chapter.json: on a NAS that is ~10,000 opens.
+    if (!this.inventoryRefresh && this.inventoryStale() && Date.now() - this.inventoryAttemptedAt >= 10000) {
       this.refreshInventory().catch(() => {});
     }
     const inventory = this.inventoryCache;
@@ -609,7 +620,7 @@ class ScraperService {
       updateChecks: Object.values(this.state.updateChecks || {}),
       libraryStatus: {
         ready: !!inventory, refreshing: !!this.inventoryRefresh,
-        stale: this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000,
+        stale: this.inventoryStale(),
         updated: this.inventoryCachedAt ? new Date(this.inventoryCachedAt).toISOString() : null,
         error: this.inventoryError,
       },

scraper/test/inventory-cache.test.js

@@ -80,6 +80,23 @@ test('background inventory failures are visible without breaking controls or lau
   assert.equal(scans, 2);
 });
 
+test('status polling walks the library again only after a change or after 15 minutes', async () => {
+  let scans = 0;
+  const service = fixture(async () => { scans++; return inventory('Demo'); });
+  await service.libraryInventory(true);
+  service.inventoryCachedAt = Date.now() - 10 * 60000; service.inventoryAttemptedAt = 0;
+  for (let n = 0; n < 5; n++) await service.publicState();
+  assert.equal(scans, 1);
+  assert.deepEqual(service.revision(), { publishedRevision: 0, status: 'paused', paused: true });
+  assert.equal(scans, 1);
+  service.invalidateInventory();
+  await service.publicState(); await service.inventoryRefresh;
+  assert.equal(scans, 2);
+  service.inventoryCachedAt = Date.now() - 16 * 60000; service.inventoryAttemptedAt = 0;
+  await service.publicState(); await service.inventoryRefresh;
+  assert.equal(scans, 3);
+});
+
 test('dashboard polling reuses a recent snapshot while decisions still force a fresh scan', async () => {
   let scans = 0;
   const service = fixture(async () => { scans++; return inventory('Demo'); });

server.js

@@ -1218,10 +1218,13 @@ if (FULL_SCAN_MINUTES > 0) setInterval(() => {
 
 // A completed scraper publish increments a durable revision. Polling it here keeps the
 // reader index fresh without exposing an unauthenticated rescan endpoint to the daemon.
-let scraperRevision = null;
+let scraperRevision = null, revisionRoute = true;
 async function pollScraper() {
   try {
-    const state = await scraperRequest('/api/state');
+    // /api/revision is a few bytes; the full state is ~100 KB and makes the scraper summarise the library.
+    let state;
+    if (revisionRoute) state = await scraperRequest('/api/revision').catch(error => { if (error.code !== 404) throw error; revisionRoute = false; });
+    if (!revisionRoute) state = await scraperRequest('/api/state'); // a scraper from before /api/revision
     const revision = Number(state.publishedRevision || 0);
     if (scraperRevision === null || revision > scraperRevision) {
       const checked = await refreshChanged();