Scraper: stop walking the whole library once a minute for status polls

The reader polled /api/state every 15 s; each call rebuilt the scraper's library
summary when it was over 60 s old, reading every chapter.json (on the NAS about
5,300 opens / 16,000 NFS calls and 1.3 s CPU per walk, once a minute, plus 115 KB
per poll). The reader now polls GET /api/revision (falls back to /api/state on an
older scraper), and the summary is rebuilt after a library change or after 15
minutes. Release-date writes invalidate it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scraper/daemon.js

@@ -35,6 +35,7 @@ async function body(req) {
       if (url.pathname === '/health') return json(res, 200, { ok: true, status: service.state.status });
       if (TOKEN && req.headers.authorization !== `Bearer ${TOKEN}`) return json(res, 401, { error: 'Unauthorized' });
       if (req.method === 'GET' && url.pathname === '/api/state') return json(res, 200, await service.publicState());
+      if (req.method === 'GET' && url.pathname === '/api/revision') return json(res, 200, service.revision());
       if (req.method !== 'POST') return json(res, 404, { error: 'Not found' });
       const input = await body(req);
       if (url.pathname === '/api/queue') return json(res, 200, { item: await service.enqueue(input), state: await service.publicState() });

scraper/service.js

@@ -16,6 +16,7 @@ const { writeJsonAtomic } = require('./lib/json-file');
 const { ReleaseDateSources, refreshMangaReleaseDates } = require('./metadata/release-dates');
 
 const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs'];
+const INVENTORY_SUMMARY_MS = 15 * 60000; // status summary age limit; decisions still scan (libraryInventory)
 const FAST_RATE_SETTINGS = {
   minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000,
 };
@@ -401,7 +402,7 @@ class ScraperService {
       } catch (error) { await this.event('warning', `Release dates for ${manga.title} failed: ${error.message}`, { mangaId: manga.id }); }
     }
     // The reader reconciles fully when the revision rises without a visible folder change.
-    if (written) { this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
+    if (written) { this.invalidateInventory(); this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
     await this.event('info', `Release dates: ${written} chapter${written === 1 ? '' : 's'} dated in ${checked} ongoing series`, { reason, checked, written });
     return { checked, written };
   }
@@ -583,6 +584,15 @@ class ScraperService {
     } catch { return null; }
   }
 
+  inventoryStale() {
+    return this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= INVENTORY_SUMMARY_MS;
+  }
+
+  // What the reader polls every few seconds: no library walk, no event log read.
+  revision() {
+    return { publishedRevision: this.state.publishedRevision || 0, status: this.state.status, paused: this.state.paused };
+  }
+
   async publicState() {
     const activeItems = this.state.queue.filter(item => Object.values(this.state.currentIds || {}).includes(item.id));
     const active = await Promise.all(activeItems.map(async current => ({
@@ -593,8 +603,9 @@ class ScraperService {
     const job = active[0]?.job || null;
     // Status and watchdog probes must not wait for thousands of bind-mount reads.
     // Share one background scan, serve the previous summary, and surface its age/error.
-    if (!this.inventoryRefresh && (this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000)
-        && Date.now() - this.inventoryAttemptedAt >= 10000) {
+    // The scraper is the library's only writer and invalidates the summary on every publish, so the age
+    // limit only catches outside edits. A walk reads every chapter.json: on a NAS that is ~10,000 opens.
+    if (!this.inventoryRefresh && this.inventoryStale() && Date.now() - this.inventoryAttemptedAt >= 10000) {
       this.refreshInventory().catch(() => {});
     }
     const inventory = this.inventoryCache;
@@ -609,7 +620,7 @@ class ScraperService {
       updateChecks: Object.values(this.state.updateChecks || {}),
       libraryStatus: {
         ready: !!inventory, refreshing: !!this.inventoryRefresh,
-        stale: this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000,
+        stale: this.inventoryStale(),
         updated: this.inventoryCachedAt ? new Date(this.inventoryCachedAt).toISOString() : null,
         error: this.inventoryError,
       },

scraper/test/inventory-cache.test.js

@@ -80,6 +80,23 @@ test('background inventory failures are visible without breaking controls or lau
   assert.equal(scans, 2);
 });
 
+test('status polling walks the library again only after a change or after 15 minutes', async () => {
+  let scans = 0;
+  const service = fixture(async () => { scans++; return inventory('Demo'); });
+  await service.libraryInventory(true);
+  service.inventoryCachedAt = Date.now() - 10 * 60000; service.inventoryAttemptedAt = 0;
+  for (let n = 0; n < 5; n++) await service.publicState();
+  assert.equal(scans, 1);
+  assert.deepEqual(service.revision(), { publishedRevision: 0, status: 'paused', paused: true });
+  assert.equal(scans, 1);
+  service.invalidateInventory();
+  await service.publicState(); await service.inventoryRefresh;
+  assert.equal(scans, 2);
+  service.inventoryCachedAt = Date.now() - 16 * 60000; service.inventoryAttemptedAt = 0;
+  await service.publicState(); await service.inventoryRefresh;
+  assert.equal(scans, 3);
+});
+
 test('dashboard polling reuses a recent snapshot while decisions still force a fresh scan', async () => {
   let scans = 0;
   const service = fixture(async () => { scans++; return inventory('Demo'); });