Scraper: stop walking the whole library once a minute for status polls

The reader polled /api/state every 15 s; each call rebuilt the scraper's library
summary when it was over 60 s old, reading every chapter.json (on the NAS about
5,300 opens / 16,000 NFS calls and 1.3 s CPU per walk, once a minute, plus 115 KB
per poll). The reader now polls GET /api/revision (falls back to /api/state on an
older scraper), and the summary is rebuilt after a library change or after 15
minutes. Release-date writes invalidate it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
MaashkinoozheandClaude Opus 5.5 committed 2026-10-29 23:10:38 +01:00
1 parent b328ad98fa
commit 03767d5704
5 files changed
+40 -7

No files matched your search

+15 -4
View File
@@ -16,6 +16,7 @@ const { writeJsonAtomic } = require('./lib/json-file');
const { ReleaseDateSources, refreshMangaReleaseDates } = require('./metadata/release-dates');
const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs'];
const INVENTORY_SUMMARY_MS = 15 * 60000; // status summary age limit; decisions still scan (libraryInventory)
const FAST_RATE_SETTINGS = {
minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000,
};
@@ -401,7 +402,7 @@ class ScraperService {
} catch (error) { await this.event('warning', `Release dates for ${manga.title} failed: ${error.message}`, { mangaId: manga.id }); }
}
// The reader reconciles fully when the revision rises without a visible folder change.
if (written) { this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
if (written) { this.invalidateInventory(); this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
await this.event('info', `Release dates: ${written} chapter${written === 1 ? '' : 's'} dated in ${checked} ongoing series`, { reason, checked, written });
return { checked, written };
}
@@ -583,6 +584,15 @@ class ScraperService {
} catch { return null; }
}
inventoryStale() {
return this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= INVENTORY_SUMMARY_MS;
}
// What the reader polls every few seconds: no library walk, no event log read.
revision() {
return { publishedRevision: this.state.publishedRevision || 0, status: this.state.status, paused: this.state.paused };
}
async publicState() {
const activeItems = this.state.queue.filter(item => Object.values(this.state.currentIds || {}).includes(item.id));
const active = await Promise.all(activeItems.map(async current => ({
@@ -593,8 +603,9 @@ class ScraperService {
const job = active[0]?.job || null;
// Status and watchdog probes must not wait for thousands of bind-mount reads.
// Share one background scan, serve the previous summary, and surface its age/error.
if (!this.inventoryRefresh && (this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000)
&& Date.now() - this.inventoryAttemptedAt >= 10000) {
// The scraper is the library's only writer and invalidates the summary on every publish, so the age
// limit only catches outside edits. A walk reads every chapter.json: on a NAS that is ~10,000 opens.
if (!this.inventoryRefresh && this.inventoryStale() && Date.now() - this.inventoryAttemptedAt >= 10000) {
this.refreshInventory().catch(() => {});
}
const inventory = this.inventoryCache;
@@ -609,7 +620,7 @@ class ScraperService {
updateChecks: Object.values(this.state.updateChecks || {}),
libraryStatus: {
ready: !!inventory, refreshing: !!this.inventoryRefresh,
stale: this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000,
stale: this.inventoryStale(),
updated: this.inventoryCachedAt ? new Date(this.inventoryCachedAt).toISOString() : null,
error: this.inventoryError,
},