Scraper: stop walking the whole library once a minute for status polls
The reader polled /api/state every 15 s; each call rebuilt the scraper's library summary when it was over 60 s old, reading every chapter.json (on the NAS about 5,300 opens / 16,000 NFS calls and 1.3 s CPU per walk, once a minute, plus 115 KB per poll). The reader now polls GET /api/revision (falls back to /api/state on an older scraper), and the summary is rebuilt after a library change or after 15 minutes. Release-date writes invalidate it too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b328ad98fa
commit
03767d5704
5 files changed
+40
-7
No files matched your search
+15
-4
@@ -16,6 +16,7 @@ const { writeJsonAtomic } = require('./lib/json-file');
|
||||
const { ReleaseDateSources, refreshMangaReleaseDates } = require('./metadata/release-dates');
|
||||
|
||||
const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs'];
|
||||
const INVENTORY_SUMMARY_MS = 15 * 60000; // status summary age limit; decisions still scan (libraryInventory)
|
||||
const FAST_RATE_SETTINGS = {
|
||||
minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000,
|
||||
};
|
||||
@@ -401,7 +402,7 @@ class ScraperService {
|
||||
} catch (error) { await this.event('warning', `Release dates for ${manga.title} failed: ${error.message}`, { mangaId: manga.id }); }
|
||||
}
|
||||
// The reader reconciles fully when the revision rises without a visible folder change.
|
||||
if (written) { this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
|
||||
if (written) { this.invalidateInventory(); this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); }
|
||||
await this.event('info', `Release dates: ${written} chapter${written === 1 ? '' : 's'} dated in ${checked} ongoing series`, { reason, checked, written });
|
||||
return { checked, written };
|
||||
}
|
||||
@@ -583,6 +584,15 @@ class ScraperService {
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
inventoryStale() {
|
||||
return this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= INVENTORY_SUMMARY_MS;
|
||||
}
|
||||
|
||||
// What the reader polls every few seconds: no library walk, no event log read.
|
||||
revision() {
|
||||
return { publishedRevision: this.state.publishedRevision || 0, status: this.state.status, paused: this.state.paused };
|
||||
}
|
||||
|
||||
async publicState() {
|
||||
const activeItems = this.state.queue.filter(item => Object.values(this.state.currentIds || {}).includes(item.id));
|
||||
const active = await Promise.all(activeItems.map(async current => ({
|
||||
@@ -593,8 +603,9 @@ class ScraperService {
|
||||
const job = active[0]?.job || null;
|
||||
// Status and watchdog probes must not wait for thousands of bind-mount reads.
|
||||
// Share one background scan, serve the previous summary, and surface its age/error.
|
||||
if (!this.inventoryRefresh && (this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000)
|
||||
&& Date.now() - this.inventoryAttemptedAt >= 10000) {
|
||||
// The scraper is the library's only writer and invalidates the summary on every publish, so the age
|
||||
// limit only catches outside edits. A walk reads every chapter.json: on a NAS that is ~10,000 opens.
|
||||
if (!this.inventoryRefresh && this.inventoryStale() && Date.now() - this.inventoryAttemptedAt >= 10000) {
|
||||
this.refreshInventory().catch(() => {});
|
||||
}
|
||||
const inventory = this.inventoryCache;
|
||||
@@ -609,7 +620,7 @@ class ScraperService {
|
||||
updateChecks: Object.values(this.state.updateChecks || {}),
|
||||
libraryStatus: {
|
||||
ready: !!inventory, refreshing: !!this.inventoryRefresh,
|
||||
stale: this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000,
|
||||
stale: this.inventoryStale(),
|
||||
updated: this.inventoryCachedAt ? new Date(this.inventoryCachedAt).toISOString() : null,
|
||||
error: this.inventoryError,
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user