diff --git a/docs/scraper-service.md b/docs/scraper-service.md index 51d8506..a83ea44 100644 --- a/docs/scraper-service.md +++ b/docs/scraper-service.md @@ -199,7 +199,8 @@ All responses are JSON. When `SCRAPER_TOKEN` is set, send | Method | Path | Body / result | |---|---|---| | `GET` | `/health` | Basic liveness | -| `GET` | `/api/state` | Queue, all active source checkpoints, settings, errors, history, adapters | +| `GET` | `/api/state` | Queue, all active source checkpoints, settings, errors, history, adapters. The library summary in it is rebuilt in the background after a publish, or when it is 15 minutes old (the walk reads every `chapter.json`) | +| `GET` | `/api/revision` | `{ publishedRevision, status, paused }` only; the reader polls this every 15 s | | `POST` | `/api/queue` | `{url,title?,metadataSearch?,scope?,existingPolicy?,libraryMangaId?,inspect?}` | | `POST` | `/api/queue/remove` | `{id}`; active downloads cannot be removed | | `POST` | `/api/queue/clear-completed` | `{source?}`; remove inactive successful download job rows only; preserve library, history, checkpoints and other statuses | diff --git a/scraper/daemon.js b/scraper/daemon.js index b561ed5..58b41b8 100644 --- a/scraper/daemon.js +++ b/scraper/daemon.js @@ -35,6 +35,7 @@ async function body(req) { if (url.pathname === '/health') return json(res, 200, { ok: true, status: service.state.status }); if (TOKEN && req.headers.authorization !== `Bearer ${TOKEN}`) return json(res, 401, { error: 'Unauthorized' }); if (req.method === 'GET' && url.pathname === '/api/state') return json(res, 200, await service.publicState()); + if (req.method === 'GET' && url.pathname === '/api/revision') return json(res, 200, service.revision()); if (req.method !== 'POST') return json(res, 404, { error: 'Not found' }); const input = await body(req); if (url.pathname === '/api/queue') return json(res, 200, { item: await service.enqueue(input), state: await service.publicState() }); diff --git a/scraper/service.js b/scraper/service.js index 6811960..fd1cab8 100644 --- a/scraper/service.js +++ b/scraper/service.js @@ -16,6 +16,7 @@ const { writeJsonAtomic } = require('./lib/json-file'); const { ReleaseDateSources, refreshMangaReleaseDates } = require('./metadata/release-dates'); const RATE_KEYS = ['minMs', 'maxMs', 'chapterMinMs', 'chapterMaxMs']; +const INVENTORY_SUMMARY_MS = 15 * 60000; // status summary age limit; decisions still scan (libraryInventory) const FAST_RATE_SETTINGS = { minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000, }; @@ -401,7 +402,7 @@ class ScraperService { } catch (error) { await this.event('warning', `Release dates for ${manga.title} failed: ${error.message}`, { mangaId: manga.id }); } } // The reader reconciles fully when the revision rises without a visible folder change. - if (written) { this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); } + if (written) { this.invalidateInventory(); this.state.publishedRevision = Number(this.state.publishedRevision || 0) + 1; await this.persist(); } await this.event('info', `Release dates: ${written} chapter${written === 1 ? '' : 's'} dated in ${checked} ongoing series`, { reason, checked, written }); return { checked, written }; } @@ -583,6 +584,15 @@ class ScraperService { } catch { return null; } } + inventoryStale() { + return this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= INVENTORY_SUMMARY_MS; + } + + // What the reader polls every few seconds: no library walk, no event log read. + revision() { + return { publishedRevision: this.state.publishedRevision || 0, status: this.state.status, paused: this.state.paused }; + } + async publicState() { const activeItems = this.state.queue.filter(item => Object.values(this.state.currentIds || {}).includes(item.id)); const active = await Promise.all(activeItems.map(async current => ({ @@ -593,8 +603,9 @@ class ScraperService { const job = active[0]?.job || null; // Status and watchdog probes must not wait for thousands of bind-mount reads. // Share one background scan, serve the previous summary, and surface its age/error. - if (!this.inventoryRefresh && (this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000) - && Date.now() - this.inventoryAttemptedAt >= 10000) { + // The scraper is the library's only writer and invalidates the summary on every publish, so the age + // limit only catches outside edits. A walk reads every chapter.json: on a NAS that is ~10,000 opens. + if (!this.inventoryRefresh && this.inventoryStale() && Date.now() - this.inventoryAttemptedAt >= 10000) { this.refreshInventory().catch(() => {}); } const inventory = this.inventoryCache; @@ -609,7 +620,7 @@ class ScraperService { updateChecks: Object.values(this.state.updateChecks || {}), libraryStatus: { ready: !!inventory, refreshing: !!this.inventoryRefresh, - stale: this.inventoryCacheGeneration !== this.inventoryGeneration || Date.now() - this.inventoryCachedAt >= 60000, + stale: this.inventoryStale(), updated: this.inventoryCachedAt ? new Date(this.inventoryCachedAt).toISOString() : null, error: this.inventoryError, }, diff --git a/scraper/test/inventory-cache.test.js b/scraper/test/inventory-cache.test.js index e76d3ae..1470e04 100644 --- a/scraper/test/inventory-cache.test.js +++ b/scraper/test/inventory-cache.test.js @@ -80,6 +80,23 @@ test('background inventory failures are visible without breaking controls or lau assert.equal(scans, 2); }); +test('status polling walks the library again only after a change or after 15 minutes', async () => { + let scans = 0; + const service = fixture(async () => { scans++; return inventory('Demo'); }); + await service.libraryInventory(true); + service.inventoryCachedAt = Date.now() - 10 * 60000; service.inventoryAttemptedAt = 0; + for (let n = 0; n < 5; n++) await service.publicState(); + assert.equal(scans, 1); + assert.deepEqual(service.revision(), { publishedRevision: 0, status: 'paused', paused: true }); + assert.equal(scans, 1); + service.invalidateInventory(); + await service.publicState(); await service.inventoryRefresh; + assert.equal(scans, 2); + service.inventoryCachedAt = Date.now() - 16 * 60000; service.inventoryAttemptedAt = 0; + await service.publicState(); await service.inventoryRefresh; + assert.equal(scans, 3); +}); + test('dashboard polling reuses a recent snapshot while decisions still force a fresh scan', async () => { let scans = 0; const service = fixture(async () => { scans++; return inventory('Demo'); }); diff --git a/server.js b/server.js index 55ad355..9addcf9 100644 --- a/server.js +++ b/server.js @@ -1218,10 +1218,13 @@ if (FULL_SCAN_MINUTES > 0) setInterval(() => { // A completed scraper publish increments a durable revision. Polling it here keeps the // reader index fresh without exposing an unauthenticated rescan endpoint to the daemon. -let scraperRevision = null; +let scraperRevision = null, revisionRoute = true; async function pollScraper() { try { - const state = await scraperRequest('/api/state'); + // /api/revision is a few bytes; the full state is ~100 KB and makes the scraper summarise the library. + let state; + if (revisionRoute) state = await scraperRequest('/api/revision').catch(error => { if (error.code !== 404) throw error; revisionRoute = false; }); + if (!revisionRoute) state = await scraperRequest('/api/state'); // a scraper from before /api/revision const revision = Number(state.publishedRevision || 0); if (scraperRevision === null || revision > scraperRevision) { const checked = await refreshChanged();