Avoid redundant full scan after warm restart

This commit is contained in:
Codex committed 2026-10-08 21:58:28 +02:00
1 parent 8ac27e1f3d
commit c8dd46d0fb
3 files changed
+23 -8

No files matched your search

+2 -1
View File
@@ -57,7 +57,8 @@ server. The daemon publishes complete folders atomically and increments a durabl
revision; the reader polls that revision and refreshes changed manga after publication. revision; the reader polls that revision and refreshes changed manga after publication.
The reader restores a disposable full-index snapshot at startup so browsing need not The reader restores a disposable full-index snapshot at startup so browsing need not
wait for a filesystem walk. Shallow directory stamps identify changed manga; a wait for a filesystem walk. Shallow directory stamps identify changed manga; a
background and daily full reconciliation catches in-place manual edits. daily full reconciliation catches in-place manual edits. The last full-scan time
survives restarts, so restarting does not itself force another complete walk.
| Path | Lines | Responsibility | | Path | Lines | Responsibility |
|------|------:|----------------| |------|------:|----------------|
+5
View File
@@ -33,6 +33,11 @@ MangaCircus does not copy or convert anything. "Import" means **indexing the lib
| **Rescan** (Settings → Admin, or `POST /api/admin/rescan {}`) | On demand; the request returns when the scan is done | used | | **Rescan** (Settings → Admin, or `POST /api/admin/rescan {}`) | On demand; the request returns when the scan is done | used |
| **Full rescan** (`POST /api/admin/rescan {"full":true}`) | On demand | ignored: every image header is read again | | **Full rescan** (`POST /api/admin/rescan {"full":true}`) | On demand | ignored: every image header is read again |
The saved index records when its last full reconciliation finished. A warm restart
does a shallow check but does not reset the daily full-scan clock; if that scan is
overdue, the reader schedules it within a minute. Set `FULL_SCAN_INTERVAL=0` to
disable automatic full reconciliations, while retaining admin Rescan.
Scans are **single-flight**. A trigger during a scan queues one coalesced follow-up, Scans are **single-flight**. A trigger during a scan queues one coalesced follow-up,
with a full reconciliation taking precedence over a shallow check. Publication with a full reconciliation taking precedence over a shallow check. Publication
revisions trigger a shallow check; if no changed directory is visible, the reader revisions trigger a shallow check; if no changed directory is visible, the reader
+16 -7
View File
@@ -86,6 +86,7 @@ for (const [k, s] of Object.entries(db.sessions)) if (Date.now() - s.t > 365 * 8
// ---------- library index ---------- // ---------- library index ----------
let lib = { mangas: [], byId: new Map() }, scanning = null, pendingScan = null, scanState = {}, scanCache = {}, scanStamps = {}; let lib = { mangas: [], byId: new Map() }, scanning = null, pendingScan = null, scanState = {}, scanCache = {}, scanStamps = {};
let lastFullAt = 0, lastFullAttemptAt = 0;
try { scanCache = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); } catch { /* first run */ } try { scanCache = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); } catch { /* first run */ }
try { try {
const saved = JSON.parse(fs.readFileSync(INDEX_FILE, 'utf8')); const saved = JSON.parse(fs.readFileSync(INDEX_FILE, 'utf8'));
@@ -95,6 +96,7 @@ try {
const present = new Set(fs.readdirSync(LIB, { withFileTypes: true }).filter(e => e.isDirectory()).map(e => e.name)); const present = new Set(fs.readdirSync(LIB, { withFileTypes: true }).filter(e => e.isDirectory()).map(e => e.name));
if (restored.mangas.length && !restored.mangas.some(m => present.has(m.id))) throw new Error('Library mount differs from snapshot'); if (restored.mangas.length && !restored.mangas.some(m => present.has(m.id))) throw new Error('Library mount differs from snapshot');
lib = restored; scanCache = saved.cache; scanStamps = saved.stamps; lib = restored; scanCache = saved.cache; scanStamps = saved.stamps;
lastFullAt = Number(saved.fullAt) || Number(saved.at) || 0;
scanState = { at: saved.at, mode: 'snapshot', stale: true, mangas: lib.mangas.length, error: null }; scanState = { at: saved.at, mode: 'snapshot', stale: true, mangas: lib.mangas.length, error: null };
console.log(`[scan] restored ${lib.mangas.length} manga from saved index; verifying in background`); console.log(`[scan] restored ${lib.mangas.length} manga from saved index; verifying in background`);
} catch (e) { if (e.code !== 'ENOENT') console.warn(`[scan] ignoring saved index: ${e.message}`); } } catch (e) { if (e.code !== 'ENOENT') console.warn(`[scan] ignoring saved index: ${e.message}`); }
@@ -112,8 +114,10 @@ async function performScan(mode) {
if (mode !== 'changed' || changed.length) { if (mode !== 'changed' || changed.length) {
announce(index); announce(index);
lib = index; scanCache = cache; scanStamps = stamps; lib = index; scanCache = cache; scanStamps = stamps;
if (mode !== 'changed') lastFullAt = Date.now();
writeAtomic(CACHE_FILE, JSON.stringify(cache)); writeAtomic(CACHE_FILE, JSON.stringify(cache));
writeAtomic(INDEX_FILE, JSON.stringify({ version: 1, root: LIB, at: Date.now(), mangas: snapshotIndex(index), cache, stamps })); writeAtomic(INDEX_FILE, JSON.stringify({ version: 1, root: LIB, at: Date.now(), fullAt: lastFullAt,
mangas: snapshotIndex(index), cache, stamps }));
} }
let chapters = 0, pages = 0; let chapters = 0, pages = 0;
for (const m of lib.mangas) for (const ch of m.chapters) { chapters++; for (const v of ch.versions) pages += v.pages.length; } for (const m of lib.mangas) for (const ch of m.chapters) { chapters++; for (const v of ch.versions) pages += v.pages.length; }
@@ -633,13 +637,18 @@ http.createServer(async (req, res) => {
} }
}).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT} (library: ${LIB}, data: ${DATA})`)); }).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT} (library: ${LIB}, data: ${DATA})`));
if (lib.mangas.length) { if (lib.mangas.length) refreshChanged();
refreshChanged(); else rescan();
// Catch in-place manual edits that cannot change a parent directory stamp.
setTimeout(() => rescan(), 60000).unref();
} else rescan();
if (SCAN_MINUTES > 0) setInterval(() => refreshChanged(), SCAN_MINUTES * 60000).unref(); if (SCAN_MINUTES > 0) setInterval(() => refreshChanged(), SCAN_MINUTES * 60000).unref();
if (FULL_SCAN_MINUTES > 0) setInterval(() => rescan(), FULL_SCAN_MINUTES * 60000).unref(); // Persist the last reconciliation time. Restarting a healthy reader should not
// trigger another full tree walk; overdue reconciliations run within a minute.
if (FULL_SCAN_MINUTES > 0) setInterval(() => {
const now = Date.now();
if (now - Math.max(lastFullAt, lastFullAttemptAt) >= FULL_SCAN_MINUTES * 60000) {
lastFullAttemptAt = now;
rescan();
}
}, 60000).unref();
// A completed scraper publish increments a durable revision. Polling it here keeps the // A completed scraper publish increments a durable revision. Polling it here keeps the
// reader index fresh without exposing an unauthenticated rescan endpoint to the daemon. // reader index fresh without exposing an unauthenticated rescan endpoint to the daemon.