Avoid redundant full scan after warm restart
This commit is contained in:
1 parent
8ac27e1f3d
commit
c8dd46d0fb
3 files changed
+23
-8
No files matched your search
@@ -57,7 +57,8 @@ server. The daemon publishes complete folders atomically and increments a durabl
|
||||
revision; the reader polls that revision and refreshes changed manga after publication.
|
||||
The reader restores a disposable full-index snapshot at startup so browsing need not
|
||||
wait for a filesystem walk. Shallow directory stamps identify changed manga; a
|
||||
background and daily full reconciliation catches in-place manual edits.
|
||||
daily full reconciliation catches in-place manual edits. The last full-scan time
|
||||
survives restarts, so restarting does not itself force another complete walk.
|
||||
|
||||
| Path | Lines | Responsibility |
|
||||
|------|------:|----------------|
|
||||
|
||||
@@ -33,6 +33,11 @@ MangaCircus does not copy or convert anything. "Import" means **indexing the lib
|
||||
| **Rescan** (Settings → Admin, or `POST /api/admin/rescan {}`) | On demand; the request returns when the scan is done | used |
|
||||
| **Full rescan** (`POST /api/admin/rescan {"full":true}`) | On demand | ignored: every image header is read again |
|
||||
|
||||
The saved index records when its last full reconciliation finished. A warm restart
|
||||
does a shallow check but does not reset the daily full-scan clock; if that scan is
|
||||
overdue, the reader schedules it within a minute. Set `FULL_SCAN_INTERVAL=0` to
|
||||
disable automatic full reconciliations, while retaining admin Rescan.
|
||||
|
||||
Scans are **single-flight**. A trigger during a scan queues one coalesced follow-up,
|
||||
with a full reconciliation taking precedence over a shallow check. Publication
|
||||
revisions trigger a shallow check; if no changed directory is visible, the reader
|
||||
|
||||
@@ -86,6 +86,7 @@ for (const [k, s] of Object.entries(db.sessions)) if (Date.now() - s.t > 365 * 8
|
||||
|
||||
// ---------- library index ----------
|
||||
let lib = { mangas: [], byId: new Map() }, scanning = null, pendingScan = null, scanState = {}, scanCache = {}, scanStamps = {};
|
||||
let lastFullAt = 0, lastFullAttemptAt = 0;
|
||||
try { scanCache = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); } catch { /* first run */ }
|
||||
try {
|
||||
const saved = JSON.parse(fs.readFileSync(INDEX_FILE, 'utf8'));
|
||||
@@ -95,6 +96,7 @@ try {
|
||||
const present = new Set(fs.readdirSync(LIB, { withFileTypes: true }).filter(e => e.isDirectory()).map(e => e.name));
|
||||
if (restored.mangas.length && !restored.mangas.some(m => present.has(m.id))) throw new Error('Library mount differs from snapshot');
|
||||
lib = restored; scanCache = saved.cache; scanStamps = saved.stamps;
|
||||
lastFullAt = Number(saved.fullAt) || Number(saved.at) || 0;
|
||||
scanState = { at: saved.at, mode: 'snapshot', stale: true, mangas: lib.mangas.length, error: null };
|
||||
console.log(`[scan] restored ${lib.mangas.length} manga from saved index; verifying in background`);
|
||||
} catch (e) { if (e.code !== 'ENOENT') console.warn(`[scan] ignoring saved index: ${e.message}`); }
|
||||
@@ -112,8 +114,10 @@ async function performScan(mode) {
|
||||
if (mode !== 'changed' || changed.length) {
|
||||
announce(index);
|
||||
lib = index; scanCache = cache; scanStamps = stamps;
|
||||
if (mode !== 'changed') lastFullAt = Date.now();
|
||||
writeAtomic(CACHE_FILE, JSON.stringify(cache));
|
||||
writeAtomic(INDEX_FILE, JSON.stringify({ version: 1, root: LIB, at: Date.now(), mangas: snapshotIndex(index), cache, stamps }));
|
||||
writeAtomic(INDEX_FILE, JSON.stringify({ version: 1, root: LIB, at: Date.now(), fullAt: lastFullAt,
|
||||
mangas: snapshotIndex(index), cache, stamps }));
|
||||
}
|
||||
let chapters = 0, pages = 0;
|
||||
for (const m of lib.mangas) for (const ch of m.chapters) { chapters++; for (const v of ch.versions) pages += v.pages.length; }
|
||||
@@ -633,13 +637,18 @@ http.createServer(async (req, res) => {
|
||||
}
|
||||
}).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT} (library: ${LIB}, data: ${DATA})`));
|
||||
|
||||
if (lib.mangas.length) {
|
||||
refreshChanged();
|
||||
// Catch in-place manual edits that cannot change a parent directory stamp.
|
||||
setTimeout(() => rescan(), 60000).unref();
|
||||
} else rescan();
|
||||
if (lib.mangas.length) refreshChanged();
|
||||
else rescan();
|
||||
if (SCAN_MINUTES > 0) setInterval(() => refreshChanged(), SCAN_MINUTES * 60000).unref();
|
||||
if (FULL_SCAN_MINUTES > 0) setInterval(() => rescan(), FULL_SCAN_MINUTES * 60000).unref();
|
||||
// Persist the last reconciliation time. Restarting a healthy reader should not
|
||||
// trigger another full tree walk; overdue reconciliations run within a minute.
|
||||
if (FULL_SCAN_MINUTES > 0) setInterval(() => {
|
||||
const now = Date.now();
|
||||
if (now - Math.max(lastFullAt, lastFullAttemptAt) >= FULL_SCAN_MINUTES * 60000) {
|
||||
lastFullAttemptAt = now;
|
||||
rescan();
|
||||
}
|
||||
}, 60000).unref();
|
||||
|
||||
// A completed scraper publish increments a durable revision. Polling it here keeps the
|
||||
// reader index fresh without exposing an unauthenticated rescan endpoint to the daemon.
|
||||
|
||||
Reference in new issue
Block a user