Cache library index and refresh changed manga
This commit is contained in:
Codex committed
1 parent
eef4ffa07c
commit
8ac27e1f3d
6 files changed
+269
-30
No files matched your search
@@ -5,13 +5,14 @@
|
||||
* Environment:
|
||||
* PORT=8080 LIBRARY=./library DATA=./data
|
||||
* SIGNUP=closed|open (the very first account can always be created and becomes admin)
|
||||
* SCAN_INTERVAL=10 minutes between automatic library rescans, 0 = off
|
||||
* SCAN_INTERVAL=10 minutes between shallow change checks, 0 = off
|
||||
* FULL_SCAN_INTERVAL=1440 minutes between full reconciliations, 0 = off
|
||||
* COOKIE_SECURE=1 mark the session cookie Secure (when served over HTTPS)
|
||||
* SCRAPER_URL=http://127.0.0.1:8091 scraper daemon API
|
||||
* SCRAPER_TOKEN=... shared daemon token (optional on loopback)
|
||||
*/
|
||||
const http = require('http'), fs = require('fs'), path = require('path'), crypto = require('crypto');
|
||||
const { scanLibrary } = require('./library');
|
||||
const { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex } = require('./library');
|
||||
const { Catalog, CatalogError, collectionViews, TYPES, ROLES } = require('./catalog');
|
||||
const { secret } = require('./runtime/config');
|
||||
const {
|
||||
@@ -23,6 +24,7 @@ const LIB = path.resolve(process.env.LIBRARY || 'library');
|
||||
const DATA = path.resolve(process.env.DATA || 'data');
|
||||
const OPEN_SIGNUP = process.env.SIGNUP === 'open';
|
||||
const SCAN_MINUTES = +(process.env.SCAN_INTERVAL || 10);
|
||||
const FULL_SCAN_MINUTES = +(process.env.FULL_SCAN_INTERVAL || 1440);
|
||||
const SECURE_COOKIE = process.env.COOKIE_SECURE === '1';
|
||||
const SCRAPER_URL = String(process.env.SCRAPER_URL || 'http://127.0.0.1:8091').replace(/\/$/, '');
|
||||
const SCRAPER_TOKEN = secret('SCRAPER_TOKEN');
|
||||
@@ -38,7 +40,7 @@ const MIME = {
|
||||
// ---------- storage: one JSON file, written atomically ----------
|
||||
fs.mkdirSync(DATA, { recursive: true });
|
||||
const catalog = new Catalog(DATA);
|
||||
const DB_FILE = path.join(DATA, 'db.json'), CACHE_FILE = path.join(DATA, 'scan-cache.json');
|
||||
const DB_FILE = path.join(DATA, 'db.json'), CACHE_FILE = path.join(DATA, 'scan-cache.json'), INDEX_FILE = path.join(DATA, 'library-index.json');
|
||||
const db = { users: {}, sessions: {}, progress: {}, reading: {}, follows: {}, notes: {}, votes: {}, known: {} };
|
||||
// A corrupt db.json stops the server instead of silently starting empty and overwriting it.
|
||||
if (fs.existsSync(DB_FILE)) Object.assign(db, JSON.parse(fs.readFileSync(DB_FILE, 'utf8')));
|
||||
@@ -83,28 +85,63 @@ function userOf(req) { const t = tokenOf(req), s = t && db.sessions[sha(t)]; ret
|
||||
for (const [k, s] of Object.entries(db.sessions)) if (Date.now() - s.t > 365 * 864e5 || !db.users[s.u]) delete db.sessions[k];
|
||||
|
||||
// ---------- library index ----------
|
||||
let lib = { mangas: [], byId: new Map() }, scanning = null, scanState = {}, scanCache = {};
|
||||
let lib = { mangas: [], byId: new Map() }, scanning = null, pendingScan = null, scanState = {}, scanCache = {}, scanStamps = {};
|
||||
try { scanCache = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); } catch { /* first run */ }
|
||||
try {
|
||||
const saved = JSON.parse(fs.readFileSync(INDEX_FILE, 'utf8'));
|
||||
if (saved.version !== 1 || saved.root !== LIB || !saved.cache || !saved.stamps) throw new Error('Invalid library index snapshot');
|
||||
const restored = restoreIndex(saved.mangas);
|
||||
// A disconnected/replaced library mount must not appear populated from cache.
|
||||
const present = new Set(fs.readdirSync(LIB, { withFileTypes: true }).filter(e => e.isDirectory()).map(e => e.name));
|
||||
if (restored.mangas.length && !restored.mangas.some(m => present.has(m.id))) throw new Error('Library mount differs from snapshot');
|
||||
lib = restored; scanCache = saved.cache; scanStamps = saved.stamps;
|
||||
scanState = { at: saved.at, mode: 'snapshot', stale: true, mangas: lib.mangas.length, error: null };
|
||||
console.log(`[scan] restored ${lib.mangas.length} manga from saved index; verifying in background`);
|
||||
} catch (e) { if (e.code !== 'ENOENT') console.warn(`[scan] ignoring saved index: ${e.message}`); }
|
||||
|
||||
function rescan(full) {
|
||||
scanning = scanning || (async () => {
|
||||
async function performScan(mode) {
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const { index, cache } = await scanLibrary(LIB, full ? {} : scanCache);
|
||||
announce(index);
|
||||
lib = index; scanCache = cache;
|
||||
writeAtomic(CACHE_FILE, JSON.stringify(cache));
|
||||
// Capture before the scan: a publication during the walk remains visible
|
||||
// as a changed stamp on the next check, rather than being silently missed.
|
||||
const stamps = await libraryStamps(LIB);
|
||||
const result = mode === 'changed' && Object.keys(scanStamps).length
|
||||
? await scanChangedLibrary(LIB, lib, scanCache, scanStamps, stamps)
|
||||
: { ...(await scanLibrary(LIB, mode === 'deep' ? {} : scanCache)), stamps, changed: Object.keys(stamps) };
|
||||
const { index, cache, changed } = result;
|
||||
if (mode !== 'changed' || changed.length) {
|
||||
announce(index);
|
||||
lib = index; scanCache = cache; scanStamps = stamps;
|
||||
writeAtomic(CACHE_FILE, JSON.stringify(cache));
|
||||
writeAtomic(INDEX_FILE, JSON.stringify({ version: 1, root: LIB, at: Date.now(), mangas: snapshotIndex(index), cache, stamps }));
|
||||
}
|
||||
let chapters = 0, pages = 0;
|
||||
for (const m of index.mangas) for (const ch of m.chapters) { chapters++; for (const v of ch.versions) pages += v.pages.length; }
|
||||
scanState = { at: Date.now(), ms: Date.now() - t0, mangas: index.mangas.length, chapters, pages, error: null };
|
||||
console.log(`[scan] ${index.mangas.length} manga, ${chapters} chapters, ${pages} pages in ${scanState.ms} ms`);
|
||||
for (const m of lib.mangas) for (const ch of m.chapters) { chapters++; for (const v of ch.versions) pages += v.pages.length; }
|
||||
scanState = { at: Date.now(), ms: Date.now() - t0, mode, changedMangas: changed.length,
|
||||
mangas: lib.mangas.length, chapters, pages, stale: false, error: null };
|
||||
console.log(`[scan] ${mode}: ${changed.length} changed manga, ${lib.mangas.length} manga, ${chapters} chapters, ${pages} pages in ${scanState.ms} ms`);
|
||||
return scanState;
|
||||
} catch (e) {
|
||||
scanState = { ...scanState, at: Date.now(), error: e.message };
|
||||
console.error('[scan] failed:', e.message);
|
||||
} finally { scanning = null; }
|
||||
})();
|
||||
return scanState;
|
||||
}
|
||||
}
|
||||
function queueScan(mode) {
|
||||
if (scanning) {
|
||||
const priority = { changed: 1, full: 2, deep: 3 };
|
||||
if (!pendingScan || priority[mode] > priority[pendingScan]) pendingScan = mode;
|
||||
return scanning;
|
||||
}
|
||||
scanning = (async () => {
|
||||
let next = mode, result;
|
||||
do { result = await performScan(next); next = pendingScan; pendingScan = null; } while (next);
|
||||
return result;
|
||||
})().finally(() => { scanning = null; });
|
||||
return scanning;
|
||||
}
|
||||
function rescan(full) { return queueScan(full ? 'deep' : 'full'); }
|
||||
function refreshChanged() { return queueScan('changed'); }
|
||||
|
||||
// New chapter ids (compared with every id ever seen for that manga) notify the followers.
|
||||
function announce(index) {
|
||||
@@ -596,8 +633,13 @@ http.createServer(async (req, res) => {
|
||||
}
|
||||
}).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT} (library: ${LIB}, data: ${DATA})`));
|
||||
|
||||
rescan();
|
||||
if (SCAN_MINUTES > 0) setInterval(() => rescan(), SCAN_MINUTES * 60000).unref();
|
||||
if (lib.mangas.length) {
|
||||
refreshChanged();
|
||||
// Catch in-place manual edits that cannot change a parent directory stamp.
|
||||
setTimeout(() => rescan(), 60000).unref();
|
||||
} else rescan();
|
||||
if (SCAN_MINUTES > 0) setInterval(() => refreshChanged(), SCAN_MINUTES * 60000).unref();
|
||||
if (FULL_SCAN_MINUTES > 0) setInterval(() => rescan(), FULL_SCAN_MINUTES * 60000).unref();
|
||||
|
||||
// A completed scraper publish increments a durable revision. Polling it here keeps the
|
||||
// reader index fresh without exposing an unauthenticated rescan endpoint to the daemon.
|
||||
@@ -606,7 +648,16 @@ async function pollScraper() {
|
||||
try {
|
||||
const state = await scraperRequest('/api/state');
|
||||
const revision = Number(state.publishedRevision || 0);
|
||||
if (scraperRevision !== null && revision > scraperRevision) await rescan();
|
||||
if (scraperRevision === null || revision > scraperRevision) {
|
||||
const checked = await refreshChanged();
|
||||
if (checked.error) throw new Error(checked.error);
|
||||
// A publication without a visible directory change should not disappear
|
||||
// because of coarse filesystem timestamps or a missed scan race.
|
||||
if (scraperRevision !== null && revision > scraperRevision && !checked.changedMangas) {
|
||||
const reconciled = await rescan();
|
||||
if (reconciled.error) throw new Error(reconciled.error);
|
||||
}
|
||||
}
|
||||
scraperRevision = revision;
|
||||
} catch { /* the reader also works when the optional scraper daemon is offline */ }
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user