Cache library index and refresh changed manga
This commit is contained in:
1 parent
eef4ffa07c
commit
8ac27e1f3d
6 files changed
+267
-28
No files matched your search
@@ -60,9 +60,10 @@ see administration; personal settings do not poll scraper services. See
|
|||||||
|-----------------|-------------|---------|
|
|-----------------|-------------|---------|
|
||||||
| `PORT` | `8080` | HTTP port |
|
| `PORT` | `8080` | HTTP port |
|
||||||
| `LIBRARY` | `./library` | Library folder (read-only access is enough) |
|
| `LIBRARY` | `./library` | Library folder (read-only access is enough) |
|
||||||
| `DATA` | `./data` | Where `db.json` (users, progress, votes, notifications) and `scan-cache.json` live |
|
| `DATA` | `./data` | Where `db.json` (users, progress, votes, notifications) and disposable library-index caches live |
|
||||||
| `SIGNUP` | `closed` | `open` lets anyone create an account. Otherwise the admin adds users under Settings |
|
| `SIGNUP` | `closed` | `open` lets anyone create an account. Otherwise the admin adds users under Settings |
|
||||||
| `SCAN_INTERVAL` | `10` | Minutes between automatic library rescans (`0` = only at start and on "Rescan") |
|
| `SCAN_INTERVAL` | `10` | Minutes between shallow library change checks (`0` disables this timer) |
|
||||||
|
| `FULL_SCAN_INTERVAL` | `1440` | Minutes between full library reconciliations (`0` disables this timer) |
|
||||||
| `COOKIE_SECURE` | – | `1` marks the login cookie `Secure` (use it when you serve over HTTPS) |
|
| `COOKIE_SECURE` | – | `1` marks the login cookie `Secure` (use it when you serve over HTTPS) |
|
||||||
| `SCRAPER_URL` | `http://127.0.0.1:8091` | Private scraper daemon URL |
|
| `SCRAPER_URL` | `http://127.0.0.1:8091` | Private scraper daemon URL |
|
||||||
| `SCRAPER_TOKEN` | empty | Shared daemon token; required when it listens beyond loopback |
|
| `SCRAPER_TOKEN` | empty | Shared daemon token; required when it listens beyond loopback |
|
||||||
|
|||||||
+10
-3
@@ -54,7 +54,10 @@ The catalog and relation store is reader-owned `data/catalog.json`. Sentinel wri
|
|||||||
The admin browser never talks to the scraper daemon directly. Admin-only reader
|
The admin browser never talks to the scraper daemon directly. Admin-only reader
|
||||||
routes proxy commands to the daemon's private API, so its shared token stays on the
|
routes proxy commands to the daemon's private API, so its shared token stays on the
|
||||||
server. The daemon publishes complete folders atomically and increments a durable
|
server. The daemon publishes complete folders atomically and increments a durable
|
||||||
revision; the reader polls that revision and rescans the library after publication.
|
revision; the reader polls that revision and refreshes changed manga after publication.
|
||||||
|
The reader restores a disposable full-index snapshot at startup so browsing need not
|
||||||
|
wait for a filesystem walk. Shallow directory stamps identify changed manga; a
|
||||||
|
background and daily full reconciliation catches in-place manual edits.
|
||||||
|
|
||||||
| Path | Lines | Responsibility |
|
| Path | Lines | Responsibility |
|
||||||
|------|------:|----------------|
|
|------|------:|----------------|
|
||||||
@@ -78,7 +81,8 @@ revision; the reader polls that revision and rescans the library after publicati
|
|||||||
- `lib` is the current library index, replaced as a whole after each scan.
|
- `lib` is the current library index, replaced as a whole after each scan.
|
||||||
- `db` is all user data, loaded from `db.json` at start.
|
- `db` is all user data, loaded from `db.json` at start.
|
||||||
- **Timers:**
|
- **Timers:**
|
||||||
- a library scan at start and every `SCAN_INTERVAL` minutes,
|
- a saved-index restore and shallow check at start (full scan when no snapshot),
|
||||||
|
- a shallow check every `SCAN_INTERVAL` minutes, full reconciliation every `FULL_SCAN_INTERVAL` minutes,
|
||||||
- a debounced `db.json` write, 500 ms after the last change.
|
- a debounced `db.json` write, 500 ms after the last change.
|
||||||
- **Shutdown:** `SIGINT` / `SIGTERM` flush `db.json`; the daemon cooperatively stops at a page boundary and keeps its archive checkpoint.
|
- **Shutdown:** `SIGINT` / `SIGTERM` flush `db.json`; the daemon cooperatively stops at a page boundary and keeps its archive checkpoint.
|
||||||
- **Durability:** every file write goes to `*.tmp` first and is then renamed over the target, so a file is never half-written. A hard kill loses at most the changes of the last ~0.5 s. A corrupt `db.json` makes the server **refuse to start** rather than start empty and overwrite it.
|
- **Durability:** every file write goes to `*.tmp` first and is then renamed over the target, so a file is never half-written. A hard kill loses at most the changes of the last ~0.5 s. A corrupt `db.json` makes the server **refuse to start** rather than start empty and overwrite it.
|
||||||
@@ -146,6 +150,9 @@ This keeps ids human-readable and derivable without a database. The price: renam
|
|||||||
```
|
```
|
||||||
|
|
||||||
- `data/scan-cache.json` holds `{ "<Manga>/<Book>/<Chapter>": { m: <folder mtimeMs>, p: [[file, w, h], …] } }`. It is disposable.
|
- `data/scan-cache.json` holds `{ "<Manga>/<Book>/<Chapter>": { m: <folder mtimeMs>, p: [[file, w, h], …] } }`. It is disposable.
|
||||||
|
- `data/library-index.json` holds a disposable, versioned index snapshot and shallow
|
||||||
|
directory stamps. The library folders remain authoritative; a missing or invalid
|
||||||
|
snapshot triggers a full scan. Rollback-compatible `scan-cache.json` is retained.
|
||||||
- **Backups:** copy `data/db.json`. The atomic writes make a copy at any time consistent.
|
- **Backups:** copy `data/db.json`. The atomic writes make a copy at any time consistent.
|
||||||
|
|
||||||
## 6. Server
|
## 6. Server
|
||||||
@@ -331,7 +338,7 @@ docker compose up -d --build
|
|||||||
- **Volumes:**
|
- **Volumes:**
|
||||||
- `/library`: read-only in the reader and writable in the scraper. Files must be readable—and scraper targets writable—by uid 1000.
|
- `/library`: read-only in the reader and writable in the scraper. Files must be readable—and scraper targets writable—by uid 1000.
|
||||||
- `/data`: reader state must be writable; the scraper also needs its writable `/data/scraper` state.
|
- `/data`: reader state must be writable; the scraper also needs its writable `/data/scraper` state.
|
||||||
- **Configuration:** process wiring uses environment variables (`PORT`, `LIBRARY`, `DATA`, `SIGNUP`, `SCAN_INTERVAL`, `COOKIE_SECURE`, `SCRAPER_URL`, and `SCRAPER_TOKEN`). Queue pacing and automatic-update schedule are persisted in `data/scraper/queue.json` and edited through the authenticated admin UI.
|
- **Configuration:** process wiring uses environment variables (`PORT`, `LIBRARY`, `DATA`, `SIGNUP`, `SCAN_INTERVAL`, `FULL_SCAN_INTERVAL`, `COOKIE_SECURE`, `SCRAPER_URL`, and `SCRAPER_TOKEN`). Queue pacing and automatic-update schedule are persisted in `data/scraper/queue.json` and edited through the authenticated admin UI.
|
||||||
- **Without Docker:** `npm run start:all` with Node ≥ 18, or supervise `npm start` and `npm run start:scraper` independently. On the current Windows machine that supervision is already owned by `\MangaCircus Supervisor`.
|
- **Without Docker:** `npm run start:all` with Node ≥ 18, or supervise `npm start` and `npm run start:scraper` independently. On the current Windows machine that supervision is already owned by `\MangaCircus Supervisor`.
|
||||||
- **Behind a reverse proxy:** terminate TLS, set `COOKIE_SECURE=1`. A sub-path works.
|
- **Behind a reverse proxy:** terminate TLS, set `COOKIE_SECURE=1`. A sub-path works.
|
||||||
- **Upgrades:** replace the code and restart. Reader state has no explicit migration;
|
- **Upgrades:** replace the code and restart. Reader state has no explicit migration;
|
||||||
|
|||||||
+16
-6
@@ -4,11 +4,12 @@ MangaCircus does not copy or convert anything. "Import" means **indexing the lib
|
|||||||
|
|
||||||
```
|
```
|
||||||
┌───────────── triggers ─────────────┐
|
┌───────────── triggers ─────────────┐
|
||||||
│ start-up · every SCAN_INTERVAL min │
|
│ start-up · shallow interval · daily full │
|
||||||
│ admin "Rescan" · admin "Full rescan"│
|
│ admin "Rescan" · admin "Full rescan"│
|
||||||
└────────────────┬───────────────────┘
|
└────────────────┬───────────────────┘
|
||||||
▼
|
▼
|
||||||
library/ ──read──▶ scanLibrary(root, oldCache) library.js
|
data/library-index.json ──▶ immediate in-memory index at start (if valid)
|
||||||
|
library/ ──read──▶ shallow changed-manga check or full scan library.js
|
||||||
├─ scanManga (per manga folder)
|
├─ scanManga (per manga folder)
|
||||||
│ ├─ book folders → parseKey, book.json, cover
|
│ ├─ book folders → parseKey, book.json, cover
|
||||||
│ └─ chapter folders → scanVersion
|
│ └─ chapter folders → scanVersion
|
||||||
@@ -20,19 +21,23 @@ MangaCircus does not copy or convert anything. "Import" means **indexing the lib
|
|||||||
▼
|
▼
|
||||||
lib = index (atomic swap, requests see old or new, never half)
|
lib = index (atomic swap, requests see old or new, never half)
|
||||||
▼
|
▼
|
||||||
data/scan-cache.json (written atomically)
|
data/scan-cache.json + data/library-index.json (atomic)
|
||||||
```
|
```
|
||||||
|
|
||||||
## 1. Triggers
|
## 1. Triggers
|
||||||
|
|
||||||
| Trigger | When | Cache |
|
| Trigger | When | Cache |
|
||||||
|---------|------|-------|
|
|---------|------|-------|
|
||||||
| Start-up | Right after the HTTP server starts listening. Requests are served meanwhile; the library page shows "Scanning…" until the first scan is done | used |
|
| Start-up | Restore the saved index immediately if its library mount is present, then check changed manga. Without a valid snapshot, scan the whole library; the page shows "Scanning…" until done | used |
|
||||||
| Interval | Every `SCAN_INTERVAL` minutes (default 10, `0` = off) | used |
|
| Interval | Shallow change check every `SCAN_INTERVAL` minutes (default 10, `0` = off); full reconciliation every `FULL_SCAN_INTERVAL` minutes (default 1440, `0` = off) | used |
|
||||||
| **Rescan** (Settings → Admin, or `POST /api/admin/rescan {}`) | On demand; the request returns when the scan is done | used |
|
| **Rescan** (Settings → Admin, or `POST /api/admin/rescan {}`) | On demand; the request returns when the scan is done | used |
|
||||||
| **Full rescan** (`POST /api/admin/rescan {"full":true}`) | On demand | ignored: every image header is read again |
|
| **Full rescan** (`POST /api/admin/rescan {"full":true}`) | On demand | ignored: every image header is read again |
|
||||||
|
|
||||||
Scans are **single-flight**. A trigger that fires while a scan is running joins that scan instead of starting a second one.
|
Scans are **single-flight**. A trigger during a scan queues one coalesced follow-up,
|
||||||
|
with a full reconciliation taking precedence over a shallow check. Publication
|
||||||
|
revisions trigger a shallow check; if no changed directory is visible, the reader
|
||||||
|
falls back to a full scan. In-place edits to `chapter.json` are found by the full
|
||||||
|
reconciliation or an admin Rescan, not by shallow parent-directory stamps.
|
||||||
|
|
||||||
## 2. Walking the tree
|
## 2. Walking the tree
|
||||||
|
|
||||||
@@ -155,6 +160,11 @@ The reader lays pages out before they load, so it needs every image's size. Inst
|
|||||||
## 4. The scan cache
|
## 4. The scan cache
|
||||||
|
|
||||||
- Stored in `DATA/scan-cache.json`. It is rewritten atomically after every successful scan and holds only the folders seen in that scan, so deleted chapters drop out.
|
- Stored in `DATA/scan-cache.json`. It is rewritten atomically after every successful scan and holds only the folders seen in that scan, so deleted chapters drop out.
|
||||||
|
- `DATA/library-index.json` also stores the complete rebuildable index and shallow
|
||||||
|
manga/book directory stamps. It lets a restarted reader serve the dashboard and
|
||||||
|
manga pages before the background filesystem check completes. Both files can be
|
||||||
|
deleted safely; without the index the next startup walks the library, and without
|
||||||
|
the size cache it also rereads image headers.
|
||||||
- Invalidation is **per chapter folder, by its mtime**. Adding, removing or renaming a file changes the folder mtime, so that folder is re-read on the next scan.
|
- Invalidation is **per chapter folder, by its mtime**. Adding, removing or renaming a file changes the folder mtime, so that folder is re-read on the next scan.
|
||||||
- **Blind spot:** overwriting an image *in place* (same name) usually doesn't change the folder's mtime, so the old size stays cached. Run a **Full rescan** after such edits, or write new files and rename them (§7).
|
- **Blind spot:** overwriting an image *in place* (same name) usually doesn't change the folder's mtime, so the old size stays cached. Run a **Full rescan** after such edits, or write new files and rename them (§7).
|
||||||
- Deleting the cache file is harmless. The next scan is simply slower.
|
- Deleting the cache file is harmless. The next scan is simply slower.
|
||||||
|
|||||||
+69
-1
@@ -164,4 +164,72 @@ async function scanLibrary(root, oldCache = {}) {
|
|||||||
return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache };
|
return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache };
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { scanLibrary, parseKey, sizeFromBuffer, imageSize };
|
// Book-directory renames are the publisher's atomic commit point. A shallow
|
||||||
|
// stamp pass costs one stat per book, rather than one stat/JSON read per chapter.
|
||||||
|
// In-place chapter.json edits are picked up by the scheduled deep reconciliation.
|
||||||
|
async function fileStamp(file) {
|
||||||
|
try { const s = await fsp.stat(file); return `${s.mtimeMs}:${s.size}`; }
|
||||||
|
catch (e) { if (e.code === 'ENOENT') return ''; throw e; }
|
||||||
|
}
|
||||||
|
async function libraryStamps(root) {
|
||||||
|
const stamps = {};
|
||||||
|
for (const name of (await list(root)).dirs) {
|
||||||
|
const mangaDir = path.join(root, name), books = {};
|
||||||
|
const listed = await list(mangaDir);
|
||||||
|
for (const bookName of listed.dirs.filter(parseKey)) {
|
||||||
|
const bookDir = path.join(mangaDir, bookName);
|
||||||
|
books[bookName] = [await fileStamp(bookDir), await fileStamp(path.join(bookDir, 'book.json'))];
|
||||||
|
}
|
||||||
|
stamps[name] = [await fileStamp(mangaDir), await fileStamp(path.join(mangaDir, 'manga.json')), books];
|
||||||
|
}
|
||||||
|
return stamps;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps) {
|
||||||
|
stamps = stamps || await libraryStamps(root);
|
||||||
|
const names = new Set([...Object.keys(oldStamps || {}), ...Object.keys(stamps)]);
|
||||||
|
const changed = [...names].filter(name => JSON.stringify(oldStamps?.[name]) !== JSON.stringify(stamps[name]));
|
||||||
|
if (!changed.length) return { index: prior, cache: oldCache, stamps, changed };
|
||||||
|
const byId = new Map(prior.mangas.map(m => [m.id, m]));
|
||||||
|
const cache = { ...oldCache };
|
||||||
|
for (const name of changed) {
|
||||||
|
byId.delete(name);
|
||||||
|
for (const key of Object.keys(cache)) if (key.startsWith(name + '/')) delete cache[key];
|
||||||
|
if (!Object.hasOwn(stamps, name)) continue;
|
||||||
|
const m = await scanManga(root, name, oldCache, cache);
|
||||||
|
if (m.chapters.length) byId.set(name, m);
|
||||||
|
}
|
||||||
|
const mangas = [...byId.values()].sort((a, b) => cmp(a.title, b.title));
|
||||||
|
return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache, stamps, changed };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store each chapter only once. Maps and book chapter references are rebuilt
|
||||||
|
// from the snapshot, which is a disposable cache rather than library authority.
|
||||||
|
function snapshotIndex(index) {
|
||||||
|
return index.mangas.map(m => ({ ...m, chapterById: undefined,
|
||||||
|
books: m.books.map(b => ({ ...b, chapters: b.chapters.map(ch => ch.id) })) }));
|
||||||
|
}
|
||||||
|
function restoreIndex(mangas) {
|
||||||
|
if (!Array.isArray(mangas)) throw new Error('Invalid library index snapshot');
|
||||||
|
const byId = new Map();
|
||||||
|
for (const m of mangas) {
|
||||||
|
if (!m || typeof m.id !== 'string' || !Array.isArray(m.books) || !Array.isArray(m.chapters) || byId.has(m.id))
|
||||||
|
throw new Error('Invalid manga in library index snapshot');
|
||||||
|
m.chapterById = new Map();
|
||||||
|
for (const ch of m.chapters) {
|
||||||
|
if (!ch || typeof ch.id !== 'string' || !Array.isArray(ch.versions) || ch.versions.some(v => !Array.isArray(v.pages)))
|
||||||
|
throw new Error('Invalid chapter in library index snapshot');
|
||||||
|
m.chapterById.set(ch.id, ch);
|
||||||
|
}
|
||||||
|
for (const book of m.books) {
|
||||||
|
if (!Array.isArray(book.chapters) || book.chapters.some(id => !m.chapterById.has(id)))
|
||||||
|
throw new Error('Invalid book in library index snapshot');
|
||||||
|
book.chapters = book.chapters.map(id => m.chapterById.get(id));
|
||||||
|
}
|
||||||
|
byId.set(m.id, m);
|
||||||
|
}
|
||||||
|
return { mangas, byId };
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex,
|
||||||
|
parseKey, sizeFromBuffer, imageSize };
|
||||||
@@ -5,13 +5,14 @@
|
|||||||
* Environment:
|
* Environment:
|
||||||
* PORT=8080 LIBRARY=./library DATA=./data
|
* PORT=8080 LIBRARY=./library DATA=./data
|
||||||
* SIGNUP=closed|open (the very first account can always be created and becomes admin)
|
* SIGNUP=closed|open (the very first account can always be created and becomes admin)
|
||||||
* SCAN_INTERVAL=10 minutes between automatic library rescans, 0 = off
|
* SCAN_INTERVAL=10 minutes between shallow change checks, 0 = off
|
||||||
|
* FULL_SCAN_INTERVAL=1440 minutes between full reconciliations, 0 = off
|
||||||
* COOKIE_SECURE=1 mark the session cookie Secure (when served over HTTPS)
|
* COOKIE_SECURE=1 mark the session cookie Secure (when served over HTTPS)
|
||||||
* SCRAPER_URL=http://127.0.0.1:8091 scraper daemon API
|
* SCRAPER_URL=http://127.0.0.1:8091 scraper daemon API
|
||||||
* SCRAPER_TOKEN=... shared daemon token (optional on loopback)
|
* SCRAPER_TOKEN=... shared daemon token (optional on loopback)
|
||||||
*/
|
*/
|
||||||
const http = require('http'), fs = require('fs'), path = require('path'), crypto = require('crypto');
|
const http = require('http'), fs = require('fs'), path = require('path'), crypto = require('crypto');
|
||||||
const { scanLibrary } = require('./library');
|
const { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex } = require('./library');
|
||||||
const { Catalog, CatalogError, collectionViews, TYPES, ROLES } = require('./catalog');
|
const { Catalog, CatalogError, collectionViews, TYPES, ROLES } = require('./catalog');
|
||||||
const { secret } = require('./runtime/config');
|
const { secret } = require('./runtime/config');
|
||||||
const {
|
const {
|
||||||
@@ -23,6 +24,7 @@ const LIB = path.resolve(process.env.LIBRARY || 'library');
|
|||||||
const DATA = path.resolve(process.env.DATA || 'data');
|
const DATA = path.resolve(process.env.DATA || 'data');
|
||||||
const OPEN_SIGNUP = process.env.SIGNUP === 'open';
|
const OPEN_SIGNUP = process.env.SIGNUP === 'open';
|
||||||
const SCAN_MINUTES = +(process.env.SCAN_INTERVAL || 10);
|
const SCAN_MINUTES = +(process.env.SCAN_INTERVAL || 10);
|
||||||
|
const FULL_SCAN_MINUTES = +(process.env.FULL_SCAN_INTERVAL || 1440);
|
||||||
const SECURE_COOKIE = process.env.COOKIE_SECURE === '1';
|
const SECURE_COOKIE = process.env.COOKIE_SECURE === '1';
|
||||||
const SCRAPER_URL = String(process.env.SCRAPER_URL || 'http://127.0.0.1:8091').replace(/\/$/, '');
|
const SCRAPER_URL = String(process.env.SCRAPER_URL || 'http://127.0.0.1:8091').replace(/\/$/, '');
|
||||||
const SCRAPER_TOKEN = secret('SCRAPER_TOKEN');
|
const SCRAPER_TOKEN = secret('SCRAPER_TOKEN');
|
||||||
@@ -38,7 +40,7 @@ const MIME = {
|
|||||||
// ---------- storage: one JSON file, written atomically ----------
|
// ---------- storage: one JSON file, written atomically ----------
|
||||||
fs.mkdirSync(DATA, { recursive: true });
|
fs.mkdirSync(DATA, { recursive: true });
|
||||||
const catalog = new Catalog(DATA);
|
const catalog = new Catalog(DATA);
|
||||||
const DB_FILE = path.join(DATA, 'db.json'), CACHE_FILE = path.join(DATA, 'scan-cache.json');
|
const DB_FILE = path.join(DATA, 'db.json'), CACHE_FILE = path.join(DATA, 'scan-cache.json'), INDEX_FILE = path.join(DATA, 'library-index.json');
|
||||||
const db = { users: {}, sessions: {}, progress: {}, reading: {}, follows: {}, notes: {}, votes: {}, known: {} };
|
const db = { users: {}, sessions: {}, progress: {}, reading: {}, follows: {}, notes: {}, votes: {}, known: {} };
|
||||||
// A corrupt db.json stops the server instead of silently starting empty and overwriting it.
|
// A corrupt db.json stops the server instead of silently starting empty and overwriting it.
|
||||||
if (fs.existsSync(DB_FILE)) Object.assign(db, JSON.parse(fs.readFileSync(DB_FILE, 'utf8')));
|
if (fs.existsSync(DB_FILE)) Object.assign(db, JSON.parse(fs.readFileSync(DB_FILE, 'utf8')));
|
||||||
@@ -83,28 +85,63 @@ function userOf(req) { const t = tokenOf(req), s = t && db.sessions[sha(t)]; ret
|
|||||||
for (const [k, s] of Object.entries(db.sessions)) if (Date.now() - s.t > 365 * 864e5 || !db.users[s.u]) delete db.sessions[k];
|
for (const [k, s] of Object.entries(db.sessions)) if (Date.now() - s.t > 365 * 864e5 || !db.users[s.u]) delete db.sessions[k];
|
||||||
|
|
||||||
// ---------- library index ----------
|
// ---------- library index ----------
|
||||||
let lib = { mangas: [], byId: new Map() }, scanning = null, scanState = {}, scanCache = {};
|
let lib = { mangas: [], byId: new Map() }, scanning = null, pendingScan = null, scanState = {}, scanCache = {}, scanStamps = {};
|
||||||
try { scanCache = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); } catch { /* first run */ }
|
try { scanCache = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); } catch { /* first run */ }
|
||||||
|
try {
|
||||||
|
const saved = JSON.parse(fs.readFileSync(INDEX_FILE, 'utf8'));
|
||||||
|
if (saved.version !== 1 || saved.root !== LIB || !saved.cache || !saved.stamps) throw new Error('Invalid library index snapshot');
|
||||||
|
const restored = restoreIndex(saved.mangas);
|
||||||
|
// A disconnected/replaced library mount must not appear populated from cache.
|
||||||
|
const present = new Set(fs.readdirSync(LIB, { withFileTypes: true }).filter(e => e.isDirectory()).map(e => e.name));
|
||||||
|
if (restored.mangas.length && !restored.mangas.some(m => present.has(m.id))) throw new Error('Library mount differs from snapshot');
|
||||||
|
lib = restored; scanCache = saved.cache; scanStamps = saved.stamps;
|
||||||
|
scanState = { at: saved.at, mode: 'snapshot', stale: true, mangas: lib.mangas.length, error: null };
|
||||||
|
console.log(`[scan] restored ${lib.mangas.length} manga from saved index; verifying in background`);
|
||||||
|
} catch (e) { if (e.code !== 'ENOENT') console.warn(`[scan] ignoring saved index: ${e.message}`); }
|
||||||
|
|
||||||
function rescan(full) {
|
async function performScan(mode) {
|
||||||
scanning = scanning || (async () => {
|
|
||||||
const t0 = Date.now();
|
const t0 = Date.now();
|
||||||
try {
|
try {
|
||||||
const { index, cache } = await scanLibrary(LIB, full ? {} : scanCache);
|
// Capture before the scan: a publication during the walk remains visible
|
||||||
|
// as a changed stamp on the next check, rather than being silently missed.
|
||||||
|
const stamps = await libraryStamps(LIB);
|
||||||
|
const result = mode === 'changed' && Object.keys(scanStamps).length
|
||||||
|
? await scanChangedLibrary(LIB, lib, scanCache, scanStamps, stamps)
|
||||||
|
: { ...(await scanLibrary(LIB, mode === 'deep' ? {} : scanCache)), stamps, changed: Object.keys(stamps) };
|
||||||
|
const { index, cache, changed } = result;
|
||||||
|
if (mode !== 'changed' || changed.length) {
|
||||||
announce(index);
|
announce(index);
|
||||||
lib = index; scanCache = cache;
|
lib = index; scanCache = cache; scanStamps = stamps;
|
||||||
writeAtomic(CACHE_FILE, JSON.stringify(cache));
|
writeAtomic(CACHE_FILE, JSON.stringify(cache));
|
||||||
|
writeAtomic(INDEX_FILE, JSON.stringify({ version: 1, root: LIB, at: Date.now(), mangas: snapshotIndex(index), cache, stamps }));
|
||||||
|
}
|
||||||
let chapters = 0, pages = 0;
|
let chapters = 0, pages = 0;
|
||||||
for (const m of index.mangas) for (const ch of m.chapters) { chapters++; for (const v of ch.versions) pages += v.pages.length; }
|
for (const m of lib.mangas) for (const ch of m.chapters) { chapters++; for (const v of ch.versions) pages += v.pages.length; }
|
||||||
scanState = { at: Date.now(), ms: Date.now() - t0, mangas: index.mangas.length, chapters, pages, error: null };
|
scanState = { at: Date.now(), ms: Date.now() - t0, mode, changedMangas: changed.length,
|
||||||
console.log(`[scan] ${index.mangas.length} manga, ${chapters} chapters, ${pages} pages in ${scanState.ms} ms`);
|
mangas: lib.mangas.length, chapters, pages, stale: false, error: null };
|
||||||
|
console.log(`[scan] ${mode}: ${changed.length} changed manga, ${lib.mangas.length} manga, ${chapters} chapters, ${pages} pages in ${scanState.ms} ms`);
|
||||||
|
return scanState;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
scanState = { ...scanState, at: Date.now(), error: e.message };
|
scanState = { ...scanState, at: Date.now(), error: e.message };
|
||||||
console.error('[scan] failed:', e.message);
|
console.error('[scan] failed:', e.message);
|
||||||
} finally { scanning = null; }
|
return scanState;
|
||||||
})();
|
}
|
||||||
|
}
|
||||||
|
function queueScan(mode) {
|
||||||
|
if (scanning) {
|
||||||
|
const priority = { changed: 1, full: 2, deep: 3 };
|
||||||
|
if (!pendingScan || priority[mode] > priority[pendingScan]) pendingScan = mode;
|
||||||
|
return scanning;
|
||||||
|
}
|
||||||
|
scanning = (async () => {
|
||||||
|
let next = mode, result;
|
||||||
|
do { result = await performScan(next); next = pendingScan; pendingScan = null; } while (next);
|
||||||
|
return result;
|
||||||
|
})().finally(() => { scanning = null; });
|
||||||
return scanning;
|
return scanning;
|
||||||
}
|
}
|
||||||
|
function rescan(full) { return queueScan(full ? 'deep' : 'full'); }
|
||||||
|
function refreshChanged() { return queueScan('changed'); }
|
||||||
|
|
||||||
// New chapter ids (compared with every id ever seen for that manga) notify the followers.
|
// New chapter ids (compared with every id ever seen for that manga) notify the followers.
|
||||||
function announce(index) {
|
function announce(index) {
|
||||||
@@ -596,8 +633,13 @@ http.createServer(async (req, res) => {
|
|||||||
}
|
}
|
||||||
}).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT} (library: ${LIB}, data: ${DATA})`));
|
}).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT} (library: ${LIB}, data: ${DATA})`));
|
||||||
|
|
||||||
rescan();
|
if (lib.mangas.length) {
|
||||||
if (SCAN_MINUTES > 0) setInterval(() => rescan(), SCAN_MINUTES * 60000).unref();
|
refreshChanged();
|
||||||
|
// Catch in-place manual edits that cannot change a parent directory stamp.
|
||||||
|
setTimeout(() => rescan(), 60000).unref();
|
||||||
|
} else rescan();
|
||||||
|
if (SCAN_MINUTES > 0) setInterval(() => refreshChanged(), SCAN_MINUTES * 60000).unref();
|
||||||
|
if (FULL_SCAN_MINUTES > 0) setInterval(() => rescan(), FULL_SCAN_MINUTES * 60000).unref();
|
||||||
|
|
||||||
// A completed scraper publish increments a durable revision. Polling it here keeps the
|
// A completed scraper publish increments a durable revision. Polling it here keeps the
|
||||||
// reader index fresh without exposing an unauthenticated rescan endpoint to the daemon.
|
// reader index fresh without exposing an unauthenticated rescan endpoint to the daemon.
|
||||||
@@ -606,7 +648,16 @@ async function pollScraper() {
|
|||||||
try {
|
try {
|
||||||
const state = await scraperRequest('/api/state');
|
const state = await scraperRequest('/api/state');
|
||||||
const revision = Number(state.publishedRevision || 0);
|
const revision = Number(state.publishedRevision || 0);
|
||||||
if (scraperRevision !== null && revision > scraperRevision) await rescan();
|
if (scraperRevision === null || revision > scraperRevision) {
|
||||||
|
const checked = await refreshChanged();
|
||||||
|
if (checked.error) throw new Error(checked.error);
|
||||||
|
// A publication without a visible directory change should not disappear
|
||||||
|
// because of coarse filesystem timestamps or a missed scan race.
|
||||||
|
if (scraperRevision !== null && revision > scraperRevision && !checked.changedMangas) {
|
||||||
|
const reconciled = await rescan();
|
||||||
|
if (reconciled.error) throw new Error(reconciled.error);
|
||||||
|
}
|
||||||
|
}
|
||||||
scraperRevision = revision;
|
scraperRevision = revision;
|
||||||
} catch { /* the reader also works when the optional scraper daemon is offline */ }
|
} catch { /* the reader also works when the optional scraper daemon is offline */ }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
'use strict';
|
||||||
|
const test = require('node:test'), assert = require('node:assert/strict');
|
||||||
|
const fs = require('fs'), os = require('os'), path = require('path'), net = require('net');
|
||||||
|
const { spawn } = require('child_process');
|
||||||
|
const { Canvas } = require('../tools/png');
|
||||||
|
const { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex } = require('../library');
|
||||||
|
|
||||||
|
const image = new Canvas(20, 40, [20, 40, 80]).png();
|
||||||
|
function chapter(root, manga, number) {
|
||||||
|
const dir = path.join(root, manga, 'v01', `c${String(number).padStart(4, '0')}`);
|
||||||
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(dir, '0001.png'), image);
|
||||||
|
fs.writeFileSync(path.join(dir, 'chapter.json'), JSON.stringify({ title: `Part ${number}` }));
|
||||||
|
}
|
||||||
|
|
||||||
|
test('saved index restores chapter maps and book references without rereading images', async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-index-'));
|
||||||
|
try {
|
||||||
|
chapter(root, 'Alpha', 1); chapter(root, 'Beta', 1);
|
||||||
|
const first = await scanLibrary(root);
|
||||||
|
const stamps = await libraryStamps(root);
|
||||||
|
const restored = restoreIndex(JSON.parse(JSON.stringify(snapshotIndex(first.index))));
|
||||||
|
assert.equal(restored.byId.get('Alpha').chapterById.get('1/1').title, 'Part 1');
|
||||||
|
assert.strictEqual(restored.byId.get('Alpha').books[0].chapters[0], restored.byId.get('Alpha').chapters[0]);
|
||||||
|
const same = await scanChangedLibrary(root, restored, first.cache, stamps);
|
||||||
|
assert.deepEqual(same.changed, []);
|
||||||
|
assert.strictEqual(same.index, restored);
|
||||||
|
assert.throws(() => restoreIndex([{ id: 'Broken', books: [], chapters: [{ id: 'x', versions: [{}] }] }]), /Invalid chapter/);
|
||||||
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('shallow refresh replaces only a published manga and removes deleted cache entries', async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-changed-'));
|
||||||
|
try {
|
||||||
|
chapter(root, 'Alpha', 1); chapter(root, 'Beta', 1);
|
||||||
|
const first = await scanLibrary(root), stamps = await libraryStamps(root);
|
||||||
|
const beta = first.index.byId.get('Beta');
|
||||||
|
chapter(root, 'Alpha', 2);
|
||||||
|
const next = await scanChangedLibrary(root, first.index, first.cache, stamps);
|
||||||
|
assert.deepEqual(next.changed, ['Alpha']);
|
||||||
|
assert.equal(next.index.byId.get('Alpha').chapters.length, 2);
|
||||||
|
assert.strictEqual(next.index.byId.get('Beta'), beta);
|
||||||
|
assert.equal(Object.keys(next.cache).length, 3);
|
||||||
|
fs.rmSync(path.join(root, 'Alpha'), { recursive: true });
|
||||||
|
const removed = await scanChangedLibrary(root, next.index, next.cache, next.stamps);
|
||||||
|
assert.deepEqual(removed.changed, ['Alpha']);
|
||||||
|
assert.equal(removed.index.byId.has('Alpha'), false);
|
||||||
|
assert.equal(Object.keys(removed.cache).length, 1);
|
||||||
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('in-place chapter metadata edits require a full reconciliation', async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-deep-'));
|
||||||
|
try {
|
||||||
|
chapter(root, 'Alpha', 1);
|
||||||
|
const first = await scanLibrary(root), stamps = await libraryStamps(root);
|
||||||
|
fs.writeFileSync(path.join(root, 'Alpha', 'v01', 'c0001', 'chapter.json'), JSON.stringify({ title: 'Revised Part 1' }));
|
||||||
|
const shallow = await scanChangedLibrary(root, first.index, first.cache, stamps);
|
||||||
|
assert.deepEqual(shallow.changed, []);
|
||||||
|
assert.equal((await scanLibrary(root, first.cache)).index.byId.get('Alpha').chapters[0].title, 'Revised Part 1');
|
||||||
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('reader serves a saved index immediately after restart and verifies it in background', { timeout: 15000 }, async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-warm-start-'));
|
||||||
|
let child;
|
||||||
|
try {
|
||||||
|
const library = path.join(root, 'library'), data = path.join(root, 'data');
|
||||||
|
fs.mkdirSync(data);
|
||||||
|
chapter(library, 'Alpha', 1);
|
||||||
|
const scanned = await scanLibrary(library), stamps = await libraryStamps(library);
|
||||||
|
fs.writeFileSync(path.join(data, 'library-index.json'), JSON.stringify({ version: 1, root: library,
|
||||||
|
at: Date.now(), mangas: snapshotIndex(scanned.index), cache: scanned.cache, stamps }));
|
||||||
|
const listener = net.createServer();
|
||||||
|
await new Promise(resolve => listener.listen(0, '127.0.0.1', resolve));
|
||||||
|
const port = listener.address().port;
|
||||||
|
await new Promise(resolve => listener.close(resolve));
|
||||||
|
const output = [];
|
||||||
|
child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), windowsHide: true,
|
||||||
|
env: { ...process.env, LIBRARY: library, DATA: data, PORT: String(port), SCAN_INTERVAL: '0', FULL_SCAN_INTERVAL: '0',
|
||||||
|
SCRAPER_URL: 'http://127.0.0.1:9', SCRAPER_TOKEN: 'fixture-token', SCRAPER_TOKEN_FILE: '' } });
|
||||||
|
child.stdout.on('data', data => output.push(String(data)));
|
||||||
|
child.stderr.on('data', data => output.push(String(data)));
|
||||||
|
const base = `http://127.0.0.1:${port}`;
|
||||||
|
let ready = false;
|
||||||
|
for (let n = 0; n < 100 && !ready; n++) {
|
||||||
|
ready = await fetch(base + '/health').then(r => r.ok).catch(() => false);
|
||||||
|
if (!ready) await new Promise(resolve => setTimeout(resolve, 50));
|
||||||
|
}
|
||||||
|
assert.ok(ready, output.join(''));
|
||||||
|
const signup = await fetch(base + '/api/signup', { method: 'POST', headers: { 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify({ name: 'fixture', password: 'fixture-password' }) });
|
||||||
|
assert.equal(signup.status, 200);
|
||||||
|
const response = await fetch(base + '/api/library', { headers: { cookie: signup.headers.get('set-cookie').split(';')[0] } });
|
||||||
|
assert.equal(response.status, 200);
|
||||||
|
assert.equal((await response.json()).mangas[0].title, 'Alpha');
|
||||||
|
assert.match(output.join(''), /restored 1 manga from saved index/);
|
||||||
|
} finally {
|
||||||
|
if (child) await new Promise(resolve => { if (child.exitCode !== null) return resolve(); child.once('exit', resolve); child.kill(); });
|
||||||
|
fs.rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
Reference in new issue
Block a user