From 5e14e542862e60e35f095a199776e4723a3df4d4 Mon Sep 17 00:00:00 2001 From: maashkinoozhe Date: Thu, 22 Oct 2026 19:13:43 +0200 Subject: [PATCH] Faster first library scan: bounded concurrency and a 32 KB first read An uncached scan probed one image at a time and read 256 KB of each. On a NAS that is bound by per-file latency (the server/Synology: ~20 ms per image, ~100 min for 298k images). Now chapter folders of a book and their images are probed SCAN_CONCURRENCY (default 8, 1-64) at a time and assembled in folder order, and imageSize() reads 32 KB first, then 256 KB, then the whole file: a prefix gives either no size or the same size, so results do not change. Tests: every format and read stage equals the whole-file answer; concurrency 1 and 8 build the same index and cache; the limiter keeps its bound and per-task errors. (BL-0043) Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + docker-compose.yml | 1 + docs/architecture.md | 2 +- docs/import.md | 6 ++- library.js | 74 +++++++++++++++++++------- server.js | 1 + test/library-scan.test.js | 106 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 169 insertions(+), 22 deletions(-) create mode 100644 test/library-scan.test.js diff --git a/README.md b/README.md index 1678569..86a8249 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ see administration; personal settings do not poll scraper services. See | `SIGNUP` | `closed` | `open` lets anyone create an account. Otherwise the admin adds users under Settings | | `SCAN_INTERVAL` | `10` | Minutes between shallow library change checks (`0` disables this timer) | | `FULL_SCAN_INTERVAL` | `1440` | Minutes between full library reconciliations (`0` disables this timer) | +| `SCAN_CONCURRENCY` | `8` | Chapter folders and images probed at once by an uncached scan (1–64); higher helps on network storage | | `COOKIE_SECURE` | – | `1` marks the login cookie `Secure` (use it when you serve over HTTPS) | | `SCRAPER_URL` | `http://127.0.0.1:8091` | Private scraper daemon URL | | `SCRAPER_TOKEN` | empty | Shared daemon token; required when it listens beyond loopback | diff --git a/docker-compose.yml b/docker-compose.yml index 66e34ce..d9fbf6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: TZ: ${TZ:-Europe/Berlin} SIGNUP: closed SCAN_INTERVAL: "10" + SCAN_CONCURRENCY: ${SCAN_CONCURRENCY:-8} COOKIE_SECURE: ${COOKIE_SECURE:-0} READING_WINDOWS_JSON: ${READING_WINDOWS_JSON:-} READING_TIMEZONE: ${READING_TIMEZONE:-Europe/Berlin} diff --git a/docs/architecture.md b/docs/architecture.md index cf464ac..1ef0ef5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -364,7 +364,7 @@ docker compose up -d --build | Aspect | Behaviour | |--------|-----------| | Library size | The index keeps a small object per page (file name, size, tags), in the order of a few hundred bytes. Hundreds of thousands of pages fit in a modest container. Unchanged chapters cost one `stat` + one small read per scan | -| First scan | Reads ≤ 256 KB per image once. Later scans use the cache | +| First scan | Reads 32 KB per image once (≤ 256 KB, or the whole file for JPEGs with huge EXIF/ICC blocks), `SCAN_CONCURRENCY` (8) at a time. Later scans use the cache | | User data | `db.json` is rewritten as a whole, at most about twice per second while people read. Comfortable up to a few MB (a household or small group, thousands of chapters read) | | Concurrency | One process, one writer. No clustering or horizontal scaling by design | | Images | Served as stored. No resizing or thumbnails, so large covers make the library page heavy | diff --git a/docs/import.md b/docs/import.md index 26dcd2d..ecb9fd9 100644 --- a/docs/import.md +++ b/docs/import.md @@ -148,7 +148,9 @@ Everything is plain data, and the whole index lives in memory. ## 3. Image sizes -The reader lays pages out before they load, so it needs every image's size. Instead of decoding images, the scanner reads the **first 256 KB** and parses the header: +The reader lays pages out before they load, so it needs every image's size. Instead of decoding images, the scanner reads the **first 32 KB** and parses the header; only if the size isn't there yet does it read on to 256 KB (and, for JPEG, the whole file). Parsing a prefix either finds nothing or the same size as the whole file, so the small first read never changes a result. + +Uncached chapter folders and their images are probed **`SCAN_CONCURRENCY` at a time** (default 8, 1–64; chapters of a book in parallel, then assembled in folder order). A first scan over a NAS is bound by per-file latency, not bandwidth: on the server against a Synology (NFS) the old sequential 256 KB probe took about 20 ms per image; see §3.1 for the measurements. `SCAN_CONCURRENCY=1` restores the sequential order. | Format | Where the size is | |--------|-------------------| @@ -156,7 +158,7 @@ The reader lays pages out before they load, so it needs every image's size. Inst | GIF | logical screen (bytes 6–9) | | BMP | DIB header (bytes 18–25) | | WebP | `VP8 ` / `VP8L` / `VP8X` chunk | -| JPEG | first SOFn marker (the scan skips APPn/EXIF/ICC segments). If it isn't within 256 KB, the whole file is read once | +| JPEG | first SOFn marker (the scan skips APPn/EXIF/ICC segments). If it isn't within 32 KB, then 256 KB, the whole file is read once | | AVIF / HEIF | first `ispe` box | - If the size can't be read, it is stored as `0×0`. The reader then assumes 1000×1414 (2000×1414 for `double`) and corrects the layout when the image arrives. diff --git a/library.js b/library.js index 5938586..86322bf 100644 --- a/library.js +++ b/library.js @@ -5,6 +5,8 @@ * * Only folder names, small JSON files and the first bytes of each image (for its size) are read. * Image sizes are cached per chapter folder and only re-read when the folder's mtime changes. + * Uncached chapters and their images are probed SCAN_CONCURRENCY (default 8) at a time: on a NAS a + * first scan is bound by per-file latency, not bandwidth, so a few requests in flight roughly halve it. */ const fsp = require('fs').promises, path = require('path'); const { parseSpecialFolder, parseKey } = require('./chapter-identity'); @@ -15,6 +17,25 @@ const COVER = /^cover\.(jpe?g|png|webp|gif|avif|bmp)$/i; const cmp = new Intl.Collator('en', { numeric: true, sensitivity: 'base' }).compare; const str = x => (typeof x === 'string' ? x.trim() : ''); const order = (a, b) => Number(!!a.special) - Number(!!b.special) || a.x - b.x || a.num - b.num || cmp(a.key, b.key); +const SCAN_CONCURRENCY = Math.min(64, Math.max(1, Math.floor(+process.env.SCAN_CONCURRENCY) || 8)); + +// Runs at most n of the given async tasks at once, in submission order. +function limiter(n) { + let active = 0; + const waiting = []; + const next = () => { + if (active >= n || !waiting.length) return; + active++; + waiting.shift()(); + }; + return task => new Promise((resolve, reject) => { + waiting.push(() => Promise.resolve().then(task).then(resolve, reject).finally(() => { active--; next(); })); + next(); + }); +} +// One pair of limits per scan: chapters (stat, chapter.json, listing) and image probes. +// Chapter tasks wait for image slots but never hold one, so the two cannot deadlock. +const scanLimits = (concurrency = SCAN_CONCURRENCY) => ({ chapters: limiter(concurrency), images: limiter(concurrency) }); async function readJson(file) { try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^/, '')); } @@ -60,16 +81,24 @@ function sizeFromBuffer(b) { return null; } +// Most headers fit in the first few KB, so read 32 KB first and only then up to 256 KB. +// sizeFromBuffer() on a prefix is either null or the same answer as on the whole file, +// so the smaller first read never changes a result. +const HEAD_SMALL = 32 * 1024, HEAD_LARGE = 256 * 1024; async function imageSize(file) { const fh = await fsp.open(file, 'r'); let head; try { - const buf = Buffer.alloc(256 * 1024); - const { bytesRead } = await fh.read(buf, 0, buf.length, 0); - head = buf.subarray(0, bytesRead); + const buf = Buffer.alloc(HEAD_LARGE); + let n = (await fh.read(buf, 0, HEAD_SMALL, 0)).bytesRead; + head = buf.subarray(0, n); + const size = sizeFromBuffer(head); + if (size || n < HEAD_SMALL) return size; + n += (await fh.read(buf, n, HEAD_LARGE - n, n)).bytesRead; + head = buf.subarray(0, n); } finally { await fh.close(); } // JPEGs with big EXIF/ICC blocks can have their size marker further in. - return sizeFromBuffer(head) || (head.length === 256 * 1024 ? sizeFromBuffer(await fsp.readFile(file)) : null); + return sizeFromBuffer(head) || (head.length === HEAD_LARGE ? sizeFromBuffer(await fsp.readFile(file)) : null); } function pageInfo([f, w, h]) { @@ -78,17 +107,16 @@ function pageInfo([f, w, h]) { return { f, w, h, t, r: rot ? +rot.slice(3) : 0 }; } -async function scanVersion(root, rel, ck, oldCache, cache) { +async function scanVersion(root, rel, ck, oldCache, cache, limits) { const dir = path.join(root, rel), st = await fsp.stat(dir); const meta = await readJson(path.join(dir, 'chapter.json')); let c = oldCache[rel]; if (!c || c.m !== st.mtimeMs) { - const p = []; - for (const f of (await list(dir)).files) { - if (!IMG.test(f)) continue; + const files = (await list(dir)).files.filter(f => IMG.test(f)); + const p = await Promise.all(files.map(f => limits.images(async () => { const s = await imageSize(path.join(dir, f)).catch(() => null); - p.push([f, s ? s.w : 0, s ? s.h : 0]); - } + return [f, s ? s.w : 0, s ? s.h : 0]; + }))); c = { m: st.mtimeMs, p }; } cache[rel] = c; @@ -104,7 +132,7 @@ async function scanVersion(root, rel, ck, oldCache, cache) { const thumbOf = v => { const p = v.pages.find(p => p.t.includes('cover')) || v.pages[0]; return v.rel + '/' + p.f; }; const coverIn = (files, rel) => { const f = files.find(f => COVER.test(f)); return f ? rel + '/' + f : null; }; -async function scanManga(root, name, oldCache, cache) { +async function scanManga(root, name, oldCache, cache, limits = scanLimits()) { const { dirs, files } = await list(path.join(root, name)); const meta = await readJson(path.join(root, name, 'manga.json')); const m = { id: name, title: str(meta.title) || name, meta, direction: resolveReadingDirection([meta]).value, cover: coverIn(files, name), books: [], chapters: [], chapterById: new Map(), updated: 0 }; @@ -120,11 +148,17 @@ async function scanManga(root, name, oldCache, cache) { key: bk.key, num: bk.num, x: bk.x, extra: bk.x || bMeta.extra === true, label: (bk.x ? 'Extra book ' : 'Book ') + bk.num, title: str(bMeta.title) || bk.title, cover: coverIn(bl.files, bRel), chapters: [], }; - const byKey = new Map(); + const byKey = new Map(), folders = []; for (const cName of bl.dirs) { const ck = parseSpecialFolder(cName) || parseKey(cName); if (!ck) { console.warn(`[scan] ignoring "${bRel}/${cName}": chapter folders must be named [ title], e.g. "c0001 The Seed"`); continue; } - const v = await scanVersion(root, bRel + '/' + cName, ck, oldCache, cache); + folders.push({ cName, ck }); + } + // Probe the book's chapters concurrently, then assemble them in folder order as before. + const versions = await Promise.all(folders.map(({ cName, ck }) => + limits.chapters(() => scanVersion(root, bRel + '/' + cName, ck, oldCache, cache, limits)))); + for (const [i, { ck }] of folders.entries()) { + const v = versions[i]; if (!v.pages.length) continue; v.directionInfo = resolveReadingDirection([v.meta, bMeta, meta], v.pages); v.direction = v.directionInfo.value; @@ -151,11 +185,12 @@ async function scanManga(root, name, oldCache, cache) { } // Returns { index, cache }. Pass the previous cache to skip re-reading unchanged chapter folders. -async function scanLibrary(root, oldCache = {}) { - const cache = {}, mangas = []; +// options.concurrency overrides SCAN_CONCURRENCY (tests, benchmarks). +async function scanLibrary(root, oldCache = {}, options = {}) { + const cache = {}, mangas = [], limits = scanLimits(options.concurrency); for (const name of (await list(root)).dirs) { try { - const m = await scanManga(root, name, oldCache, cache); + const m = await scanManga(root, name, oldCache, cache, limits); if (m.chapters.length) mangas.push(m); else console.warn(`[scan] "${name}" has no chapters (expected ///)`); } catch (e) { console.warn(`[scan] skipped "${name}": ${e.message}`); } @@ -185,8 +220,9 @@ async function libraryStamps(root) { return stamps; } -async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps) { +async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps, options = {}) { stamps = stamps || await libraryStamps(root); + const limits = scanLimits(options.concurrency); const names = new Set([...Object.keys(oldStamps || {}), ...Object.keys(stamps)]); const changed = [...names].filter(name => JSON.stringify(oldStamps?.[name]) !== JSON.stringify(stamps[name])); if (!changed.length) return { index: prior, cache: oldCache, stamps, changed }; @@ -196,7 +232,7 @@ async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps) { byId.delete(name); for (const key of Object.keys(cache)) if (key.startsWith(name + '/')) delete cache[key]; if (!Object.hasOwn(stamps, name)) continue; - const m = await scanManga(root, name, oldCache, cache); + const m = await scanManga(root, name, oldCache, cache, limits); if (m.chapters.length) byId.set(name, m); } const mangas = [...byId.values()].sort((a, b) => cmp(a.title, b.title)); @@ -232,4 +268,4 @@ function restoreIndex(mangas) { } module.exports = { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex, - parseKey, sizeFromBuffer, imageSize }; + parseKey, sizeFromBuffer, imageSize, limiter }; diff --git a/server.js b/server.js index 2fe2728..08ae1e3 100644 --- a/server.js +++ b/server.js @@ -7,6 +7,7 @@ * SIGNUP=closed|open (the very first account can always be created and becomes admin) * SCAN_INTERVAL=10 minutes between shallow change checks, 0 = off * FULL_SCAN_INTERVAL=1440 minutes between full reconciliations, 0 = off + * SCAN_CONCURRENCY=8 chapter folders / images probed at once by an uncached scan (library.js) * COOKIE_SECURE=1 mark the session cookie Secure (when served over HTTPS) * SCRAPER_URL=http://127.0.0.1:8091 scraper daemon API * SCRAPER_TOKEN=... shared daemon token (optional on loopback) diff --git a/test/library-scan.test.js b/test/library-scan.test.js new file mode 100644 index 0000000..cda1f70 --- /dev/null +++ b/test/library-scan.test.js @@ -0,0 +1,106 @@ +'use strict'; +// First-scan speed (BL-0043): small first reads and bounded concurrency must not change any result. +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('fs'), os = require('os'), path = require('path'); +const { Canvas } = require('../tools/png'); +const { scanLibrary, snapshotIndex, sizeFromBuffer, imageSize, limiter } = require('../library'); + +const tmp = prefix => fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +const u16be = n => Buffer.from([n >> 8, n & 255]); + +// JPEG with `appBytes` of APP1 payload (in 64 KB segments) before the SOF0 marker. +function jpeg(w, h, appBytes) { + const parts = [Buffer.from([0xff, 0xd8])]; + for (let left = appBytes; left > 0;) { + const n = Math.min(left, 65533); + parts.push(Buffer.from([0xff, 0xe1]), u16be(n + 2), Buffer.alloc(n, 0x41)); + left -= n; + } + parts.push(Buffer.from([0xff, 0xc0, 0x00, 0x11, 0x08]), u16be(h), u16be(w), Buffer.alloc(12)); + parts.push(Buffer.from([0xff, 0xda]), Buffer.alloc(4096, 0x55), Buffer.from([0xff, 0xd9])); + return Buffer.concat(parts); +} +function webpVp8x(w, h) { + const b = Buffer.alloc(64); + b.write('RIFF', 0, 'latin1'); b.writeUInt32LE(56, 4); b.write('WEBP', 8, 'latin1'); b.write('VP8X', 12, 'latin1'); + b.writeUIntLE(w - 1, 24, 3); b.writeUIntLE(h - 1, 27, 3); + return b; +} +function gif(w, h) { + const b = Buffer.alloc(32); + b.write('GIF89a', 0, 'latin1'); b.writeUInt16LE(w, 6); b.writeUInt16LE(h, 8); + return b; +} +// AVIF-like: ftyp box, then an `ispe` box after `pad` bytes. +function avif(w, h, pad) { + const b = Buffer.alloc(32 + pad + 20); + b.writeUInt32BE(24, 0); b.write('ftypavif', 4, 'latin1'); + const k = 32 + pad; + b.write('ispe', k, 'latin1'); b.writeUInt32BE(w, k + 8); b.writeUInt32BE(h, k + 12); + return b; +} + +test('the 32 KB first read gives the same size as the whole file, in every read stage', async () => { + const dir = tmp('mc-probe-'); + try { + const cases = { + 'small.png': new Canvas(37, 53, [1, 2, 3]).png(), + 'plain.jpg': jpeg(800, 1200, 0), + 'exif-20k.jpg': jpeg(801, 1201, 20 * 1024), // found in the first 32 KB + 'exif-100k.jpg': jpeg(802, 1202, 100 * 1024), // needs the 256 KB read + 'exif-400k.jpg': jpeg(803, 1203, 400 * 1024), // needs the whole file + 'tiny.webp': webpVp8x(640, 960), + 'tiny.gif': gif(10, 20), + 'deep.avif': avif(1024, 1536, 50 * 1024), // ispe past 32 KB + 'broken.jpg': Buffer.concat([Buffer.from([0xff, 0xd8]), Buffer.alloc(40 * 1024, 0x41)]), + }; + for (const [name, bytes] of Object.entries(cases)) { + fs.writeFileSync(path.join(dir, name), bytes); + assert.deepEqual(await imageSize(path.join(dir, name)), sizeFromBuffer(bytes), name); + } + assert.deepEqual(await imageSize(path.join(dir, 'exif-400k.jpg')), { w: 803, h: 1203 }); + assert.deepEqual(await imageSize(path.join(dir, 'deep.avif')), { w: 1024, h: 1536 }); + assert.equal(await imageSize(path.join(dir, 'broken.jpg')), null); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); + +test('concurrent scanning builds the same index and cache as a sequential scan', async () => { + const root = tmp('mc-concurrency-'); + try { + let n = 0; + for (const manga of ['Alpha', 'Beta']) for (const book of ['v01', 'v02']) for (let c = 1; c <= 6; c++) { + const dir = path.join(root, manga, book, `c${String(c).padStart(4, '0')} [Src]`); + fs.mkdirSync(dir, { recursive: true }); + for (let p = 1; p <= 5; p++, n++) { + const file = path.join(dir, `${String(p).padStart(4, '0')}${p === 3 ? '_double' : ''}`); + if (n % 3 === 0) fs.writeFileSync(file + '.jpg', jpeg(700 + n, 1000 + n, (n % 5) * 30 * 1024)); + else fs.writeFileSync(file + '.png', new Canvas(10 + n, 20 + n, [n, n, n]).png()); + } + fs.writeFileSync(path.join(dir, 'chapter.json'), JSON.stringify({ title: `${manga} ${book} ${c}` })); + } + fs.mkdirSync(path.join(root, 'Alpha', 'v01', 'not a chapter')); + const one = await scanLibrary(root, {}, { concurrency: 1 }); + const many = await scanLibrary(root, {}, { concurrency: 8 }); + assert.deepEqual(snapshotIndex(many.index), snapshotIndex(one.index)); + const sorted = c => Object.fromEntries(Object.entries(c).sort(([a], [b]) => a.localeCompare(b))); + assert.deepEqual(sorted(many.cache), sorted(one.cache)); + assert.equal(Object.keys(one.cache).length, 24); + assert.equal(one.index.byId.get('Beta').chapters.length, 12); + } finally { fs.rmSync(root, { recursive: true, force: true }); } +}); + +test('limiter runs at most n tasks at once and keeps results and errors per task', async () => { + const limit = limiter(3); + let active = 0, peak = 0; + const task = i => limit(async () => { + active++; peak = Math.max(peak, active); + await new Promise(r => setTimeout(r, 5 + (i % 4) * 3)); + active--; + if (i === 7) throw new Error('boom'); + return i * 2; + }); + const results = await Promise.allSettled(Array.from({ length: 20 }, (_, i) => task(i))); + assert.equal(peak, 3); + assert.equal(results[7].status, 'rejected'); + assert.deepEqual(results.filter((_, i) => i !== 7).map(r => r.value), Array.from({ length: 20 }, (_, i) => i * 2).filter((_, i) => i !== 7)); +});