Faster first library scan: bounded concurrency and a 32 KB first read

An uncached scan probed one image at a time and read 256 KB of each. On
a NAS that is bound by per-file latency (the server/Synology: ~20 ms per
image, ~100 min for 298k images). Now chapter folders of a book and
their images are probed SCAN_CONCURRENCY (default 8, 1-64) at a time
and assembled in folder order, and imageSize() reads 32 KB first, then
256 KB, then the whole file: a prefix gives either no size or the same
size, so results do not change. Tests: every format and read stage
equals the whole-file answer; concurrency 1 and 8 build the same index
and cache; the limiter keeps its bound and per-task errors. (BL-0043)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

library.js

@@ -5,6 +5,8 @@
  *
  * Only folder names, small JSON files and the first bytes of each image (for its size) are read.
  * Image sizes are cached per chapter folder and only re-read when the folder's mtime changes.
+ * Uncached chapters and their images are probed SCAN_CONCURRENCY (default 8) at a time: on a NAS a
+ * first scan is bound by per-file latency, not bandwidth, so a few requests in flight roughly halve it.
  */
 const fsp = require('fs').promises, path = require('path');
 const { parseSpecialFolder, parseKey } = require('./chapter-identity');
@@ -15,6 +17,25 @@ const COVER = /^cover\.(jpe?g|png|webp|gif|avif|bmp)$/i;
 const cmp = new Intl.Collator('en', { numeric: true, sensitivity: 'base' }).compare;
 const str = x => (typeof x === 'string' ? x.trim() : '');
 const order = (a, b) => Number(!!a.special) - Number(!!b.special) || a.x - b.x || a.num - b.num || cmp(a.key, b.key);
+const SCAN_CONCURRENCY = Math.min(64, Math.max(1, Math.floor(+process.env.SCAN_CONCURRENCY) || 8));
+
+// Runs at most n of the given async tasks at once, in submission order.
+function limiter(n) {
+  let active = 0;
+  const waiting = [];
+  const next = () => {
+    if (active >= n || !waiting.length) return;
+    active++;
+    waiting.shift()();
+  };
+  return task => new Promise((resolve, reject) => {
+    waiting.push(() => Promise.resolve().then(task).then(resolve, reject).finally(() => { active--; next(); }));
+    next();
+  });
+}
+// One pair of limits per scan: chapters (stat, chapter.json, listing) and image probes.
+// Chapter tasks wait for image slots but never hold one, so the two cannot deadlock.
+const scanLimits = (concurrency = SCAN_CONCURRENCY) => ({ chapters: limiter(concurrency), images: limiter(concurrency) });
 
 async function readJson(file) {
   try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^/, '')); }
@@ -60,16 +81,24 @@ function sizeFromBuffer(b) {
   return null;
 }
 
+// Most headers fit in the first few KB, so read 32 KB first and only then up to 256 KB.
+// sizeFromBuffer() on a prefix is either null or the same answer as on the whole file,
+// so the smaller first read never changes a result.
+const HEAD_SMALL = 32 * 1024, HEAD_LARGE = 256 * 1024;
 async function imageSize(file) {
   const fh = await fsp.open(file, 'r');
   let head;
   try {
-    const buf = Buffer.alloc(256 * 1024);
-    const { bytesRead } = await fh.read(buf, 0, buf.length, 0);
-    head = buf.subarray(0, bytesRead);
+    const buf = Buffer.alloc(HEAD_LARGE);
+    let n = (await fh.read(buf, 0, HEAD_SMALL, 0)).bytesRead;
+    head = buf.subarray(0, n);
+    const size = sizeFromBuffer(head);
+    if (size || n < HEAD_SMALL) return size;
+    n += (await fh.read(buf, n, HEAD_LARGE - n, n)).bytesRead;
+    head = buf.subarray(0, n);
   } finally { await fh.close(); }
   // JPEGs with big EXIF/ICC blocks can have their size marker further in.
-  return sizeFromBuffer(head) || (head.length === 256 * 1024 ? sizeFromBuffer(await fsp.readFile(file)) : null);
+  return sizeFromBuffer(head) || (head.length === HEAD_LARGE ? sizeFromBuffer(await fsp.readFile(file)) : null);
 }
 
 function pageInfo([f, w, h]) {
@@ -78,17 +107,16 @@ function pageInfo([f, w, h]) {
   return { f, w, h, t, r: rot ? +rot.slice(3) : 0 };
 }
 
-async function scanVersion(root, rel, ck, oldCache, cache) {
+async function scanVersion(root, rel, ck, oldCache, cache, limits) {
   const dir = path.join(root, rel), st = await fsp.stat(dir);
   const meta = await readJson(path.join(dir, 'chapter.json'));
   let c = oldCache[rel];
   if (!c || c.m !== st.mtimeMs) {
-    const p = [];
-    for (const f of (await list(dir)).files) {
-      if (!IMG.test(f)) continue;
+    const files = (await list(dir)).files.filter(f => IMG.test(f));
+    const p = await Promise.all(files.map(f => limits.images(async () => {
       const s = await imageSize(path.join(dir, f)).catch(() => null);
-      p.push([f, s ? s.w : 0, s ? s.h : 0]);
-    }
+      return [f, s ? s.w : 0, s ? s.h : 0];
+    })));
     c = { m: st.mtimeMs, p };
   }
   cache[rel] = c;
@@ -104,7 +132,7 @@ async function scanVersion(root, rel, ck, oldCache, cache) {
 const thumbOf = v => { const p = v.pages.find(p => p.t.includes('cover')) || v.pages[0]; return v.rel + '/' + p.f; };
 const coverIn = (files, rel) => { const f = files.find(f => COVER.test(f)); return f ? rel + '/' + f : null; };
 
-async function scanManga(root, name, oldCache, cache) {
+async function scanManga(root, name, oldCache, cache, limits = scanLimits()) {
   const { dirs, files } = await list(path.join(root, name));
   const meta = await readJson(path.join(root, name, 'manga.json'));
   const m = { id: name, title: str(meta.title) || name, meta, direction: resolveReadingDirection([meta]).value, cover: coverIn(files, name), books: [], chapters: [], chapterById: new Map(), updated: 0 };
@@ -120,11 +148,17 @@ async function scanManga(root, name, oldCache, cache) {
       key: bk.key, num: bk.num, x: bk.x, extra: bk.x || bMeta.extra === true,
       label: (bk.x ? 'Extra book ' : 'Book ') + bk.num, title: str(bMeta.title) || bk.title, cover: coverIn(bl.files, bRel), chapters: [],
     };
-    const byKey = new Map();
+    const byKey = new Map(), folders = [];
     for (const cName of bl.dirs) {
       const ck = parseSpecialFolder(cName) || parseKey(cName);
       if (!ck) { console.warn(`[scan] ignoring "${bRel}/${cName}": chapter folders must be named <letters><number>[ title], e.g. "c0001 The Seed"`); continue; }
-      const v = await scanVersion(root, bRel + '/' + cName, ck, oldCache, cache);
+      folders.push({ cName, ck });
+    }
+    // Probe the book's chapters concurrently, then assemble them in folder order as before.
+    const versions = await Promise.all(folders.map(({ cName, ck }) =>
+      limits.chapters(() => scanVersion(root, bRel + '/' + cName, ck, oldCache, cache, limits))));
+    for (const [i, { ck }] of folders.entries()) {
+      const v = versions[i];
       if (!v.pages.length) continue;
       v.directionInfo = resolveReadingDirection([v.meta, bMeta, meta], v.pages);
       v.direction = v.directionInfo.value;
@@ -151,11 +185,12 @@ async function scanManga(root, name, oldCache, cache) {
 }
 
 // Returns { index, cache }. Pass the previous cache to skip re-reading unchanged chapter folders.
-async function scanLibrary(root, oldCache = {}) {
-  const cache = {}, mangas = [];
+// options.concurrency overrides SCAN_CONCURRENCY (tests, benchmarks).
+async function scanLibrary(root, oldCache = {}, options = {}) {
+  const cache = {}, mangas = [], limits = scanLimits(options.concurrency);
   for (const name of (await list(root)).dirs) {
     try {
-      const m = await scanManga(root, name, oldCache, cache);
+      const m = await scanManga(root, name, oldCache, cache, limits);
       if (m.chapters.length) mangas.push(m);
       else console.warn(`[scan] "${name}" has no chapters (expected <manga>/<book>/<chapter>/<images>)`);
     } catch (e) { console.warn(`[scan] skipped "${name}": ${e.message}`); }
@@ -185,8 +220,9 @@ async function libraryStamps(root) {
   return stamps;
 }
 
-async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps) {
+async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps, options = {}) {
   stamps = stamps || await libraryStamps(root);
+  const limits = scanLimits(options.concurrency);
   const names = new Set([...Object.keys(oldStamps || {}), ...Object.keys(stamps)]);
   const changed = [...names].filter(name => JSON.stringify(oldStamps?.[name]) !== JSON.stringify(stamps[name]));
   if (!changed.length) return { index: prior, cache: oldCache, stamps, changed };
@@ -196,7 +232,7 @@ async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps) {
     byId.delete(name);
     for (const key of Object.keys(cache)) if (key.startsWith(name + '/')) delete cache[key];
     if (!Object.hasOwn(stamps, name)) continue;
-    const m = await scanManga(root, name, oldCache, cache);
+    const m = await scanManga(root, name, oldCache, cache, limits);
     if (m.chapters.length) byId.set(name, m);
   }
   const mangas = [...byId.values()].sort((a, b) => cmp(a.title, b.title));
@@ -232,4 +268,4 @@ function restoreIndex(mangas) {
 }
 
 module.exports = { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex,
-  parseKey, sizeFromBuffer, imageSize };
+  parseKey, sizeFromBuffer, imageSize, limiter };