Faster first library scan: bounded concurrency and a 32 KB first read

An uncached scan probed one image at a time and read 256 KB of each. On
a NAS that is bound by per-file latency (the server/Synology: ~20 ms per
image, ~100 min for 298k images). Now chapter folders of a book and
their images are probed SCAN_CONCURRENCY (default 8, 1-64) at a time
and assembled in folder order, and imageSize() reads 32 KB first, then
256 KB, then the whole file: a prefix gives either no size or the same
size, so results do not change. Tests: every format and read stage
equals the whole-file answer; concurrency 1 and 8 build the same index
and cache; the limiter keeps its bound and per-task errors. (BL-0043)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

test/library-scan.test.js

@@ -0,0 +1,106 @@
+'use strict';
+// First-scan speed (BL-0043): small first reads and bounded concurrency must not change any result.
+const test = require('node:test'), assert = require('node:assert/strict');
+const fs = require('fs'), os = require('os'), path = require('path');
+const { Canvas } = require('../tools/png');
+const { scanLibrary, snapshotIndex, sizeFromBuffer, imageSize, limiter } = require('../library');
+
+const tmp = prefix => fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+const u16be = n => Buffer.from([n >> 8, n & 255]);
+
+// JPEG with `appBytes` of APP1 payload (in 64 KB segments) before the SOF0 marker.
+function jpeg(w, h, appBytes) {
+  const parts = [Buffer.from([0xff, 0xd8])];
+  for (let left = appBytes; left > 0;) {
+    const n = Math.min(left, 65533);
+    parts.push(Buffer.from([0xff, 0xe1]), u16be(n + 2), Buffer.alloc(n, 0x41));
+    left -= n;
+  }
+  parts.push(Buffer.from([0xff, 0xc0, 0x00, 0x11, 0x08]), u16be(h), u16be(w), Buffer.alloc(12));
+  parts.push(Buffer.from([0xff, 0xda]), Buffer.alloc(4096, 0x55), Buffer.from([0xff, 0xd9]));
+  return Buffer.concat(parts);
+}
+function webpVp8x(w, h) {
+  const b = Buffer.alloc(64);
+  b.write('RIFF', 0, 'latin1'); b.writeUInt32LE(56, 4); b.write('WEBP', 8, 'latin1'); b.write('VP8X', 12, 'latin1');
+  b.writeUIntLE(w - 1, 24, 3); b.writeUIntLE(h - 1, 27, 3);
+  return b;
+}
+function gif(w, h) {
+  const b = Buffer.alloc(32);
+  b.write('GIF89a', 0, 'latin1'); b.writeUInt16LE(w, 6); b.writeUInt16LE(h, 8);
+  return b;
+}
+// AVIF-like: ftyp box, then an `ispe` box after `pad` bytes.
+function avif(w, h, pad) {
+  const b = Buffer.alloc(32 + pad + 20);
+  b.writeUInt32BE(24, 0); b.write('ftypavif', 4, 'latin1');
+  const k = 32 + pad;
+  b.write('ispe', k, 'latin1'); b.writeUInt32BE(w, k + 8); b.writeUInt32BE(h, k + 12);
+  return b;
+}
+
+test('the 32 KB first read gives the same size as the whole file, in every read stage', async () => {
+  const dir = tmp('mc-probe-');
+  try {
+    const cases = {
+      'small.png': new Canvas(37, 53, [1, 2, 3]).png(),
+      'plain.jpg': jpeg(800, 1200, 0),
+      'exif-20k.jpg': jpeg(801, 1201, 20 * 1024),      // found in the first 32 KB
+      'exif-100k.jpg': jpeg(802, 1202, 100 * 1024),    // needs the 256 KB read
+      'exif-400k.jpg': jpeg(803, 1203, 400 * 1024),    // needs the whole file
+      'tiny.webp': webpVp8x(640, 960),
+      'tiny.gif': gif(10, 20),
+      'deep.avif': avif(1024, 1536, 50 * 1024),        // ispe past 32 KB
+      'broken.jpg': Buffer.concat([Buffer.from([0xff, 0xd8]), Buffer.alloc(40 * 1024, 0x41)]),
+    };
+    for (const [name, bytes] of Object.entries(cases)) {
+      fs.writeFileSync(path.join(dir, name), bytes);
+      assert.deepEqual(await imageSize(path.join(dir, name)), sizeFromBuffer(bytes), name);
+    }
+    assert.deepEqual(await imageSize(path.join(dir, 'exif-400k.jpg')), { w: 803, h: 1203 });
+    assert.deepEqual(await imageSize(path.join(dir, 'deep.avif')), { w: 1024, h: 1536 });
+    assert.equal(await imageSize(path.join(dir, 'broken.jpg')), null);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('concurrent scanning builds the same index and cache as a sequential scan', async () => {
+  const root = tmp('mc-concurrency-');
+  try {
+    let n = 0;
+    for (const manga of ['Alpha', 'Beta']) for (const book of ['v01', 'v02']) for (let c = 1; c <= 6; c++) {
+      const dir = path.join(root, manga, book, `c${String(c).padStart(4, '0')} [Src]`);
+      fs.mkdirSync(dir, { recursive: true });
+      for (let p = 1; p <= 5; p++, n++) {
+        const file = path.join(dir, `${String(p).padStart(4, '0')}${p === 3 ? '_double' : ''}`);
+        if (n % 3 === 0) fs.writeFileSync(file + '.jpg', jpeg(700 + n, 1000 + n, (n % 5) * 30 * 1024));
+        else fs.writeFileSync(file + '.png', new Canvas(10 + n, 20 + n, [n, n, n]).png());
+      }
+      fs.writeFileSync(path.join(dir, 'chapter.json'), JSON.stringify({ title: `${manga} ${book} ${c}` }));
+    }
+    fs.mkdirSync(path.join(root, 'Alpha', 'v01', 'not a chapter'));
+    const one = await scanLibrary(root, {}, { concurrency: 1 });
+    const many = await scanLibrary(root, {}, { concurrency: 8 });
+    assert.deepEqual(snapshotIndex(many.index), snapshotIndex(one.index));
+    const sorted = c => Object.fromEntries(Object.entries(c).sort(([a], [b]) => a.localeCompare(b)));
+    assert.deepEqual(sorted(many.cache), sorted(one.cache));
+    assert.equal(Object.keys(one.cache).length, 24);
+    assert.equal(one.index.byId.get('Beta').chapters.length, 12);
+  } finally { fs.rmSync(root, { recursive: true, force: true }); }
+});
+
+test('limiter runs at most n tasks at once and keeps results and errors per task', async () => {
+  const limit = limiter(3);
+  let active = 0, peak = 0;
+  const task = i => limit(async () => {
+    active++; peak = Math.max(peak, active);
+    await new Promise(r => setTimeout(r, 5 + (i % 4) * 3));
+    active--;
+    if (i === 7) throw new Error('boom');
+    return i * 2;
+  });
+  const results = await Promise.allSettled(Array.from({ length: 20 }, (_, i) => task(i)));
+  assert.equal(peak, 3);
+  assert.equal(results[7].status, 'rejected');
+  assert.deepEqual(results.filter((_, i) => i !== 7).map(r => r.value), Array.from({ length: 20 }, (_, i) => i * 2).filter((_, i) => i !== 7));
+});