MangaCircus/library.js

'use strict';
/*
 * Library scanner: turns   <library>/<Manga>/<Book>/<Chapter>/<page images>
 * into an in-memory index. The folder format is documented in README.md.
 *
 * Only folder names, small JSON files and the first bytes of each image (for its size) are read.
 * Image sizes are cached per chapter folder and only re-read when the folder's mtime changes.
 * Uncached chapters and their images are probed SCAN_CONCURRENCY (default 8) at a time: on a NAS a
 * first scan is bound by per-file latency, not bandwidth, so a few requests in flight roughly halve it.
 */
const fsp = require('fs').promises, path = require('path');
const { parseSpecialFolder, parseKey } = require('./chapter-identity');
const { resolveReadingDirection } = require('./reading-direction');

const IMG = /\.(jpe?g|png|webp|gif|avif|bmp)$/i;
const COVER = /^cover\.(jpe?g|png|webp|gif|avif|bmp)$/i;
const cmp = new Intl.Collator('en', { numeric: true, sensitivity: 'base' }).compare;
const str = x => (typeof x === 'string' ? x.trim() : '');
const order = (a, b) => Number(!!a.special) - Number(!!b.special) || a.x - b.x || a.num - b.num || cmp(a.key, b.key);
const SCAN_CONCURRENCY = Math.min(64, Math.max(1, Math.floor(+process.env.SCAN_CONCURRENCY) || 8));

// Runs at most n of the given async tasks at once, in submission order.
function limiter(n) {
  let active = 0;
  const waiting = [];
  const next = () => {
    if (active >= n || !waiting.length) return;
    active++;
    waiting.shift()();
  };
  return task => new Promise((resolve, reject) => {
    waiting.push(() => Promise.resolve().then(task).then(resolve, reject).finally(() => { active--; next(); }));
    next();
  });
}
// One pair of limits per scan: chapters (stat, chapter.json, listing) and image probes.
// Chapter tasks wait for image slots but never hold one, so the two cannot deadlock.
const scanLimits = (concurrency = SCAN_CONCURRENCY) => ({ chapters: limiter(concurrency), images: limiter(concurrency) });

async function readJson(file) {
  try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^/, '')); }
  catch (e) { if (e.code !== 'ENOENT') console.warn(`[scan] ignoring broken ${file}: ${e.message}`); return {}; }
}

async function list(dir) {
  const dirs = [], files = [];
  for (const e of await fsp.readdir(dir, { withFileTypes: true })) {
    if (e.name.startsWith('.')) continue;
    let isDir = e.isDirectory();
    if (e.isSymbolicLink()) isDir = await fsp.stat(path.join(dir, e.name)).then(s => s.isDirectory(), () => false);
    (isDir ? dirs : files).push(e.name);
  }
  return { dirs: dirs.sort(cmp), files: files.sort(cmp) };
}

// Width/height from the image header (PNG, GIF, BMP, WebP, JPEG, AVIF/HEIF).
function sizeFromBuffer(b) {
  try {
    if (b.readUInt32BE(0) === 0x89504e47) return { w: b.readUInt32BE(16), h: b.readUInt32BE(20) };
    if (b.toString('latin1', 0, 3) === 'GIF') return { w: b.readUInt16LE(6), h: b.readUInt16LE(8) };
    if (b.toString('latin1', 0, 2) === 'BM') return { w: b.readInt32LE(18), h: Math.abs(b.readInt32LE(22)) };
    if (b.toString('latin1', 0, 4) === 'RIFF' && b.toString('latin1', 8, 12) === 'WEBP') {
      const f = b.toString('latin1', 12, 16);
      if (f === 'VP8 ') return { w: b.readUInt16LE(26) & 0x3fff, h: b.readUInt16LE(28) & 0x3fff };
      if (f === 'VP8L') { const v = b.readUInt32LE(21); return { w: (v & 0x3fff) + 1, h: ((v >> 14) & 0x3fff) + 1 }; }
      if (f === 'VP8X') return { w: b.readUIntLE(24, 3) + 1, h: b.readUIntLE(27, 3) + 1 };
    }
    if (b[0] === 0xff && b[1] === 0xd8) {
      for (let i = 2; i + 9 < b.length;) {
        if (b[i] !== 0xff) { i++; continue; }
        const mk = b[i + 1];
        if (mk === 0xff) { i++; continue; }
        if (mk === 0x01 || (mk >= 0xd0 && mk <= 0xd8)) { i += 2; continue; }
        if (mk >= 0xc0 && mk <= 0xcf && mk !== 0xc4 && mk !== 0xc8 && mk !== 0xcc) return { w: b.readUInt16BE(i + 7), h: b.readUInt16BE(i + 5) };
        i += 2 + b.readUInt16BE(i + 2);
      }
    }
    const k = b.indexOf('ispe');
    if (k > 0 && b.toString('latin1', 4, 8) === 'ftyp') return { w: b.readUInt32BE(k + 8), h: b.readUInt32BE(k + 12) };
  } catch { /* truncated or unknown header */ }
  return null;
}

// Most headers fit in the first few KB, so read 32 KB first and only then up to 256 KB.
// sizeFromBuffer() on a prefix is either null or the same answer as on the whole file,
// so the smaller first read never changes a result.
const HEAD_SMALL = 32 * 1024, HEAD_LARGE = 256 * 1024;
async function imageSize(file) {
  const fh = await fsp.open(file, 'r');
  let head;
  try {
    const buf = Buffer.alloc(HEAD_LARGE);
    let n = (await fh.read(buf, 0, HEAD_SMALL, 0)).bytesRead;
    head = buf.subarray(0, n);
    const size = sizeFromBuffer(head);
    if (size || n < HEAD_SMALL) return size;
    n += (await fh.read(buf, n, HEAD_LARGE - n, n)).bytesRead;
    head = buf.subarray(0, n);
  } finally { await fh.close(); }
  // JPEGs with big EXIF/ICC blocks can have their size marker further in.
  return sizeFromBuffer(head) || (head.length === HEAD_LARGE ? sizeFromBuffer(await fsp.readFile(file)) : null);
}

function pageInfo([f, w, h]) {
  const t = f.slice(0, f.lastIndexOf('.')).toLowerCase().split('_').slice(1);
  const rot = t.find(x => /^rot(90|180|270)$/.test(x));
  return { f, w, h, t, r: rot ? +rot.slice(3) : 0 };
}

async function scanVersion(root, rel, ck, oldCache, cache, limits) {
  const dir = path.join(root, rel), st = await fsp.stat(dir);
  const meta = await readJson(path.join(dir, 'chapter.json'));
  let c = oldCache[rel];
  if (!c || c.m !== st.mtimeMs) {
    const files = (await list(dir)).files.filter(f => IMG.test(f));
    const p = await Promise.all(files.map(f => limits.images(async () => {
      const s = await imageSize(path.join(dir, f)).catch(() => null);
      return [f, s ? s.w : 0, s ? s.h : 0];
    })));
    c = { m: st.mtimeMs, p };
  }
  cache[rel] = c;
  const src = meta.source && typeof meta.source === 'object' ? meta.source : { name: meta.source };
  return {
    id: rel.slice(rel.lastIndexOf('/') + 1), rel, meta,
    source: str(src.name) || ck.src, sourceUrl: str(src.url),
    added: Date.parse(meta.added) || st.mtimeMs,
    pages: c.p.map(pageInfo),
  };
}

const thumbOf = v => { const p = v.pages.find(p => p.t.includes('cover')) || v.pages[0]; return v.rel + '/' + p.f; };
const coverIn = (files, rel) => { const f = files.find(f => COVER.test(f)); return f ? rel + '/' + f : null; };

async function scanManga(root, name, oldCache, cache, limits = scanLimits()) {
  const { dirs, files } = await list(path.join(root, name));
  const meta = await readJson(path.join(root, name, 'manga.json'));
  const m = { id: name, title: str(meta.title) || name, meta, direction: resolveReadingDirection([meta]).value, cover: coverIn(files, name), books: [], chapters: [], chapterById: new Map(), updated: 0 };
  const bookKeys = new Set();
  for (const bName of dirs) {
    const bk = parseKey(bName);
    if (!bk) { console.warn(`[scan] ignoring "${name}/${bName}": book folders must be named <letters><number>[ title], e.g. "v01 Spring"`); continue; }
    if (bookKeys.has(bk.key)) { console.warn(`[scan] ignoring "${name}/${bName}": another book folder already uses number ${bk.key}`); continue; }
    bookKeys.add(bk.key);
    const bRel = name + '/' + bName, bl = await list(path.join(root, bRel));
    const bMeta = await readJson(path.join(root, bRel, 'book.json'));
    const book = {
      key: bk.key, num: bk.num, x: bk.x, extra: bk.x || bMeta.extra === true,
      label: (bk.x ? 'Extra book ' : 'Book ') + bk.num, title: str(bMeta.title) || bk.title, cover: coverIn(bl.files, bRel), chapters: [],
    };
    const byKey = new Map(), folders = [];
    for (const cName of bl.dirs) {
      const ck = parseSpecialFolder(cName) || parseKey(cName);
      if (!ck) { console.warn(`[scan] ignoring "${bRel}/${cName}": chapter folders must be named <letters><number>[ title], e.g. "c0001 The Seed"`); continue; }
      folders.push({ cName, ck });
    }
    // Probe the book's chapters concurrently, then assemble them in folder order as before.
    const versions = await Promise.all(folders.map(({ cName, ck }) =>
      limits.chapters(() => scanVersion(root, bRel + '/' + cName, ck, oldCache, cache, limits))));
    for (const [i, { ck }] of folders.entries()) {
      const v = versions[i];
      if (!v.pages.length) continue;
      v.directionInfo = resolveReadingDirection([v.meta, bMeta, meta], v.pages);
      v.direction = v.directionInfo.value;
      let ch = byKey.get(ck.key);
      if (!ch) {
        ch = { id: book.key + '/' + ck.key, book: book.key, key: ck.key, num: ck.num, x: ck.x, special: !!ck.special, extra: book.extra || ck.x, label: ck.special ? str(v.meta.label) || 'Special' : (ck.x ? 'Extra ' : 'Ch. ') + ck.num, title: '', versions: [] };
        byKey.set(ck.key, ch);
      }
      if (v.meta.extra === true) ch.extra = true;
      ch.title = ch.title || str(v.meta.title) || ck.title;
      ch.versions.push(v);
      m.updated = Math.max(m.updated, v.added);
    }
    book.chapters = [...byKey.values()].sort(order);
    if (book.chapters.length) m.books.push(book);
  }
  m.books.sort(order);
  for (const b of m.books) {
    b.cover = b.cover || thumbOf(b.chapters[0].versions[0]);
    for (const ch of b.chapters) { m.chapters.push(ch); m.chapterById.set(ch.id, ch); }
  }
  m.cover = m.cover || (m.books[0] && m.books[0].cover) || null;
  return m;
}

// Returns { index, cache }. Pass the previous cache to skip re-reading unchanged chapter folders.
// options.concurrency overrides SCAN_CONCURRENCY (tests, benchmarks).
async function scanLibrary(root, oldCache = {}, options = {}) {
  const cache = {}, mangas = [], limits = scanLimits(options.concurrency);
  for (const name of (await list(root)).dirs) {
    try {
      const m = await scanManga(root, name, oldCache, cache, limits);
      if (m.chapters.length) mangas.push(m);
      else console.warn(`[scan] "${name}" has no chapters (expected <manga>/<book>/<chapter>/<images>)`);
    } catch (e) { console.warn(`[scan] skipped "${name}": ${e.message}`); }
  }
  mangas.sort((a, b) => cmp(a.title, b.title));
  return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache };
}

// Book-directory renames are the publisher's atomic commit point. A shallow
// stamp pass costs one stat per book, rather than one stat/JSON read per chapter.
// In-place chapter.json edits are picked up by the scheduled deep reconciliation.
async function fileStamp(file) {
  try { const s = await fsp.stat(file); return `${s.mtimeMs}:${s.size}`; }
  catch (e) { if (e.code === 'ENOENT') return ''; throw e; }
}
async function libraryStamps(root) {
  const stamps = {};
  for (const name of (await list(root)).dirs) {
    const mangaDir = path.join(root, name), books = {};
    const listed = await list(mangaDir);
    for (const bookName of listed.dirs.filter(parseKey)) {
      const bookDir = path.join(mangaDir, bookName);
      books[bookName] = [await fileStamp(bookDir), await fileStamp(path.join(bookDir, 'book.json'))];
    }
    stamps[name] = [await fileStamp(mangaDir), await fileStamp(path.join(mangaDir, 'manga.json')), books];
  }
  return stamps;
}

async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps, options = {}) {
  stamps = stamps || await libraryStamps(root);
  const limits = scanLimits(options.concurrency);
  const names = new Set([...Object.keys(oldStamps || {}), ...Object.keys(stamps)]);
  const changed = [...names].filter(name => JSON.stringify(oldStamps?.[name]) !== JSON.stringify(stamps[name]));
  if (!changed.length) return { index: prior, cache: oldCache, stamps, changed };
  const byId = new Map(prior.mangas.map(m => [m.id, m]));
  const cache = { ...oldCache };
  for (const name of changed) {
    byId.delete(name);
    for (const key of Object.keys(cache)) if (key.startsWith(name + '/')) delete cache[key];
    if (!Object.hasOwn(stamps, name)) continue;
    const m = await scanManga(root, name, oldCache, cache, limits);
    if (m.chapters.length) byId.set(name, m);
  }
  const mangas = [...byId.values()].sort((a, b) => cmp(a.title, b.title));
  return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache, stamps, changed };
}

// Store each chapter only once. Maps and book chapter references are rebuilt
// from the snapshot, which is a disposable cache rather than library authority.
function snapshotIndex(index) {
  return index.mangas.map(m => ({ ...m, chapterById: undefined,
    books: m.books.map(b => ({ ...b, chapters: b.chapters.map(ch => ch.id) })) }));
}
function restoreIndex(mangas) {
  if (!Array.isArray(mangas)) throw new Error('Invalid library index snapshot');
  const byId = new Map();
  for (const m of mangas) {
    if (!m || typeof m.id !== 'string' || !Array.isArray(m.books) || !Array.isArray(m.chapters) || byId.has(m.id))
      throw new Error('Invalid manga in library index snapshot');
    m.chapterById = new Map();
    for (const ch of m.chapters) {
      if (!ch || typeof ch.id !== 'string' || !Array.isArray(ch.versions) || ch.versions.some(v => !Array.isArray(v.pages)))
        throw new Error('Invalid chapter in library index snapshot');
      m.chapterById.set(ch.id, ch);
    }
    for (const book of m.books) {
      if (!Array.isArray(book.chapters) || book.chapters.some(id => !m.chapterById.has(id)))
        throw new Error('Invalid book in library index snapshot');
      book.chapters = book.chapters.map(id => m.chapterById.get(id));
    }
    byId.set(m.id, m);
  }
  return { mangas, byId };
}

module.exports = { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex,
  parseKey, sizeFromBuffer, imageSize, limiter };