'use strict'; /* * Library scanner: turns //// * into an in-memory index. The folder format is documented in README.md. * * Only folder names, small JSON files and the first bytes of each image (for its size) are read. * Image sizes are cached per chapter folder and only re-read when the folder's mtime changes. */ const fsp = require('fs').promises, path = require('path'); const { parseSpecialFolder, parseKey } = require('./chapter-identity'); const { resolveReadingDirection } = require('./reading-direction'); const IMG = /\.(jpe?g|png|webp|gif|avif|bmp)$/i; const COVER = /^cover\.(jpe?g|png|webp|gif|avif|bmp)$/i; const cmp = new Intl.Collator('en', { numeric: true, sensitivity: 'base' }).compare; const str = x => (typeof x === 'string' ? x.trim() : ''); const order = (a, b) => Number(!!a.special) - Number(!!b.special) || a.x - b.x || a.num - b.num || cmp(a.key, b.key); async function readJson(file) { try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^/, '')); } catch (e) { if (e.code !== 'ENOENT') console.warn(`[scan] ignoring broken ${file}: ${e.message}`); return {}; } } async function list(dir) { const dirs = [], files = []; for (const e of await fsp.readdir(dir, { withFileTypes: true })) { if (e.name.startsWith('.')) continue; let isDir = e.isDirectory(); if (e.isSymbolicLink()) isDir = await fsp.stat(path.join(dir, e.name)).then(s => s.isDirectory(), () => false); (isDir ? dirs : files).push(e.name); } return { dirs: dirs.sort(cmp), files: files.sort(cmp) }; } // Width/height from the image header (PNG, GIF, BMP, WebP, JPEG, AVIF/HEIF). function sizeFromBuffer(b) { try { if (b.readUInt32BE(0) === 0x89504e47) return { w: b.readUInt32BE(16), h: b.readUInt32BE(20) }; if (b.toString('latin1', 0, 3) === 'GIF') return { w: b.readUInt16LE(6), h: b.readUInt16LE(8) }; if (b.toString('latin1', 0, 2) === 'BM') return { w: b.readInt32LE(18), h: Math.abs(b.readInt32LE(22)) }; if (b.toString('latin1', 0, 4) === 'RIFF' && b.toString('latin1', 8, 12) === 'WEBP') { const f = b.toString('latin1', 12, 16); if (f === 'VP8 ') return { w: b.readUInt16LE(26) & 0x3fff, h: b.readUInt16LE(28) & 0x3fff }; if (f === 'VP8L') { const v = b.readUInt32LE(21); return { w: (v & 0x3fff) + 1, h: ((v >> 14) & 0x3fff) + 1 }; } if (f === 'VP8X') return { w: b.readUIntLE(24, 3) + 1, h: b.readUIntLE(27, 3) + 1 }; } if (b[0] === 0xff && b[1] === 0xd8) { for (let i = 2; i + 9 < b.length;) { if (b[i] !== 0xff) { i++; continue; } const mk = b[i + 1]; if (mk === 0xff) { i++; continue; } if (mk === 0x01 || (mk >= 0xd0 && mk <= 0xd8)) { i += 2; continue; } if (mk >= 0xc0 && mk <= 0xcf && mk !== 0xc4 && mk !== 0xc8 && mk !== 0xcc) return { w: b.readUInt16BE(i + 7), h: b.readUInt16BE(i + 5) }; i += 2 + b.readUInt16BE(i + 2); } } const k = b.indexOf('ispe'); if (k > 0 && b.toString('latin1', 4, 8) === 'ftyp') return { w: b.readUInt32BE(k + 8), h: b.readUInt32BE(k + 12) }; } catch { /* truncated or unknown header */ } return null; } async function imageSize(file) { const fh = await fsp.open(file, 'r'); let head; try { const buf = Buffer.alloc(256 * 1024); const { bytesRead } = await fh.read(buf, 0, buf.length, 0); head = buf.subarray(0, bytesRead); } finally { await fh.close(); } // JPEGs with big EXIF/ICC blocks can have their size marker further in. return sizeFromBuffer(head) || (head.length === 256 * 1024 ? sizeFromBuffer(await fsp.readFile(file)) : null); } function pageInfo([f, w, h]) { const t = f.slice(0, f.lastIndexOf('.')).toLowerCase().split('_').slice(1); const rot = t.find(x => /^rot(90|180|270)$/.test(x)); return { f, w, h, t, r: rot ? +rot.slice(3) : 0 }; } async function scanVersion(root, rel, ck, oldCache, cache) { const dir = path.join(root, rel), st = await fsp.stat(dir); const meta = await readJson(path.join(dir, 'chapter.json')); let c = oldCache[rel]; if (!c || c.m !== st.mtimeMs) { const p = []; for (const f of (await list(dir)).files) { if (!IMG.test(f)) continue; const s = await imageSize(path.join(dir, f)).catch(() => null); p.push([f, s ? s.w : 0, s ? s.h : 0]); } c = { m: st.mtimeMs, p }; } cache[rel] = c; const src = meta.source && typeof meta.source === 'object' ? meta.source : { name: meta.source }; return { id: rel.slice(rel.lastIndexOf('/') + 1), rel, meta, source: str(src.name) || ck.src, sourceUrl: str(src.url), added: Date.parse(meta.added) || st.mtimeMs, pages: c.p.map(pageInfo), }; } const thumbOf = v => { const p = v.pages.find(p => p.t.includes('cover')) || v.pages[0]; return v.rel + '/' + p.f; }; const coverIn = (files, rel) => { const f = files.find(f => COVER.test(f)); return f ? rel + '/' + f : null; }; async function scanManga(root, name, oldCache, cache) { const { dirs, files } = await list(path.join(root, name)); const meta = await readJson(path.join(root, name, 'manga.json')); const m = { id: name, title: str(meta.title) || name, meta, direction: resolveReadingDirection([meta]).value, cover: coverIn(files, name), books: [], chapters: [], chapterById: new Map(), updated: 0 }; const bookKeys = new Set(); for (const bName of dirs) { const bk = parseKey(bName); if (!bk) { console.warn(`[scan] ignoring "${name}/${bName}": book folders must be named [ title], e.g. "v01 Spring"`); continue; } if (bookKeys.has(bk.key)) { console.warn(`[scan] ignoring "${name}/${bName}": another book folder already uses number ${bk.key}`); continue; } bookKeys.add(bk.key); const bRel = name + '/' + bName, bl = await list(path.join(root, bRel)); const bMeta = await readJson(path.join(root, bRel, 'book.json')); const book = { key: bk.key, num: bk.num, x: bk.x, extra: bk.x || bMeta.extra === true, label: (bk.x ? 'Extra book ' : 'Book ') + bk.num, title: str(bMeta.title) || bk.title, cover: coverIn(bl.files, bRel), chapters: [], }; const byKey = new Map(); for (const cName of bl.dirs) { const ck = parseSpecialFolder(cName) || parseKey(cName); if (!ck) { console.warn(`[scan] ignoring "${bRel}/${cName}": chapter folders must be named [ title], e.g. "c0001 The Seed"`); continue; } const v = await scanVersion(root, bRel + '/' + cName, ck, oldCache, cache); if (!v.pages.length) continue; v.directionInfo = resolveReadingDirection([v.meta, bMeta, meta], v.pages); v.direction = v.directionInfo.value; let ch = byKey.get(ck.key); if (!ch) { ch = { id: book.key + '/' + ck.key, book: book.key, key: ck.key, num: ck.num, x: ck.x, special: !!ck.special, extra: book.extra || ck.x, label: ck.special ? str(v.meta.label) || 'Special' : (ck.x ? 'Extra ' : 'Ch. ') + ck.num, title: '', versions: [] }; byKey.set(ck.key, ch); } if (v.meta.extra === true) ch.extra = true; ch.title = ch.title || str(v.meta.title) || ck.title; ch.versions.push(v); m.updated = Math.max(m.updated, v.added); } book.chapters = [...byKey.values()].sort(order); if (book.chapters.length) m.books.push(book); } m.books.sort(order); for (const b of m.books) { b.cover = b.cover || thumbOf(b.chapters[0].versions[0]); for (const ch of b.chapters) { m.chapters.push(ch); m.chapterById.set(ch.id, ch); } } m.cover = m.cover || (m.books[0] && m.books[0].cover) || null; return m; } // Returns { index, cache }. Pass the previous cache to skip re-reading unchanged chapter folders. async function scanLibrary(root, oldCache = {}) { const cache = {}, mangas = []; for (const name of (await list(root)).dirs) { try { const m = await scanManga(root, name, oldCache, cache); if (m.chapters.length) mangas.push(m); else console.warn(`[scan] "${name}" has no chapters (expected ///)`); } catch (e) { console.warn(`[scan] skipped "${name}": ${e.message}`); } } mangas.sort((a, b) => cmp(a.title, b.title)); return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache }; } // Book-directory renames are the publisher's atomic commit point. A shallow // stamp pass costs one stat per book, rather than one stat/JSON read per chapter. // In-place chapter.json edits are picked up by the scheduled deep reconciliation. async function fileStamp(file) { try { const s = await fsp.stat(file); return `${s.mtimeMs}:${s.size}`; } catch (e) { if (e.code === 'ENOENT') return ''; throw e; } } async function libraryStamps(root) { const stamps = {}; for (const name of (await list(root)).dirs) { const mangaDir = path.join(root, name), books = {}; const listed = await list(mangaDir); for (const bookName of listed.dirs.filter(parseKey)) { const bookDir = path.join(mangaDir, bookName); books[bookName] = [await fileStamp(bookDir), await fileStamp(path.join(bookDir, 'book.json'))]; } stamps[name] = [await fileStamp(mangaDir), await fileStamp(path.join(mangaDir, 'manga.json')), books]; } return stamps; } async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps) { stamps = stamps || await libraryStamps(root); const names = new Set([...Object.keys(oldStamps || {}), ...Object.keys(stamps)]); const changed = [...names].filter(name => JSON.stringify(oldStamps?.[name]) !== JSON.stringify(stamps[name])); if (!changed.length) return { index: prior, cache: oldCache, stamps, changed }; const byId = new Map(prior.mangas.map(m => [m.id, m])); const cache = { ...oldCache }; for (const name of changed) { byId.delete(name); for (const key of Object.keys(cache)) if (key.startsWith(name + '/')) delete cache[key]; if (!Object.hasOwn(stamps, name)) continue; const m = await scanManga(root, name, oldCache, cache); if (m.chapters.length) byId.set(name, m); } const mangas = [...byId.values()].sort((a, b) => cmp(a.title, b.title)); return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache, stamps, changed }; } // Store each chapter only once. Maps and book chapter references are rebuilt // from the snapshot, which is a disposable cache rather than library authority. function snapshotIndex(index) { return index.mangas.map(m => ({ ...m, chapterById: undefined, books: m.books.map(b => ({ ...b, chapters: b.chapters.map(ch => ch.id) })) })); } function restoreIndex(mangas) { if (!Array.isArray(mangas)) throw new Error('Invalid library index snapshot'); const byId = new Map(); for (const m of mangas) { if (!m || typeof m.id !== 'string' || !Array.isArray(m.books) || !Array.isArray(m.chapters) || byId.has(m.id)) throw new Error('Invalid manga in library index snapshot'); m.chapterById = new Map(); for (const ch of m.chapters) { if (!ch || typeof ch.id !== 'string' || !Array.isArray(ch.versions) || ch.versions.some(v => !Array.isArray(v.pages))) throw new Error('Invalid chapter in library index snapshot'); m.chapterById.set(ch.id, ch); } for (const book of m.books) { if (!Array.isArray(book.chapters) || book.chapters.some(id => !m.chapterById.has(id))) throw new Error('Invalid book in library index snapshot'); book.chapters = book.chapters.map(id => m.chapterById.get(id)); } byId.set(m.id, m); } return { mangas, byId }; } module.exports = { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex, parseKey, sizeFromBuffer, imageSize };