236 lines
11 KiB
JavaScript
236 lines
11 KiB
JavaScript
'use strict';
|
||
/*
|
||
* Library scanner: turns <library>/<Manga>/<Book>/<Chapter>/<page images>
|
||
* into an in-memory index. The folder format is documented in README.md.
|
||
*
|
||
* Only folder names, small JSON files and the first bytes of each image (for its size) are read.
|
||
* Image sizes are cached per chapter folder and only re-read when the folder's mtime changes.
|
||
*/
|
||
const fsp = require('fs').promises, path = require('path');
|
||
const { parseSpecialFolder, parseKey } = require('./chapter-identity');
|
||
const { resolveReadingDirection } = require('./reading-direction');
|
||
|
||
const IMG = /\.(jpe?g|png|webp|gif|avif|bmp)$/i;
|
||
const COVER = /^cover\.(jpe?g|png|webp|gif|avif|bmp)$/i;
|
||
const cmp = new Intl.Collator('en', { numeric: true, sensitivity: 'base' }).compare;
|
||
const str = x => (typeof x === 'string' ? x.trim() : '');
|
||
const order = (a, b) => Number(!!a.special) - Number(!!b.special) || a.x - b.x || a.num - b.num || cmp(a.key, b.key);
|
||
|
||
async function readJson(file) {
|
||
try { return JSON.parse((await fsp.readFile(file, 'utf8')).replace(/^/, '')); }
|
||
catch (e) { if (e.code !== 'ENOENT') console.warn(`[scan] ignoring broken ${file}: ${e.message}`); return {}; }
|
||
}
|
||
|
||
async function list(dir) {
|
||
const dirs = [], files = [];
|
||
for (const e of await fsp.readdir(dir, { withFileTypes: true })) {
|
||
if (e.name.startsWith('.')) continue;
|
||
let isDir = e.isDirectory();
|
||
if (e.isSymbolicLink()) isDir = await fsp.stat(path.join(dir, e.name)).then(s => s.isDirectory(), () => false);
|
||
(isDir ? dirs : files).push(e.name);
|
||
}
|
||
return { dirs: dirs.sort(cmp), files: files.sort(cmp) };
|
||
}
|
||
|
||
// Width/height from the image header (PNG, GIF, BMP, WebP, JPEG, AVIF/HEIF).
|
||
function sizeFromBuffer(b) {
|
||
try {
|
||
if (b.readUInt32BE(0) === 0x89504e47) return { w: b.readUInt32BE(16), h: b.readUInt32BE(20) };
|
||
if (b.toString('latin1', 0, 3) === 'GIF') return { w: b.readUInt16LE(6), h: b.readUInt16LE(8) };
|
||
if (b.toString('latin1', 0, 2) === 'BM') return { w: b.readInt32LE(18), h: Math.abs(b.readInt32LE(22)) };
|
||
if (b.toString('latin1', 0, 4) === 'RIFF' && b.toString('latin1', 8, 12) === 'WEBP') {
|
||
const f = b.toString('latin1', 12, 16);
|
||
if (f === 'VP8 ') return { w: b.readUInt16LE(26) & 0x3fff, h: b.readUInt16LE(28) & 0x3fff };
|
||
if (f === 'VP8L') { const v = b.readUInt32LE(21); return { w: (v & 0x3fff) + 1, h: ((v >> 14) & 0x3fff) + 1 }; }
|
||
if (f === 'VP8X') return { w: b.readUIntLE(24, 3) + 1, h: b.readUIntLE(27, 3) + 1 };
|
||
}
|
||
if (b[0] === 0xff && b[1] === 0xd8) {
|
||
for (let i = 2; i + 9 < b.length;) {
|
||
if (b[i] !== 0xff) { i++; continue; }
|
||
const mk = b[i + 1];
|
||
if (mk === 0xff) { i++; continue; }
|
||
if (mk === 0x01 || (mk >= 0xd0 && mk <= 0xd8)) { i += 2; continue; }
|
||
if (mk >= 0xc0 && mk <= 0xcf && mk !== 0xc4 && mk !== 0xc8 && mk !== 0xcc) return { w: b.readUInt16BE(i + 7), h: b.readUInt16BE(i + 5) };
|
||
i += 2 + b.readUInt16BE(i + 2);
|
||
}
|
||
}
|
||
const k = b.indexOf('ispe');
|
||
if (k > 0 && b.toString('latin1', 4, 8) === 'ftyp') return { w: b.readUInt32BE(k + 8), h: b.readUInt32BE(k + 12) };
|
||
} catch { /* truncated or unknown header */ }
|
||
return null;
|
||
}
|
||
|
||
async function imageSize(file) {
|
||
const fh = await fsp.open(file, 'r');
|
||
let head;
|
||
try {
|
||
const buf = Buffer.alloc(256 * 1024);
|
||
const { bytesRead } = await fh.read(buf, 0, buf.length, 0);
|
||
head = buf.subarray(0, bytesRead);
|
||
} finally { await fh.close(); }
|
||
// JPEGs with big EXIF/ICC blocks can have their size marker further in.
|
||
return sizeFromBuffer(head) || (head.length === 256 * 1024 ? sizeFromBuffer(await fsp.readFile(file)) : null);
|
||
}
|
||
|
||
function pageInfo([f, w, h]) {
|
||
const t = f.slice(0, f.lastIndexOf('.')).toLowerCase().split('_').slice(1);
|
||
const rot = t.find(x => /^rot(90|180|270)$/.test(x));
|
||
return { f, w, h, t, r: rot ? +rot.slice(3) : 0 };
|
||
}
|
||
|
||
async function scanVersion(root, rel, ck, oldCache, cache) {
|
||
const dir = path.join(root, rel), st = await fsp.stat(dir);
|
||
const meta = await readJson(path.join(dir, 'chapter.json'));
|
||
let c = oldCache[rel];
|
||
if (!c || c.m !== st.mtimeMs) {
|
||
const p = [];
|
||
for (const f of (await list(dir)).files) {
|
||
if (!IMG.test(f)) continue;
|
||
const s = await imageSize(path.join(dir, f)).catch(() => null);
|
||
p.push([f, s ? s.w : 0, s ? s.h : 0]);
|
||
}
|
||
c = { m: st.mtimeMs, p };
|
||
}
|
||
cache[rel] = c;
|
||
const src = meta.source && typeof meta.source === 'object' ? meta.source : { name: meta.source };
|
||
return {
|
||
id: rel.slice(rel.lastIndexOf('/') + 1), rel, meta,
|
||
source: str(src.name) || ck.src, sourceUrl: str(src.url),
|
||
added: Date.parse(meta.added) || st.mtimeMs,
|
||
pages: c.p.map(pageInfo),
|
||
};
|
||
}
|
||
|
||
const thumbOf = v => { const p = v.pages.find(p => p.t.includes('cover')) || v.pages[0]; return v.rel + '/' + p.f; };
|
||
const coverIn = (files, rel) => { const f = files.find(f => COVER.test(f)); return f ? rel + '/' + f : null; };
|
||
|
||
async function scanManga(root, name, oldCache, cache) {
|
||
const { dirs, files } = await list(path.join(root, name));
|
||
const meta = await readJson(path.join(root, name, 'manga.json'));
|
||
const m = { id: name, title: str(meta.title) || name, meta, direction: resolveReadingDirection([meta]).value, cover: coverIn(files, name), books: [], chapters: [], chapterById: new Map(), updated: 0 };
|
||
const bookKeys = new Set();
|
||
for (const bName of dirs) {
|
||
const bk = parseKey(bName);
|
||
if (!bk) { console.warn(`[scan] ignoring "${name}/${bName}": book folders must be named <letters><number>[ title], e.g. "v01 Spring"`); continue; }
|
||
if (bookKeys.has(bk.key)) { console.warn(`[scan] ignoring "${name}/${bName}": another book folder already uses number ${bk.key}`); continue; }
|
||
bookKeys.add(bk.key);
|
||
const bRel = name + '/' + bName, bl = await list(path.join(root, bRel));
|
||
const bMeta = await readJson(path.join(root, bRel, 'book.json'));
|
||
const book = {
|
||
key: bk.key, num: bk.num, x: bk.x, extra: bk.x || bMeta.extra === true,
|
||
label: (bk.x ? 'Extra book ' : 'Book ') + bk.num, title: str(bMeta.title) || bk.title, cover: coverIn(bl.files, bRel), chapters: [],
|
||
};
|
||
const byKey = new Map();
|
||
for (const cName of bl.dirs) {
|
||
const ck = parseSpecialFolder(cName) || parseKey(cName);
|
||
if (!ck) { console.warn(`[scan] ignoring "${bRel}/${cName}": chapter folders must be named <letters><number>[ title], e.g. "c0001 The Seed"`); continue; }
|
||
const v = await scanVersion(root, bRel + '/' + cName, ck, oldCache, cache);
|
||
if (!v.pages.length) continue;
|
||
v.directionInfo = resolveReadingDirection([v.meta, bMeta, meta], v.pages);
|
||
v.direction = v.directionInfo.value;
|
||
let ch = byKey.get(ck.key);
|
||
if (!ch) {
|
||
ch = { id: book.key + '/' + ck.key, book: book.key, key: ck.key, num: ck.num, x: ck.x, special: !!ck.special, extra: book.extra || ck.x, label: ck.special ? str(v.meta.label) || 'Special' : (ck.x ? 'Extra ' : 'Ch. ') + ck.num, title: '', versions: [] };
|
||
byKey.set(ck.key, ch);
|
||
}
|
||
if (v.meta.extra === true) ch.extra = true;
|
||
ch.title = ch.title || str(v.meta.title) || ck.title;
|
||
ch.versions.push(v);
|
||
m.updated = Math.max(m.updated, v.added);
|
||
}
|
||
book.chapters = [...byKey.values()].sort(order);
|
||
if (book.chapters.length) m.books.push(book);
|
||
}
|
||
m.books.sort(order);
|
||
for (const b of m.books) {
|
||
b.cover = b.cover || thumbOf(b.chapters[0].versions[0]);
|
||
for (const ch of b.chapters) { m.chapters.push(ch); m.chapterById.set(ch.id, ch); }
|
||
}
|
||
m.cover = m.cover || (m.books[0] && m.books[0].cover) || null;
|
||
return m;
|
||
}
|
||
|
||
// Returns { index, cache }. Pass the previous cache to skip re-reading unchanged chapter folders.
|
||
async function scanLibrary(root, oldCache = {}) {
|
||
const cache = {}, mangas = [];
|
||
for (const name of (await list(root)).dirs) {
|
||
try {
|
||
const m = await scanManga(root, name, oldCache, cache);
|
||
if (m.chapters.length) mangas.push(m);
|
||
else console.warn(`[scan] "${name}" has no chapters (expected <manga>/<book>/<chapter>/<images>)`);
|
||
} catch (e) { console.warn(`[scan] skipped "${name}": ${e.message}`); }
|
||
}
|
||
mangas.sort((a, b) => cmp(a.title, b.title));
|
||
return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache };
|
||
}
|
||
|
||
// Book-directory renames are the publisher's atomic commit point. A shallow
|
||
// stamp pass costs one stat per book, rather than one stat/JSON read per chapter.
|
||
// In-place chapter.json edits are picked up by the scheduled deep reconciliation.
|
||
async function fileStamp(file) {
|
||
try { const s = await fsp.stat(file); return `${s.mtimeMs}:${s.size}`; }
|
||
catch (e) { if (e.code === 'ENOENT') return ''; throw e; }
|
||
}
|
||
async function libraryStamps(root) {
|
||
const stamps = {};
|
||
for (const name of (await list(root)).dirs) {
|
||
const mangaDir = path.join(root, name), books = {};
|
||
const listed = await list(mangaDir);
|
||
for (const bookName of listed.dirs.filter(parseKey)) {
|
||
const bookDir = path.join(mangaDir, bookName);
|
||
books[bookName] = [await fileStamp(bookDir), await fileStamp(path.join(bookDir, 'book.json'))];
|
||
}
|
||
stamps[name] = [await fileStamp(mangaDir), await fileStamp(path.join(mangaDir, 'manga.json')), books];
|
||
}
|
||
return stamps;
|
||
}
|
||
|
||
async function scanChangedLibrary(root, prior, oldCache, oldStamps, stamps) {
|
||
stamps = stamps || await libraryStamps(root);
|
||
const names = new Set([...Object.keys(oldStamps || {}), ...Object.keys(stamps)]);
|
||
const changed = [...names].filter(name => JSON.stringify(oldStamps?.[name]) !== JSON.stringify(stamps[name]));
|
||
if (!changed.length) return { index: prior, cache: oldCache, stamps, changed };
|
||
const byId = new Map(prior.mangas.map(m => [m.id, m]));
|
||
const cache = { ...oldCache };
|
||
for (const name of changed) {
|
||
byId.delete(name);
|
||
for (const key of Object.keys(cache)) if (key.startsWith(name + '/')) delete cache[key];
|
||
if (!Object.hasOwn(stamps, name)) continue;
|
||
const m = await scanManga(root, name, oldCache, cache);
|
||
if (m.chapters.length) byId.set(name, m);
|
||
}
|
||
const mangas = [...byId.values()].sort((a, b) => cmp(a.title, b.title));
|
||
return { index: { mangas, byId: new Map(mangas.map(m => [m.id, m])) }, cache, stamps, changed };
|
||
}
|
||
|
||
// Store each chapter only once. Maps and book chapter references are rebuilt
|
||
// from the snapshot, which is a disposable cache rather than library authority.
|
||
function snapshotIndex(index) {
|
||
return index.mangas.map(m => ({ ...m, chapterById: undefined,
|
||
books: m.books.map(b => ({ ...b, chapters: b.chapters.map(ch => ch.id) })) }));
|
||
}
|
||
function restoreIndex(mangas) {
|
||
if (!Array.isArray(mangas)) throw new Error('Invalid library index snapshot');
|
||
const byId = new Map();
|
||
for (const m of mangas) {
|
||
if (!m || typeof m.id !== 'string' || !Array.isArray(m.books) || !Array.isArray(m.chapters) || byId.has(m.id))
|
||
throw new Error('Invalid manga in library index snapshot');
|
||
m.chapterById = new Map();
|
||
for (const ch of m.chapters) {
|
||
if (!ch || typeof ch.id !== 'string' || !Array.isArray(ch.versions) || ch.versions.some(v => !Array.isArray(v.pages)))
|
||
throw new Error('Invalid chapter in library index snapshot');
|
||
m.chapterById.set(ch.id, ch);
|
||
}
|
||
for (const book of m.books) {
|
||
if (!Array.isArray(book.chapters) || book.chapters.some(id => !m.chapterById.has(id)))
|
||
throw new Error('Invalid book in library index snapshot');
|
||
book.chapters = book.chapters.map(id => m.chapterById.get(id));
|
||
}
|
||
byId.set(m.id, m);
|
||
}
|
||
return { mangas, byId };
|
||
}
|
||
|
||
module.exports = { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex,
|
||
parseKey, sizeFromBuffer, imageSize };
|