161 lines
10 KiB
JavaScript
161 lines
10 KiB
JavaScript
'use strict';
|
|
|
|
const fs = require('node:fs').promises;
|
|
const path = require('node:path');
|
|
const crypto = require('node:crypto');
|
|
const { PeeringState, requirePeer } = require('./state');
|
|
const client = require('./client');
|
|
const { ArchiveControlError } = require('../scraper/lib/control');
|
|
const { selectByScope } = require('../scraper/lib/scope');
|
|
const { scanLibrary, findManga, shouldDownload } = require('../scraper/lib/library-inventory');
|
|
const { publishManifest } = require('../scraper/lib/publisher');
|
|
const { writeJsonAtomic } = require('../scraper/lib/json-file');
|
|
const { sleep } = require('../scraper/lib/delay');
|
|
const { normalizeWorkIdentity, sameWorkIdentity, resolveWorkIdentity } = require('../scraper/lib/work-identity');
|
|
|
|
const dataRoot = () => path.resolve(process.env.PEER_DATA || path.join(process.env.DATA || 'data', 'peering'));
|
|
const store = () => new PeeringState(dataRoot());
|
|
const infoFor = value => {
|
|
try {
|
|
const u = new URL(String(value));
|
|
if (u.protocol !== 'peer:' || u.username || u.password || u.search || u.hash || !/^p[0-9a-f]{24}$/.test(u.hostname)
|
|
|| !/^\/m[0-9a-f]{24}$/.test(u.pathname)) return null;
|
|
const peer = requirePeer(store().load(), u.hostname);
|
|
return { source: `peer-${peer.id}`, id: `${peer.id}-${u.pathname.slice(1)}`, peerId: peer.id,
|
|
shareId: u.pathname.slice(1), title: peer.alias, url: `peer://${peer.id}${u.pathname}` };
|
|
} catch { return null; }
|
|
};
|
|
function refsOf(result) {
|
|
return result.chapters.map(ref => ({ id: ref.id, number: ref.number, volume: ref.volume,
|
|
book: { number: Number(ref.volume) || 1 }, title: ref.title, pages: ref.pageCount }));
|
|
}
|
|
function sameTitle(left, right) {
|
|
return String(left || '').normalize('NFKC').trim().replace(/\s+/g, ' ').toLowerCase()
|
|
=== String(right || '').normalize('NFKC').trim().replace(/\s+/g, ' ').toLowerCase();
|
|
}
|
|
async function pageHashes(folder) {
|
|
const names = (await fs.readdir(folder, { withFileTypes: true }))
|
|
.filter(entry => entry.isFile() && /\.(?:jpe?g|png|webp|gif|avif|bmp)$/i.test(entry.name))
|
|
.map(entry => entry.name).sort();
|
|
if (!names.length) return [];
|
|
return Promise.all(names.map(async name => crypto.createHash('sha256')
|
|
.update(await fs.readFile(path.join(folder, name))).digest('hex')));
|
|
}
|
|
async function matchLegacyPeerCopy(inventory, remote, peerStore, info) {
|
|
// Pre-work-ID peer imports can be adopted only with byte-for-byte evidence;
|
|
// a matching title alone is never sufficient to merge library works.
|
|
const candidates = inventory.mangas.filter(manga => sameTitle(manga.title, remote.title));
|
|
if (candidates.length !== 1) return null;
|
|
const manga = candidates[0];
|
|
for (const ref of remote.chapters) {
|
|
const book = String(Number(ref.volume) || Number(ref.book?.number) || 1);
|
|
const local = manga.chapters.filter(chapter => Number(chapter.book) === Number(book)
|
|
&& String(chapter.number) === String(ref.number));
|
|
if (!local.length) continue;
|
|
const manifest = await client.chapter(peerStore, info.peerId, info.shareId, ref.id);
|
|
const expected = manifest.pages.map(page => page.sha256);
|
|
for (const chapter of local) {
|
|
const actual = await pageHashes(path.join(manga.path, chapter.bookFolder, chapter.version));
|
|
if (actual.length === expected.length && actual.every((hash, index) => hash === expected[index])) return manga;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
async function discover(info) {
|
|
const result = await client.manga(store(), info.peerId, info.shareId);
|
|
return { title: result.title, workIdentity: normalizeWorkIdentity(result.workIdentity), chapters: refsOf(result), availability: { selectedChapters: result.chapters.length, coverage: 'shared-selection' } };
|
|
}
|
|
async function run(info, item, options) {
|
|
const stateStore = store(), peer = requirePeer(stateStore.load(), info.peerId);
|
|
if (!peer.paired || !peer.receive) throw new Error('Downloads from this peer are disabled');
|
|
const stop = () => { const reason = options.shouldStop?.(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); };
|
|
stop();
|
|
const remote = await client.manga(stateStore, info.peerId, info.shareId);
|
|
const workIdentity = normalizeWorkIdentity(remote.workIdentity);
|
|
const queuedIdentity = normalizeWorkIdentity(item.workIdentity);
|
|
if (queuedIdentity && !sameWorkIdentity(queuedIdentity, workIdentity)) throw new Error('Shared manga identity changed after it was queued; review the title before syncing');
|
|
const catalog = { title: remote.title, workIdentity, chapters: refsOf(remote), availability: { selectedChapters: remote.chapters.length, coverage: 'shared-selection' } };
|
|
if (options.onCatalog) await options.onCatalog(catalog);
|
|
const sourceUrl = info.url, source = info.source;
|
|
const workRoot = path.join(options.libraryRoot, `.mcscraper-work-${info.id}`);
|
|
const stateFile = path.join(workRoot, 'job.json');
|
|
await fs.mkdir(workRoot, { recursive: true });
|
|
let job = { version: 1, publicationMode: 'incremental', source, seriesUrl: sourceUrl, completed: [], jobCompleted: 0,
|
|
status: 'running', started: new Date().toISOString(), updated: new Date().toISOString(), total: 0, current: null };
|
|
try { const prior = JSON.parse(await fs.readFile(stateFile, 'utf8')); if (prior.seriesUrl === sourceUrl) job = { ...prior, status: 'running' }; }
|
|
catch (e) { if (e.code !== 'ENOENT') throw e; }
|
|
const record = async () => { job.updated = new Date().toISOString(); await writeJsonAtomic(stateFile, job); };
|
|
const inventory = await scanLibrary(options.libraryRoot);
|
|
let manga = findManga(inventory, { source, seriesId: info.id, sourceUrl, workIdentity,
|
|
localServerId: stateStore.identity()?.id, libraryMangaId: item.libraryMangaId });
|
|
if (!manga && (!workIdentity || workIdentity.kind === 'origin')) {
|
|
manga = await matchLegacyPeerCopy(inventory, remote, stateStore, info);
|
|
}
|
|
if (!manga && inventory.mangas.some(candidate => sameTitle(candidate.title, remote.title))) {
|
|
throw new Error(`A manga titled “${remote.title}” already exists, but neither its canonical identity nor shared chapter page hashes verify that it is the same work. No files were downloaded or changed; review the identity before syncing.`);
|
|
}
|
|
if (item.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${item.libraryMangaId}`);
|
|
const localIdentity = manga ? resolveWorkIdentity(manga.meta, manga.id, stateStore.identity()?.id) : null;
|
|
if (manga && localIdentity && !normalizeWorkIdentity(manga.meta['x-work-identity'])) {
|
|
manga.meta = { ...manga.meta, 'x-work-identity': localIdentity };
|
|
await writeJsonAtomic(path.join(manga.path, 'manga.json'), manga.meta);
|
|
}
|
|
const selected = selectByScope(catalog.chapters.filter(ref => shouldDownload(manga, ref, source, item.existingPolicy)), item.scope);
|
|
job.total = selected.length; job.jobCompleted = 0; job.current = null; await record();
|
|
let publishedCount = 0;
|
|
const identityToPersist = localIdentity || workIdentity;
|
|
const series = { title: remote.title, direction: ['rtl', 'ltr', 'ttb'].includes(remote.direction) ? remote.direction : 'rtl',
|
|
sources: [{ name: `Peer ${peer.alias}`, url: sourceUrl }],
|
|
metadata: { title: remote.title, ...(identityToPersist ? { 'x-work-identity': identityToPersist } : {}) },
|
|
scraper: { ids: { [source]: info.id } } };
|
|
for (let index = 0; index < selected.length; index++) {
|
|
stop();
|
|
const ref = selected[index];
|
|
if (!shouldDownload(manga, ref, source, item.existingPolicy)) { job.jobCompleted++; await record(); continue; }
|
|
const chapter = await client.chapter(stateStore, info.peerId, info.shareId, ref.id);
|
|
if (String(chapter.number) !== String(ref.number) || String(chapter.volume) !== String(ref.volume)) throw new Error('Peer chapter identity changed');
|
|
const digest = crypto.createHash('sha256').update(JSON.stringify(chapter.pages.map(p => [p.id, p.sha256]))).digest('hex').slice(0, 20);
|
|
const stage = path.join(workRoot, 'pages', `${ref.id}-${digest}`);
|
|
await fs.mkdir(stage, { recursive: true });
|
|
job.current = { number: ref.number, sourceId: ref.id, position: index + 1, total: selected.length,
|
|
downloadedPages: 0, expectedPages: chapter.pages.length };
|
|
await record();
|
|
const pages = chapter.pages.map((page, i) => ({ ...page, path: path.join(stage, `${String(i + 1).padStart(4, '0')}.${page.mime.split('/')[1] === 'jpeg' ? 'jpg' : page.mime.split('/')[1]}`) }));
|
|
const concurrency = Math.min(4, Math.max(1, Number(requirePeer(stateStore.load(), info.peerId).download?.concurrent || 2)));
|
|
for (let offset = 0; offset < pages.length; offset += concurrency) {
|
|
stop();
|
|
const batch = pages.slice(offset, offset + concurrency);
|
|
await Promise.all(batch.map(page => client.downloadPage(stateStore, info.peerId, page, page.path)));
|
|
job.current.downloadedPages = Math.min(pages.length, offset + batch.length);
|
|
await record();
|
|
stop();
|
|
}
|
|
const publication = await publishManifest({ version: 1, series, chapters: [{
|
|
number: ref.number, title: ref.title, book: { number: Number(ref.volume) || 1 },
|
|
source: { name: `Peer ${peer.alias} ${peer.id.slice(1, 7)}`, url: sourceUrl },
|
|
pages: pages.map(page => ({ path: page.path })),
|
|
}] }, { libraryRoot: options.libraryRoot, ...(manga ? { mangaDir: manga.path } : {}) });
|
|
if (publication.results[0].status === 'published') {
|
|
publishedCount++;
|
|
if (!manga) manga = { id: path.basename(publication.mangaDir), path: publication.mangaDir, chapters: [], meta: {} };
|
|
manga.chapters.push({ book: String(Number(ref.volume) || 1), number: String(ref.number), sourceKey: source });
|
|
if (options.onPublished) await options.onPublished({ mangaDir: publication.mangaDir, chapter: ref.number });
|
|
}
|
|
job.jobCompleted++; job.lastChapter = ref.number; job.current = null; await record();
|
|
// Only a fully verified and published chapter loses its resumable page cache.
|
|
await fs.rm(stage, { recursive: true, force: true });
|
|
if (index < selected.length - 1) { stop(); await sleep(Math.max(500, Number(options.settings.chapterMinMs || 500))); }
|
|
}
|
|
job.status = 'complete'; job.current = null; await record();
|
|
return { finalPath: manga?.path || null, publishedCount, selectedCount: selected.length };
|
|
}
|
|
function adapterForPeer(value) {
|
|
const info = infoFor(value);
|
|
if (!info) return null;
|
|
const adapter = { id: info.source, label: `Peer ${info.peerId.slice(1, 7)}`,
|
|
async discover() { return discover(info); },
|
|
async run(item, options) { return run(info, item, options); } };
|
|
return { adapter, info };
|
|
}
|
|
module.exports = { adapterForPeer, infoFor, discover, run };
|