diff --git a/chapter-identity.js b/chapter-identity.js index 3ae0272..760be16 100644 --- a/chapter-identity.js +++ b/chapter-identity.js @@ -3,21 +3,27 @@ // An unnumbered release is not chapter zero. Keep its upstream identity intact. const UUID = '[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}'; const SPECIAL = new RegExp(`^special-mangadex-(${UUID})$`, 'i'); -const SPECIAL_FOLDER = new RegExp(`^(special-mangadex-${UUID})(?: (.*))?$`, 'i'); +const VOLUME = /^special-weebcentral-volume-([a-z0-9]{26})$/i; +const SPECIAL_FOLDER = new RegExp(`^(special-mangadex-${UUID}|special-weebcentral-volume-[a-z0-9]{26})(?: (.*))?$`, 'i'); const NUMBERED_FOLDER = /^([a-z]*) ?(\d+(?:\.\d+)?)(?=$|[\s_\-[(])[\s_-]*(.*)$/i; -const isSpecialKey = value => SPECIAL.test(String(value)); +const isSpecialKey = value => SPECIAL.test(String(value)) || VOLUME.test(String(value)); function specialKey(id) { const key = `special-mangadex-${String(id).toLowerCase()}`; if (!isSpecialKey(key)) throw new Error('Unnumbered MangaDex release needs a stable chapter UUID'); return key; } +function volumeKey(id) { + const key = `special-weebcentral-volume-${String(id).toLowerCase()}`; + if (!VOLUME.test(key)) throw new Error('WeebCentral volume needs a stable 26-character release ID'); + return key; +} function parseSpecialFolder(name) { const match = SPECIAL_FOLDER.exec(name); if (!match) return null; let title = (match[2] || '').trim(), src = ''; const source = /\s*\[([^\]]*)\]$/.exec(title); if (source) { src = source[1].trim(); title = title.slice(0, source.index).trim(); } - return { key: match[1].toLowerCase(), special: true, x: false, num: null, title, src }; + return { key: match[1].toLowerCase(), special: true, releaseKind: VOLUME.test(match[1]) ? 'volume' : 'special', x: false, num: null, title, src }; } // Reader and scraper must assign the same identity to every numbered folder. function parseKey(name) { @@ -29,4 +35,4 @@ function parseKey(name) { if (source) { src = source[1].trim(); title = title.slice(0, source.index).trim(); } return { key: (x ? 'x' : '') + num, x, num, title: title.replace(/^[-:.]\s*/, ''), src }; } -module.exports = { isSpecialKey, specialKey, parseSpecialFolder, parseKey }; +module.exports = { isSpecialKey, specialKey, volumeKey, parseSpecialFolder, parseKey }; diff --git a/docs/library-format.md b/docs/library-format.md index 7e3a041..7f097dd 100644 --- a/docs/library-format.md +++ b/docs/library-format.md @@ -82,6 +82,14 @@ Different source UUIDs remain distinct releases; no content equivalence is guess Numeric IDs and source-version folders are unchanged. Older reader releases ignore these new folders but retain them on disk; upgrade again to restore visibility. +Whole-volume WeebCentral releases use the analogous +`special-weebcentral-volume-<26-character-release-id>` chapter key. They live in +the matching volume book and carry `releaseKind:"volume"`, `volume`, `sourceId`, +and a `Volume N` label in `chapter.json`. This keeps an entire volume distinct +from a numbered chapter in the same book. The default `skip-any` download policy +also skips a whole-volume release if that book already contains numbered chapters; +`add-source` allows the extra source edition explicitly. + The number is normalised: leading zeros and trailing decimal zeros are dropped (`0010.50` → `10.5`), and an `x` prefix is kept. The result is the **key**: | Folder | Key | diff --git a/docs/scraper-service.md b/docs/scraper-service.md index aff3de2..0685db8 100644 --- a/docs/scraper-service.md +++ b/docs/scraper-service.md @@ -154,6 +154,12 @@ exclude them; missing, exact-ID and applicable volume scopes include them. Catal ordering puts numbered chapters first and source-ID-ordered specials afterward; first/last/latest operate on that ordering, not on publication timestamps. +WeebCentral whole-volume releases use `special-weebcentral-volume-` +and a `Volume N` label rather than an invented chapter number. MangaPill is a +first-party module in the same scraper process; its catalog and chapter HTML +must remain on `mangapill.com`, and its page URLs must match the reviewed image +host. It uses the ordinary source lane, pacing, checkpoint and publication rules. + `existingPolicy:"skip-any"` skips a chapter when any version already exists. `existingPolicy:"add-source"` permits another host's version but still skips a version already present from the selected host. Existing version folders are never overwritten. diff --git a/scraper/adapters/index.js b/scraper/adapters/index.js index 00b7d37..a27bed8 100644 --- a/scraper/adapters/index.js +++ b/scraper/adapters/index.js @@ -5,6 +5,7 @@ const adapters = Object.freeze([ require('../sources/weebcentral'), require('../sources/mangadex'), + require('../sources/mangapill'), ]); const { adapterForPeer } = require('../../peering/adapter'); diff --git a/scraper/adapters/mangapill.js b/scraper/adapters/mangapill.js new file mode 100644 index 0000000..3d53f86 --- /dev/null +++ b/scraper/adapters/mangapill.js @@ -0,0 +1,35 @@ +'use strict'; + +const { parse } = require('../plugins/registry'); + +function mangaPillInfo(value) { + let url; + try { url = new URL(String(value)); } catch { return null; } + if (url.protocol !== 'https:' || url.hostname !== 'mangapill.com' || url.username || url.password || url.port) return null; + const match = /^\/manga\/(\d+)\/([a-z0-9-]+)\/?$/i.exec(url.pathname); + if (!match) return null; + return { source: 'mangapill', id: match[1], title: match[2].replace(/-/g, ' '), + url: `https://mangapill.com/manga/${match[1]}/${match[2]}` }; +} + +async function discoverMangaPillCatalog(seriesUrl, http) { + const html = await (await http.get(seriesUrl, { accept: 'text/html' })).text(); + const parsed = parse('mangapill', 'catalog', { html, url: seriesUrl }); + const chapters = parsed.chapters.sort((a, b) => Number(a.number) - Number(b.number)); + return { ...parsed, chapters, + ...(parsed.chapterLinks && !chapters.length ? { availability: { reason: 'unrecognized-labels', + message: `MangaPill lists ${parsed.chapterLinks} chapter links, but their numbering labels are not recognized.` } } : {}), + }; +} + +async function discoverMangaPillChapter(ref, http) { + const html = await (await http.get(ref.url, { accept: 'text/html' })).text(); + const parsed = parse('mangapill', 'pages', { html, url: ref.url }); + if (!parsed.pages.length) throw new Error(`No pages found for MangaPill chapter ${ref.number}`); + return { number: ref.number, title: ref.title || '', + source: { name: 'MangaPill', url: ref.url }, + pages: parsed.pages.map(url => ({ url, headers: { referer: ref.url } })), + }; +} + +module.exports = { mangaPillInfo, discoverMangaPillCatalog, discoverMangaPillChapter }; diff --git a/scraper/adapters/weebcentral.js b/scraper/adapters/weebcentral.js index ba5f1a4..53af60d 100644 --- a/scraper/adapters/weebcentral.js +++ b/scraper/adapters/weebcentral.js @@ -25,7 +25,9 @@ async function discoverWeebCatalog(seriesUrl, http) { const parsed = parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }); const chapterLinks = parsed.chapterLinks; const chapters = parsed.chapters.sort((a, b) => - Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number)); + Number(a.book?.number || 1) - Number(b.book?.number || 1) + || Number(a.releaseKind === 'volume') - Number(b.releaseKind === 'volume') + || Number(a.number) - Number(b.number)); return { title, chapters, chapterLinks, ...(chapterLinks && !chapters.length ? { availability: { reason: 'unrecognized-labels', message: `WeebCentral lists ${chapterLinks} chapter links, but their numbering labels are not recognized by the parser.` } } : {}), @@ -41,6 +43,8 @@ async function discoverWeebChapter(chapter, http) { if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`); return { number: chapter.number, title: chapter.title || '', + ...(chapter.special ? { special: true, label: chapter.label, sourceId: chapter.id, + sourceNumber: chapter.volume, releaseKind: chapter.releaseKind, volume: chapter.volume } : {}), ...(chapter.book ? { book: chapter.book } : {}), added, source: { name: 'WeebCentral', url: chapter.url }, pages, diff --git a/scraper/jobs/weebcentral-archive.js b/scraper/jobs/weebcentral-archive.js index a4ce16c..a5f0d3a 100644 --- a/scraper/jobs/weebcentral-archive.js +++ b/scraper/jobs/weebcentral-archive.js @@ -40,14 +40,18 @@ async function renameRetry(from, to) { } async function runArchive(options) { + const source = options.source || 'weebcentral'; + const sourceLabel = options.sourceLabel || 'WeebCentral'; + const discoverCatalog = options.discoverCatalog || discoverWeebCatalog; + const discoverChapter = options.discoverChapter || discoverWeebChapter; const checkControl = () => { const reason = options.shouldStop && options.shouldStop(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); }; const libraryRoot = path.resolve(options.libraryRoot || 'library'); - const sourceId = options.seriesUrl.split('/series/')[1]?.split('/')[0]; - if (!sourceId) throw new Error('Could not read WeebCentral series id'); - const workRoot = path.join(libraryRoot, `.mcscraper-work-${sourceId}`); + const sourceId = options.sourceId || options.seriesUrl.split('/series/')[1]?.split('/')[0]; + if (!sourceId) throw new Error(`Could not read ${sourceLabel} series id`); + const workRoot = path.join(libraryRoot, source === 'weebcentral' ? `.mcscraper-work-${sourceId}` : `.mcscraper-work-${source}-${sourceId}`); const stateFile = path.join(workRoot, 'job.json'); const logFile = path.join(workRoot, 'job.log'); await fs.mkdir(workRoot, { recursive: true }); @@ -72,13 +76,13 @@ async function runArchive(options) { minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2, onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }), }; - const http = createWeebHttp(httpOptions); + const http = options.createHttp ? options.createHttp(httpOptions) : createWeebHttp(httpOptions); checkControl(); console.log(`[archive] discovering catalog ${options.seriesUrl}`); - const catalog = await discoverWeebCatalog(options.seriesUrl, http); + const catalog = await discoverCatalog(options.seriesUrl, http); if (options.onCatalog) await options.onCatalog(catalog); - if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || 'WeebCentral lists no chapter links for this series.'), { + if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || `${sourceLabel} lists no chapter links for this series.`), { failureKind: catalog.chapterLinks ? 'adapter' : 'unavailable', }); const keys = new Map(); @@ -92,20 +96,20 @@ async function runArchive(options) { const metadata = options.metadataSearch ? await lookupAniList(options.metadataSearch) : {}; const series = { title: catalog.title, ...require('../../reading-direction').fallbackDirection(), - sources: [{ name: 'WeebCentral', url: options.seriesUrl }], + sources: [{ name: sourceLabel, url: options.seriesUrl }], metadata: { ...metadata, title: catalog.title }, coverUrl: metadata['x-anilist']?.coverUrl, - scraper: { ids: { weebcentral: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } }, + scraper: { ids: { [source]: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } }, }; let inventory = await scanLibrary(libraryRoot); let manga = findManga(inventory, { - source: 'weebcentral', seriesId: sourceId, sourceUrl: options.seriesUrl, + source, seriesId: sourceId, sourceUrl: options.seriesUrl, anilist: metadata['x-anilist']?.id, libraryMangaId: options.libraryMangaId, }); if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`); const eligible = incremental - ? catalog.chapters.filter(ref => shouldDownload(manga, ref, 'weebcentral', options.existingPolicy)) + ? catalog.chapters.filter(ref => shouldDownload(manga, ref, source, options.existingPolicy)) : catalog.chapters; const refs = incremental ? selectByScope(eligible, options.scope) : eligible; state.status = 'running'; @@ -128,27 +132,30 @@ async function runArchive(options) { for (let index = 0; index < refs.length; index++) { checkControl(); const ref = refs[index]; + const display = ref.label || ref.number; if (!incremental && completed.has(ref.id)) { if (!incremental) console.log(`[archive] skip completed ${ref.number} (${index + 1}/${refs.length})`); continue; } - if (incremental && !shouldDownload(manga, ref, 'weebcentral', options.existingPolicy)) { + if (incremental && !shouldDownload(manga, ref, source, options.existingPolicy)) { state.jobCompleted++; - await event('info', `Skipped existing chapter ${ref.number}`, { chapter: ref.number }); + await event('info', `Skipped existing ${display}`, { chapter: ref.number }); continue; } - console.log(`[archive] inspect ${ref.number} (${index + 1}/${refs.length})`); + console.log(`[archive] inspect ${display} (${index + 1}/${refs.length})`); let chapter; let result; try { state.status = 'inspecting'; - state.current = { number: ref.number, ...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: null }; - await event('info', `Inspecting chapter ${ref.number}`, { chapter: ref.number }); - chapter = await discoverWeebChapter(ref, http); + state.current = { number: ref.number, ...(ref.label ? { label: ref.label } : {}), + ...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1, + total: refs.length, downloadedPages: 0, expectedPages: null }; + await event('info', `Inspecting ${display}`, { chapter: ref.number }); + chapter = await discoverChapter(ref, http); state.status = 'downloading'; state.current.expectedPages = chapter.pages.length; - await event('info', `Downloading chapter ${ref.number}: ${chapter.pages.length} pages`, { chapter: ref.number }); - console.log(`[archive] download ${ref.number}: ${chapter.pages.length} pages`); + await event('info', `Downloading ${display}: ${chapter.pages.length} pages`, { chapter: ref.number }); + console.log(`[archive] download ${display}: ${chapter.pages.length} pages`); result = await publishManifest({ version: 1, series, chapters: [chapter] }, { libraryRoot: incremental ? libraryRoot : workRoot, ...(incremental && manga ? { mangaDir: manga.path } : {}), @@ -185,11 +192,12 @@ async function runArchive(options) { if (published && incremental) { publishedCount++; if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] }; - manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number), sourceKey: sourceKey(chapter.source.url, chapter.source.name) }); + manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number), + releaseKind: ref.releaseKind || null, sourceKey: sourceKey(chapter.source.url, chapter.source.name) }); if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number }); } - await event('info', `Completed chapter ${ref.number} (${state.jobCompleted}/${state.total})`, { chapter: ref.number }); - console.log(`[archive] complete ${ref.number}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`); + await event('info', `Completed ${display} (${state.jobCompleted}/${state.total})`, { chapter: ref.number }); + console.log(`[archive] complete ${display}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`); if (index < refs.length - 1) { checkControl(); const pause = randomBetween(options.chapterMinMs, options.chapterMaxMs); diff --git a/scraper/lib/library-inventory.js b/scraper/lib/library-inventory.js index fdbffe6..aa89281 100644 --- a/scraper/lib/library-inventory.js +++ b/scraper/lib/library-inventory.js @@ -26,6 +26,7 @@ function sourceKey(url, name = '') { if (parsed.protocol === 'peer:' && /^p[0-9a-f]{24}$/.test(host)) return `peer-${host}`; if (host === 'weebcentral.com') return 'weebcentral'; if (host === 'mangadex.org') return 'mangadex'; + if (host === 'mangapill.com') return 'mangapill'; return host; } catch {} const label = String(name).toLowerCase(); @@ -59,6 +60,7 @@ async function scanLibrary(libraryRoot) { book, bookFolder: bookEntry.name, number, version: versionEntry.name, sourceName: String(source.name || ''), sourceUrl: String(source.url || ''), sourceKey: sourceKey(source.url, source.name), + releaseKind: chapterMeta.releaseKind || null, }); } } @@ -113,11 +115,16 @@ function existingVersions(manga, ref) { if (!manga) return []; const book = chapterBook(ref, manga); const number = isSpecialKey(ref.number) ? String(ref.number).toLowerCase() : chapterNumber(ref.number); + if (ref.releaseKind === 'volume') return manga.chapters.filter(item => item.book === book && item.releaseKind === 'volume'); return manga.chapters.filter(item => item.book === book && item.number === number); } function shouldDownload(manga, ref, source, existingPolicy = 'skip-any') { const versions = existingVersions(manga, ref); + // A whole-volume archive can overlap every numbered chapter in that book. + // Require an explicit add-source choice before storing both representations. + if (ref.releaseKind === 'volume' && existingPolicy === 'skip-any' && manga?.chapters.some(item => + item.book === chapterBook(ref, manga) && item.releaseKind !== 'volume')) return false; if (!versions.length) return true; if (existingPolicy === 'add-source') return !versions.some(item => item.sourceKey === source); return false; diff --git a/scraper/lib/publisher.js b/scraper/lib/publisher.js index f6e9406..c1599a2 100644 --- a/scraper/lib/publisher.js +++ b/scraper/lib/publisher.js @@ -11,6 +11,7 @@ const { sizeFromBuffer, imageSize } = require('../../library'); const { directionFields } = require('../../reading-direction'); const { mergeImportedMetadata } = require('./metadata-merge'); const { normalizeWorkIdentity } = require('./work-identity'); +const { parseKey } = require('../../chapter-identity'); const token = () => crypto.randomBytes(5).toString('hex'); // Staging is a recovery asset, not disposable garbage. Legacy stages without an @@ -91,6 +92,15 @@ async function findManga(libraryRoot, sourceUrls) { return null; } +async function bookDirectory(mangaDir, bookNumber, bookTitle) { + const names = (await fsp.readdir(mangaDir, { withFileTypes: true })) + .filter(entry => entry.isDirectory() && parseKey(entry.name)?.key === String(bookNumber)) + .map(entry => entry.name); + if (names.length > 1) throw new Error(`Ambiguous library book ${bookNumber}: ${names.join(', ')}`); + const defaultName = `v${String(bookNumber).padStart(2, '0')}`; + return path.join(mangaDir, names[0] || (bookTitle ? `${defaultName} ${cleanPart(bookTitle)}` : defaultName)); +} + function validateManifest(manifest) { if (!manifest || manifest.version !== 1) throw new Error('Manifest version must be 1'); if (!manifest.series?.title) throw new Error('series.title is required'); @@ -172,7 +182,10 @@ async function stageChapter(bookDir, chapter, http, onPage, priorMeta = {}) { title: chapter.title || '', source: chapter.source, added: chapter.added || new Date().toISOString(), priority: Number(chapter.priority || 0), extra: !!chapter.extra, - ...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId, sourceNumber: chapter.sourceNumber ?? null } : {}), + ...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId, + sourceNumber: chapter.sourceNumber ?? null, + ...(chapter.releaseKind ? { releaseKind: chapter.releaseKind } : {}), + ...(chapter.volume ? { volume: chapter.volume } : {}) } : {}), ...directionFields([priorMeta, chapter], dimensions), }, null, 2) + '\n', { mode: 0o644 }); // A process interruption during a write can leave a hidden pending file. @@ -230,8 +243,7 @@ async function publishManifest(manifest, options = {}) { for (const chapter of manifest.chapters) { const bookNumber = Number(chapter.book?.number || 1); if (!Number.isInteger(bookNumber) || bookNumber < 1) throw new Error(`Invalid book number for chapter ${chapter.number}`); - const bookName = `v${String(bookNumber).padStart(2, '0')}${chapter.book?.title ? ` ${cleanPart(chapter.book.title)}` : ''}`; - const bookDir = path.join(mangaDir, bookName); + const bookDir = await bookDirectory(mangaDir, bookNumber, chapter.book?.title); await fsp.mkdir(bookDir, { recursive: true, mode: 0o755 }); await cleanupOwnTemps(bookDir); if (chapter.book?.title) { diff --git a/scraper/plugins/broker.js b/scraper/plugins/broker.js index af5e653..18be820 100644 --- a/scraper/plugins/broker.js +++ b/scraper/plugins/broker.js @@ -22,6 +22,10 @@ function publicAddress(address) { } function validateWeebRequest(value, options = {}) { + return validateHostRequest(value, 'weebcentral.com', options); +} + +function validateHostRequest(value, sourceHost, options = {}) { const url = new URL(value); const host = url.hostname.toLowerCase(); if (url.protocol !== 'https:' || url.username || url.password || url.port @@ -29,8 +33,8 @@ function validateWeebRequest(value, options = {}) { || host.endsWith('.local') || !host.includes('.')) { throw new Error('Source request URL is not public HTTPS'); } - if (options.accept === 'text/html' && host !== 'weebcentral.com') { - throw new Error('WeebCentral catalog request left its source host'); + if (options.accept === 'text/html' && host !== sourceHost) { + throw new Error('Source catalog request left its source host'); } return url; } @@ -88,4 +92,9 @@ function createWeebHttp(options = {}) { return new SlowHttp({ ...options, transport: pinnedFetch, validateUrl: validateWeebRequest }); } -module.exports = { publicAddress, validateWeebRequest, pinnedLookup, pinnedFetch, createWeebHttp }; +function createHostHttp(sourceHost, options = {}) { + return new SlowHttp({ ...options, transport: pinnedFetch, + validateUrl: (value, requestOptions) => validateHostRequest(value, sourceHost, requestOptions) }); +} + +module.exports = { publicAddress, validateWeebRequest, validateHostRequest, pinnedLookup, pinnedFetch, createWeebHttp, createHostHttp }; diff --git a/scraper/plugins/mangapill.js b/scraper/plugins/mangapill.js new file mode 100644 index 0000000..0666325 --- /dev/null +++ b/scraper/plugins/mangapill.js @@ -0,0 +1,32 @@ +'use strict'; + +// First-party parser; inputs are fetched by the paced source broker, not here. +const decode = value => String(value).replace(/&/g, '&').replace(/'|'/gi, "'") + .replace(/"/g, '"').replace(/<[^>]+>/g, '').trim(); + +function catalog({ html, url }) { + const sourceId = new URL(url).pathname.split('/')[2]; + const title = decode(html.match(/]*>([\s\S]*?)<\/h1>/i)?.[1] || 'Untitled'); + const chapters = []; + const re = /]*href="(\/chapters\/(\d+-\d+)\/[^"?#]+)"[^>]*>([\s\S]*?)<\/a>/gi; + for (const match of html.matchAll(re)) { + if (match[2].split('-')[0] !== sourceId) continue; + const label = decode(match[3]); + const number = label.match(/^Chapter\s+([0-9]+(?:\.[0-9]+)?)(?:\b|$)/i)?.[1]; + if (!number) continue; + chapters.push({ id: match[2], number, title: '', url: new URL(match[1], url).href, raw: label }); + } + return { title, chapters: [...new Map(chapters.map(ref => [ref.id, ref])).values()], + chapterLinks: [...html.matchAll(/href="\/chapters\/\d+-\d+\//gi)].length }; +} + +function pages({ html }) { + const images = []; + for (const match of html.matchAll(/]*class="[^"]*\bjs-page\b[^"]*"[^>]*>/gi)) { + const url = match[0].match(/\bdata-src="([^"]+)"/i)?.[1]; + if (url) images.push(url.replace(/&/g, '&')); + } + return { pages: [...new Set(images)] }; +} + +module.exports = { catalog, pages }; diff --git a/scraper/plugins/manifest.js b/scraper/plugins/manifest.js index d1b8de0..dabc551 100644 --- a/scraper/plugins/manifest.js +++ b/scraper/plugins/manifest.js @@ -17,6 +17,12 @@ const manifests = Object.freeze({ provenance: 'MangaCircus existing MangaDex adapter; no Uchiyomi code copied', operations: Object.freeze(['metadata', 'feed', 'catalog', 'atHome']), }), + mangapill: Object.freeze({ + id: 'mangapill', version: 1, label: 'MangaPill', status: 'in-process', + sourceHosts: Object.freeze(['mangapill.com']), implementation: 'first-party', + provenance: 'MangaCircus implementation; upstream extension reviewed, no code copied', + operations: Object.freeze(['catalog', 'pages']), + }), }); module.exports = { manifests }; diff --git a/scraper/plugins/registry.js b/scraper/plugins/registry.js index a74f230..4a3730a 100644 --- a/scraper/plugins/registry.js +++ b/scraper/plugins/registry.js @@ -6,6 +6,7 @@ const { validate } = require('./validation'); const plugins = Object.freeze({ weebcentral: require('./weebcentral'), mangadex: require('./mangadex'), + mangapill: require('./mangapill'), }); function parse(plugin, operation, input) { diff --git a/scraper/plugins/validation.js b/scraper/plugins/validation.js index b1d4b01..81dd5a5 100644 --- a/scraper/plugins/validation.js +++ b/scraper/plugins/validation.js @@ -33,8 +33,20 @@ function mangaDexUrl(value, kind) { return url.href; } +function mangaPillUrl(value, kind) { + const url = new URL(value); + if (url.protocol !== 'https:' || url.username || url.password || url.port) throw new Error('Invalid MangaPill plugin URL'); + if (kind === 'chapter' && (url.hostname !== 'mangapill.com' || !/^\/chapters\/\d+-\d+\//.test(url.pathname))) { + throw new Error('Unexpected MangaPill chapter URL'); + } + if (kind === 'page' && (url.hostname !== 'cdn.readdetectiveconan.com' || !url.pathname.startsWith('/file/mangap/'))) { + throw new Error('Unexpected MangaPill page host'); + } + return url.href; +} + function validate(plugin, operation, output) { - if (!['weebcentral', 'mangadex'].includes(plugin) || !output || typeof output !== 'object' || Array.isArray(output)) { + if (!['weebcentral', 'mangadex', 'mangapill'].includes(plugin) || !output || typeof output !== 'object' || Array.isArray(output)) { throw new Error('Invalid source plugin output'); } if (plugin === 'mangadex') { @@ -51,6 +63,21 @@ function validate(plugin, operation, output) { } else throw new Error('Unknown source plugin operation'); return output; } + if (plugin === 'mangapill') { + if (operation === 'catalog') { + if (typeof output.title !== 'string' || !Array.isArray(output.chapters) + || !Number.isInteger(output.chapterLinks) || output.chapterLinks < 0) throw new Error('Invalid MangaPill catalog'); + for (const chapter of output.chapters) { + if (typeof chapter?.id !== 'string' || !/^\d+-\d+$/.test(chapter.id) + || !/^\d+(?:\.\d+)?$/.test(chapter.number)) throw new Error('Invalid MangaPill chapter'); + mangaPillUrl(chapter.url, 'chapter'); + } + } else if (operation === 'pages') { + if (!Array.isArray(output.pages)) throw new Error('Invalid MangaPill pages'); + for (const page of output.pages) mangaPillUrl(page, 'page'); + } else throw new Error('Unknown MangaPill operation'); + return output; + } if (operation === 'series') { if (typeof output.title !== 'string' || typeof output.listUrl !== 'string' && output.listUrl !== null) throw new Error('Invalid series parse'); if (output.listUrl !== null) sourceUrl(output.listUrl, '/series/'); diff --git a/scraper/plugins/weebcentral.js b/scraper/plugins/weebcentral.js index cf92892..4d01304 100644 --- a/scraper/plugins/weebcentral.js +++ b/scraper/plugins/weebcentral.js @@ -1,4 +1,5 @@ 'use strict'; +const { volumeKey } = require('../../chapter-identity'); // First-party, pure parsing functions. This module never fetches remote content. const decode = value => value.replace(/&/g, '&').replace(/'/g, "'").replace(/"/g, '"').replace(/<[^>]+>/g, '').trim(); @@ -17,6 +18,7 @@ function catalog({ html, url }) { const text = decode(match[3]).replace(/\s+/g, ' '); const season = Number(text.match(/^S(\d+)\s*-/i)?.[1] || 0) || null; const regular = text.match(/\b(?:Chapter|Rating|Episode|Night|Log|Act|Scene|Kapitel|File|Battle)\s*[.:]?\s*([0-9]+(?:\.[0-9]+)?)/i)?.[1]; + const volume = !regular ? text.match(/^Volume\s+([0-9]+(?:\.[0-9]+)?)(?!\d)/i)?.[1] : null; const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1]; const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null); if (number) found.push({ @@ -24,6 +26,12 @@ function catalog({ html, url }) { ...(season ? { book: { number: season, title: `Season ${season}` } } : {}), url: absolute(url, match[1]), raw: text, }); + else if (volume) found.push({ + id: match[2], number: volumeKey(match[2]), title: '', special: true, + label: `Volume ${volume}`, releaseKind: 'volume', volume, + book: { number: Number(volume), title: `Volume ${volume}` }, + url: absolute(url, match[1]), raw: text, + }); } const chapters = [...new Map(found.map(chapter => [chapter.id, chapter])).values()]; const chapterLinks = [...html.matchAll(/]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length; diff --git a/scraper/service.js b/scraper/service.js index 79a3610..057e58b 100644 --- a/scraper/service.js +++ b/scraper/service.js @@ -21,6 +21,7 @@ const FAST_RATE_SETTINGS = { const DEFAULT_SOURCE_RATES = { weebcentral: { ...FAST_RATE_SETTINGS }, mangadex: { ...FAST_RATE_SETTINGS }, + mangapill: { ...FAST_RATE_SETTINGS }, }; const DEFAULT_SETTINGS = { ...FAST_RATE_SETTINGS, diff --git a/scraper/sources/mangapill.js b/scraper/sources/mangapill.js new file mode 100644 index 0000000..82acca9 --- /dev/null +++ b/scraper/sources/mangapill.js @@ -0,0 +1,29 @@ +'use strict'; + +const { createHostHttp } = require('../plugins/broker'); +const { mangaPillInfo, discoverMangaPillCatalog, discoverMangaPillChapter } = require('../adapters/mangapill'); +const { runArchive } = require('../jobs/weebcentral-archive'); +const { controlledHttp } = require('./common'); + +const createHttp = options => createHostHttp('mangapill.com', options); + +module.exports = { + id: 'mangapill', label: 'MangaPill', match: mangaPillInfo, + async discover(item, options) { + const http = createHttp({ minMs: options.settings.minMs, maxMs: options.settings.maxMs, + maxRetries: 2, onEvent: options.onEvent }); + return discoverMangaPillCatalog(item.url, controlledHttp(http, options.shouldStop)); + }, + run(item, options) { + return runArchive({ + source: 'mangapill', sourceLabel: 'MangaPill', sourceId: mangaPillInfo(item.url)?.id, + discoverCatalog: discoverMangaPillCatalog, discoverChapter: discoverMangaPillChapter, createHttp, + seriesUrl: item.url, libraryRoot: options.libraryRoot, + metadataSearch: item.metadataSearch || item.title, + minMs: options.settings.minMs, maxMs: options.settings.maxMs, + chapterMinMs: options.settings.chapterMinMs, chapterMaxMs: options.settings.chapterMaxMs, + shouldStop: options.shouldStop, scope: item.scope, existingPolicy: item.existingPolicy, + libraryMangaId: item.libraryMangaId, onPublished: options.onPublished, onCatalog: options.onCatalog, + }); + }, +}; diff --git a/scraper/test/mangapill-source.test.js b/scraper/test/mangapill-source.test.js new file mode 100644 index 0000000..ee892f3 --- /dev/null +++ b/scraper/test/mangapill-source.test.js @@ -0,0 +1,61 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { adapterFor } = require('../adapters'); +const { parse } = require('../plugins/registry'); +const { discoverMangaPillCatalog, discoverMangaPillChapter } = require('../adapters/mangapill'); +const { validateHostRequest } = require('../plugins/broker'); +const { runArchive } = require('../jobs/weebcentral-archive'); +const { Canvas } = require('../../tools/png'); +const { scanLibrary } = require('../lib/library-inventory'); + +const seriesUrl = 'https://mangapill.com/manga/552/bleach'; +const chapterUrl = 'https://mangapill.com/chapters/552-10001000/bleach-chapter-1'; +const imageUrl = 'https://cdn.readdetectiveconan.com/file/mangap/552/10001000/1.jpg'; +const catalogHtml = `

Bleach

`; +const chapterHtml = ``; + +test('MangaPill discovery stays in its host and keeps ordered source chapter IDs', async () => { + assert.equal(adapterFor(seriesUrl).adapter.id, 'mangapill'); + assert.equal(adapterFor('http://mangapill.com/manga/552/bleach'), null); + const http = { async get(url) { + assert.ok([seriesUrl, chapterUrl].includes(url)); + return { async text() { return url === seriesUrl ? catalogHtml : chapterHtml; } }; + } }; + const catalog = await discoverMangaPillCatalog(seriesUrl, http); + assert.deepEqual(catalog.chapters.map(chapter => [chapter.id, chapter.number]), + [['552-10001000', '1'], ['552-10002000', '2']]); + const chapter = await discoverMangaPillChapter(catalog.chapters[0], http); + assert.deepEqual(chapter.pages, [{ url: imageUrl, headers: { referer: chapterUrl } }]); + assert.throws(() => parse('mangapill', 'pages', { + html: '', url: chapterUrl, + }), /page host/); + assert.throws(() => validateHostRequest('https://other.example/manga/552', 'mangapill.com', { accept: 'text/html' }), /left its source host/); +}); + +test('MangaPill uses the shared checkpoint-safe publisher and skips an existing chapter', async t => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-mangapill-')); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const png = new Canvas(10, 20, [1, 2, 3]).png(); + const createHttp = () => ({ async get(url) { + if (url === seriesUrl) return { async text() { return catalogHtml.replace(/]*>Chapter 2<\/a>/, ''); } }; + if (url === chapterUrl) return { async text() { return chapterHtml; } }; + if (url === imageUrl) return { async arrayBuffer() { return png; } }; + throw new Error(`Unexpected source request: ${url}`); + } }); + const options = { source: 'mangapill', sourceLabel: 'MangaPill', sourceId: '552', + discoverCatalog: discoverMangaPillCatalog, discoverChapter: discoverMangaPillChapter, createHttp, + seriesUrl, libraryRoot: root, scope: { mode: 'missing' }, + minMs: 1000, maxMs: 1000, chapterMinMs: 5000, chapterMaxMs: 5000 }; + const first = await runArchive(options); + assert.equal(first.publishedCount, 1); + const manga = (await scanLibrary(root)).mangas[0]; + assert.equal(manga.ids.mangapill, '552'); + assert.equal(manga.chapters[0].sourceKey, 'mangapill'); + const second = await runArchive(options); + assert.equal(second.publishedCount, 0); + assert.equal(second.selectedCount, 0); +}); diff --git a/scraper/test/source-plugin.test.js b/scraper/test/source-plugin.test.js index 7ce4db2..d83c9f1 100644 --- a/scraper/test/source-plugin.test.js +++ b/scraper/test/source-plugin.test.js @@ -50,7 +50,7 @@ test('in-process parser preserves WeebCentral catalog and page discovery', async }); test('a new source is an explicit adapter, not another container or remote module', () => { - assert.deepEqual(adapters.map(adapter => adapter.id), ['weebcentral', 'mangadex']); + assert.deepEqual(adapters.map(adapter => adapter.id), ['weebcentral', 'mangadex', 'mangapill']); for (const adapter of adapters) { assert.equal(typeof adapter.match, 'function'); assert.equal(typeof adapter.discover, 'function'); diff --git a/scraper/test/weebcentral-volumes.test.js b/scraper/test/weebcentral-volumes.test.js new file mode 100644 index 0000000..cf153ae --- /dev/null +++ b/scraper/test/weebcentral-volumes.test.js @@ -0,0 +1,69 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { parseChapters, discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral'); +const { parseSpecialFolder } = require('../../chapter-identity'); +const { selectByScope } = require('../lib/scope'); +const { publishManifest } = require('../lib/publisher'); +const { scanLibrary, shouldDownload } = require('../lib/library-inventory'); +const reader = require('../../library'); +const { Canvas } = require('../../tools/png'); + +const ids = ['01JHVPNS8PNMSNF4H18ERX3SMK', '01JHVPNS8PNSDS66BEPVHH536F']; +const seriesUrl = 'https://weebcentral.com/series/01J76XY8CF6MA6W5VJWZDNWESZ/Akira'; + +test('whole-volume releases keep their source IDs and do not masquerade as chapters', async t => { + const html = `Volume 1Volume 2`; + const refs = parseChapters(html, seriesUrl); + assert.equal(refs.length, 2); + assert.deepEqual(refs.map(ref => ref.volume), ['1', '2']); + assert.ok(refs.every(ref => ref.special && ref.releaseKind === 'volume')); + assert.equal(parseSpecialFolder(refs[0].number + ' [WeebCentral]').releaseKind, 'volume'); + assert.equal(selectByScope(refs, { mode: 'volume', volume: '2' })[0].id, ids[1]); + assert.equal(selectByScope(refs, { mode: 'chapter', chapter: '1' }).length, 0); + + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-volume-')); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const page = path.join(root, '.fixture.png'); + await fs.writeFile(page, new Canvas(10, 20, [1, 2, 3]).png()); + const manifest = { version: 1, series: { title: 'Volume fixture', direction: 'rtl', sources: [{ name: 'WeebCentral', url: seriesUrl }] }, + chapters: [refs[0], { number: '1', source: { name: 'Other', url: 'https://example.org/1' } }].map(ref => ({ + ...ref, source: ref.source || { name: 'WeebCentral', url: ref.url }, pages: [{ path: page }], + })) }; + await publishManifest(manifest, { libraryRoot: root }); + const scanned = (await reader.scanLibrary(root)).index.mangas[0]; + assert.equal(scanned.chapters.length, 2); + assert.ok(scanned.chapters.some(ch => ch.label === 'Volume 1')); + assert.ok(scanned.chapters.some(ch => ch.label === 'Ch. 1')); + const manga = (await scanLibrary(root)).mangas[0]; + assert.equal(shouldDownload(manga, refs[0], 'weebcentral'), false); + assert.equal(shouldDownload(manga, refs[1], 'weebcentral'), true); + assert.equal(shouldDownload(manga, refs[0], 'weebcentral', 'add-source'), false); + const numberedOnly = { ...manga, chapters: manga.chapters.filter(chapter => chapter.releaseKind !== 'volume') }; + assert.equal(shouldDownload(numberedOnly, refs[0], 'weebcentral'), false); + assert.equal(shouldDownload(numberedOnly, refs[0], 'weebcentral', 'add-source'), true); +}); + +test('catalog ordering and chapter discovery preserve whole-volume metadata', async () => { + const html = `

Volume fixture

Volume 1` + + `Chapter 1`; + const http = { async get(url) { + const body = url === seriesUrl ? html + : url === `https://weebcentral.com/chapters/${ids[0]}` ? '' + : ''; + return { async text() { return body; } }; + } }; + const catalog = await discoverWeebCatalog(seriesUrl, http); + assert.equal(catalog.chapters.length, 2); + assert.equal(catalog.chapters[0].number, '1'); + assert.equal(catalog.chapters[1].label, 'Volume 1'); + const chapter = await discoverWeebChapter(catalog.chapters[1], http); + assert.equal(chapter.releaseKind, 'volume'); + assert.equal(chapter.sourceId, ids[0]); + assert.equal(chapter.sourceNumber, '1'); + assert.equal(chapter.book.number, 1); + assert.equal(chapter.pages.length, 1); +});