Maashkinoozhe/MangaCircus
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add first-party MangaPill source and distinct volume releases

Browse Source
This commit is contained in:
Codex committed 2026-10-14 09:05:34 +02:00
1 parent a8df57f165
commit a08e847932
20 files changed
+364 -34
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scraper/adapters/index.js
+1
View File
@@ -5,6 +5,7 @@
const adapters = Object.freeze([
require('../sources/weebcentral'),
require('../sources/mangadex'),
require('../sources/mangapill'),
]);
const { adapterForPeer } = require('../../peering/adapter');
scraper/adapters/mangapill.js
+35
View File
@@ -0,0 +1,35 @@
'use strict';
const { parse } = require('../plugins/registry');
function mangaPillInfo(value) {
let url;
try { url = new URL(String(value)); } catch { return null; }
if (url.protocol !== 'https:' || url.hostname !== 'mangapill.com' || url.username || url.password || url.port) return null;
const match = /^\/manga\/(\d+)\/([a-z0-9-]+)\/?$/i.exec(url.pathname);
if (!match) return null;
return { source: 'mangapill', id: match[1], title: match[2].replace(/-/g, ' '),
url: `https://mangapill.com/manga/${match[1]}/${match[2]}` };
}
async function discoverMangaPillCatalog(seriesUrl, http) {
const html = await (await http.get(seriesUrl, { accept: 'text/html' })).text();
const parsed = parse('mangapill', 'catalog', { html, url: seriesUrl });
const chapters = parsed.chapters.sort((a, b) => Number(a.number) - Number(b.number));
return { ...parsed, chapters,
...(parsed.chapterLinks && !chapters.length ? { availability: { reason: 'unrecognized-labels',
message: `MangaPill lists ${parsed.chapterLinks} chapter links, but their numbering labels are not recognized.` } } : {}),
};
}
async function discoverMangaPillChapter(ref, http) {
const html = await (await http.get(ref.url, { accept: 'text/html' })).text();
const parsed = parse('mangapill', 'pages', { html, url: ref.url });
if (!parsed.pages.length) throw new Error(`No pages found for MangaPill chapter ${ref.number}`);
return { number: ref.number, title: ref.title || '',
source: { name: 'MangaPill', url: ref.url },
pages: parsed.pages.map(url => ({ url, headers: { referer: ref.url } })),
};
}
module.exports = { mangaPillInfo, discoverMangaPillCatalog, discoverMangaPillChapter };
scraper/adapters/weebcentral.js
+5 -1
View File
@@ -25,7 +25,9 @@ async function discoverWeebCatalog(seriesUrl, http) {
const parsed = parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl });
const chapterLinks = parsed.chapterLinks;
const chapters = parsed.chapters.sort((a, b) =>
Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number));
Number(a.book?.number || 1) - Number(b.book?.number || 1)
|| Number(a.releaseKind === 'volume') - Number(b.releaseKind === 'volume')
|| Number(a.number) - Number(b.number));
return { title, chapters, chapterLinks,
...(chapterLinks && !chapters.length ? { availability: { reason: 'unrecognized-labels',
message: `WeebCentral lists ${chapterLinks} chapter links, but their numbering labels are not recognized by the parser.` } } : {}),
@@ -41,6 +43,8 @@ async function discoverWeebChapter(chapter, http) {
if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`);
return {
number: chapter.number, title: chapter.title || '',
...(chapter.special ? { special: true, label: chapter.label, sourceId: chapter.id,
sourceNumber: chapter.volume, releaseKind: chapter.releaseKind, volume: chapter.volume } : {}),
...(chapter.book ? { book: chapter.book } : {}),
added,
source: { name: 'WeebCentral', url: chapter.url }, pages,
scraper/jobs/weebcentral-archive.js
+29 -21
View File
@@ -40,14 +40,18 @@ async function renameRetry(from, to) {
}
async function runArchive(options) {
const source = options.source || 'weebcentral';
const sourceLabel = options.sourceLabel || 'WeebCentral';
const discoverCatalog = options.discoverCatalog || discoverWeebCatalog;
const discoverChapter = options.discoverChapter || discoverWeebChapter;
const checkControl = () => {
const reason = options.shouldStop && options.shouldStop();
if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason));
};
const libraryRoot = path.resolve(options.libraryRoot || 'library');
const sourceId = options.seriesUrl.split('/series/')[1]?.split('/')[0];
if (!sourceId) throw new Error('Could not read WeebCentral series id');
const workRoot = path.join(libraryRoot, `.mcscraper-work-${sourceId}`);
const sourceId = options.sourceId || options.seriesUrl.split('/series/')[1]?.split('/')[0];
if (!sourceId) throw new Error(`Could not read ${sourceLabel} series id`);
const workRoot = path.join(libraryRoot, source === 'weebcentral' ? `.mcscraper-work-${sourceId}` : `.mcscraper-work-${source}-${sourceId}`);
const stateFile = path.join(workRoot, 'job.json');
const logFile = path.join(workRoot, 'job.log');
await fs.mkdir(workRoot, { recursive: true });
@@ -72,13 +76,13 @@ async function runArchive(options) {
minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2,
onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }),
};
const http = createWeebHttp(httpOptions);
const http = options.createHttp ? options.createHttp(httpOptions) : createWeebHttp(httpOptions);
checkControl();
console.log(`[archive] discovering catalog ${options.seriesUrl}`);
const catalog = await discoverWeebCatalog(options.seriesUrl, http);
const catalog = await discoverCatalog(options.seriesUrl, http);
if (options.onCatalog) await options.onCatalog(catalog);
if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || 'WeebCentral lists no chapter links for this series.'), {
if (!catalog.chapters.length) throw Object.assign(new Error(catalog.availability?.message || `${sourceLabel} lists no chapter links for this series.`), {
failureKind: catalog.chapterLinks ? 'adapter' : 'unavailable',
});
const keys = new Map();
@@ -92,20 +96,20 @@ async function runArchive(options) {
const metadata = options.metadataSearch ? await lookupAniList(options.metadataSearch) : {};
const series = {
title: catalog.title, ...require('../../reading-direction').fallbackDirection(),
sources: [{ name: 'WeebCentral', url: options.seriesUrl }],
sources: [{ name: sourceLabel, url: options.seriesUrl }],
metadata: { ...metadata, title: catalog.title },
coverUrl: metadata['x-anilist']?.coverUrl,
scraper: { ids: { weebcentral: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } },
scraper: { ids: { [source]: sourceId, ...(metadata['x-anilist']?.id ? { anilist: metadata['x-anilist'].id } : {}) } },
};
let inventory = await scanLibrary(libraryRoot);
let manga = findManga(inventory, {
source: 'weebcentral', seriesId: sourceId, sourceUrl: options.seriesUrl,
source, seriesId: sourceId, sourceUrl: options.seriesUrl,
anilist: metadata['x-anilist']?.id, libraryMangaId: options.libraryMangaId,
});
if (options.libraryMangaId && !manga) throw new Error(`Library manga target not found: ${options.libraryMangaId}`);
const eligible = incremental
? catalog.chapters.filter(ref => shouldDownload(manga, ref, 'weebcentral', options.existingPolicy))
? catalog.chapters.filter(ref => shouldDownload(manga, ref, source, options.existingPolicy))
: catalog.chapters;
const refs = incremental ? selectByScope(eligible, options.scope) : eligible;
state.status = 'running';
@@ -128,27 +132,30 @@ async function runArchive(options) {
for (let index = 0; index < refs.length; index++) {
checkControl();
const ref = refs[index];
const display = ref.label || ref.number;
if (!incremental && completed.has(ref.id)) {
if (!incremental) console.log(`[archive] skip completed ${ref.number} (${index + 1}/${refs.length})`);
continue;
}
if (incremental && !shouldDownload(manga, ref, 'weebcentral', options.existingPolicy)) {
if (incremental && !shouldDownload(manga, ref, source, options.existingPolicy)) {
state.jobCompleted++;
await event('info', `Skipped existing chapter ${ref.number}`, { chapter: ref.number });
await event('info', `Skipped existing ${display}`, { chapter: ref.number });
continue;
}
console.log(`[archive] inspect ${ref.number} (${index + 1}/${refs.length})`);
console.log(`[archive] inspect ${display} (${index + 1}/${refs.length})`);
let chapter;
let result;
try {
state.status = 'inspecting';
state.current = { number: ref.number, ...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1, total: refs.length, downloadedPages: 0, expectedPages: null };
await event('info', `Inspecting chapter ${ref.number}`, { chapter: ref.number });
chapter = await discoverWeebChapter(ref, http);
state.current = { number: ref.number, ...(ref.label ? { label: ref.label } : {}),
...(ref.book ? { book: ref.book } : {}), sourceId: ref.id, position: index + 1,
total: refs.length, downloadedPages: 0, expectedPages: null };
await event('info', `Inspecting ${display}`, { chapter: ref.number });
chapter = await discoverChapter(ref, http);
state.status = 'downloading';
state.current.expectedPages = chapter.pages.length;
await event('info', `Downloading chapter ${ref.number}: ${chapter.pages.length} pages`, { chapter: ref.number });
console.log(`[archive] download ${ref.number}: ${chapter.pages.length} pages`);
await event('info', `Downloading ${display}: ${chapter.pages.length} pages`, { chapter: ref.number });
console.log(`[archive] download ${display}: ${chapter.pages.length} pages`);
result = await publishManifest({ version: 1, series, chapters: [chapter] }, {
libraryRoot: incremental ? libraryRoot : workRoot,
...(incremental && manga ? { mangaDir: manga.path } : {}),
@@ -185,11 +192,12 @@ async function runArchive(options) {
if (published && incremental) {
publishedCount++;
if (!manga) manga = { id: path.basename(result.mangaDir), path: result.mangaDir, meta: { 'x-scraper': { strategy: 'flat' } }, chapters: [] };
manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number), sourceKey: sourceKey(chapter.source.url, chapter.source.name) });
manga.chapters.push({ book: String(ref.book?.number || 1), number: String(ref.number),
releaseKind: ref.releaseKind || null, sourceKey: sourceKey(chapter.source.url, chapter.source.name) });
if (options.onPublished) await options.onPublished({ mangaDir: result.mangaDir, chapter: ref.number });
}
await event('info', `Completed chapter ${ref.number} (${state.jobCompleted}/${state.total})`, { chapter: ref.number });
console.log(`[archive] complete ${ref.number}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`);
await event('info', `Completed ${display} (${state.jobCompleted}/${state.total})`, { chapter: ref.number });
console.log(`[archive] complete ${display}: ${result.results[0].status} (${state.jobCompleted}/${state.total})`);
if (index < refs.length - 1) {
checkControl();
const pause = randomBetween(options.chapterMinMs, options.chapterMaxMs);
scraper/lib/library-inventory.js
+7
View File
@@ -26,6 +26,7 @@ function sourceKey(url, name = '') {
if (parsed.protocol === 'peer:' && /^p[0-9a-f]{24}$/.test(host)) return `peer-${host}`;
if (host === 'weebcentral.com') return 'weebcentral';
if (host === 'mangadex.org') return 'mangadex';
if (host === 'mangapill.com') return 'mangapill';
return host;
} catch {}
const label = String(name).toLowerCase();
@@ -59,6 +60,7 @@ async function scanLibrary(libraryRoot) {
book, bookFolder: bookEntry.name, number, version: versionEntry.name,
sourceName: String(source.name || ''), sourceUrl: String(source.url || ''),
sourceKey: sourceKey(source.url, source.name),
releaseKind: chapterMeta.releaseKind || null,
});
}
}
@@ -113,11 +115,16 @@ function existingVersions(manga, ref) {
if (!manga) return [];
const book = chapterBook(ref, manga);
const number = isSpecialKey(ref.number) ? String(ref.number).toLowerCase() : chapterNumber(ref.number);
if (ref.releaseKind === 'volume') return manga.chapters.filter(item => item.book === book && item.releaseKind === 'volume');
return manga.chapters.filter(item => item.book === book && item.number === number);
}
function shouldDownload(manga, ref, source, existingPolicy = 'skip-any') {
const versions = existingVersions(manga, ref);
// A whole-volume archive can overlap every numbered chapter in that book.
// Require an explicit add-source choice before storing both representations.
if (ref.releaseKind === 'volume' && existingPolicy === 'skip-any' && manga?.chapters.some(item =>
item.book === chapterBook(ref, manga) && item.releaseKind !== 'volume')) return false;
if (!versions.length) return true;
if (existingPolicy === 'add-source') return !versions.some(item => item.sourceKey === source);
return false;
scraper/lib/publisher.js
+15 -3
View File
@@ -11,6 +11,7 @@ const { sizeFromBuffer, imageSize } = require('../../library');
const { directionFields } = require('../../reading-direction');
const { mergeImportedMetadata } = require('./metadata-merge');
const { normalizeWorkIdentity } = require('./work-identity');
const { parseKey } = require('../../chapter-identity');
const token = () => crypto.randomBytes(5).toString('hex');
// Staging is a recovery asset, not disposable garbage. Legacy stages without an
@@ -91,6 +92,15 @@ async function findManga(libraryRoot, sourceUrls) {
return null;
}
async function bookDirectory(mangaDir, bookNumber, bookTitle) {
const names = (await fsp.readdir(mangaDir, { withFileTypes: true }))
.filter(entry => entry.isDirectory() && parseKey(entry.name)?.key === String(bookNumber))
.map(entry => entry.name);
if (names.length > 1) throw new Error(`Ambiguous library book ${bookNumber}: ${names.join(', ')}`);
const defaultName = `v${String(bookNumber).padStart(2, '0')}`;
return path.join(mangaDir, names[0] || (bookTitle ? `${defaultName} ${cleanPart(bookTitle)}` : defaultName));
}
function validateManifest(manifest) {
if (!manifest || manifest.version !== 1) throw new Error('Manifest version must be 1');
if (!manifest.series?.title) throw new Error('series.title is required');
@@ -172,7 +182,10 @@ async function stageChapter(bookDir, chapter, http, onPage, priorMeta = {}) {
title: chapter.title || '', source: chapter.source,
added: chapter.added || new Date().toISOString(),
priority: Number(chapter.priority || 0), extra: !!chapter.extra,
...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId, sourceNumber: chapter.sourceNumber ?? null } : {}),
...(chapter.special ? { special: true, label: chapter.label || 'Special', sourceId: chapter.sourceId,
sourceNumber: chapter.sourceNumber ?? null,
...(chapter.releaseKind ? { releaseKind: chapter.releaseKind } : {}),
...(chapter.volume ? { volume: chapter.volume } : {}) } : {}),
...directionFields([priorMeta, chapter], dimensions),
}, null, 2) + '\n', { mode: 0o644 });
// A process interruption during a write can leave a hidden pending file.
@@ -230,8 +243,7 @@ async function publishManifest(manifest, options = {}) {
for (const chapter of manifest.chapters) {
const bookNumber = Number(chapter.book?.number || 1);
if (!Number.isInteger(bookNumber) || bookNumber < 1) throw new Error(`Invalid book number for chapter ${chapter.number}`);
const bookName = `v${String(bookNumber).padStart(2, '0')}${chapter.book?.title ? ` ${cleanPart(chapter.book.title)}` : ''}`;
const bookDir = path.join(mangaDir, bookName);
const bookDir = await bookDirectory(mangaDir, bookNumber, chapter.book?.title);
await fsp.mkdir(bookDir, { recursive: true, mode: 0o755 });
await cleanupOwnTemps(bookDir);
if (chapter.book?.title) {
scraper/plugins/broker.js
+12 -3
View File
@@ -22,6 +22,10 @@ function publicAddress(address) {
}
function validateWeebRequest(value, options = {}) {
return validateHostRequest(value, 'weebcentral.com', options);
}
function validateHostRequest(value, sourceHost, options = {}) {
const url = new URL(value);
const host = url.hostname.toLowerCase();
if (url.protocol !== 'https:' || url.username || url.password || url.port
@@ -29,8 +33,8 @@ function validateWeebRequest(value, options = {}) {
|| host.endsWith('.local') || !host.includes('.')) {
throw new Error('Source request URL is not public HTTPS');
}
if (options.accept === 'text/html' && host !== 'weebcentral.com') {
throw new Error('WeebCentral catalog request left its source host');
if (options.accept === 'text/html' && host !== sourceHost) {
throw new Error('Source catalog request left its source host');
}
return url;
}
@@ -88,4 +92,9 @@ function createWeebHttp(options = {}) {
return new SlowHttp({ ...options, transport: pinnedFetch, validateUrl: validateWeebRequest });
}
module.exports = { publicAddress, validateWeebRequest, pinnedLookup, pinnedFetch, createWeebHttp };
function createHostHttp(sourceHost, options = {}) {
return new SlowHttp({ ...options, transport: pinnedFetch,
validateUrl: (value, requestOptions) => validateHostRequest(value, sourceHost, requestOptions) });
}
module.exports = { publicAddress, validateWeebRequest, validateHostRequest, pinnedLookup, pinnedFetch, createWeebHttp, createHostHttp };
scraper/plugins/mangapill.js
+32
View File
@@ -0,0 +1,32 @@
'use strict';
// First-party parser; inputs are fetched by the paced source broker, not here.
const decode = value => String(value).replace(/&amp;/g, '&').replace(/&#39;|&#x27;/gi, "'")
.replace(/&quot;/g, '"').replace(/<[^>]+>/g, '').trim();
function catalog({ html, url }) {
const sourceId = new URL(url).pathname.split('/')[2];
const title = decode(html.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || 'Untitled');
const chapters = [];
const re = /<a\b[^>]*href="(\/chapters\/(\d+-\d+)\/[^"?#]+)"[^>]*>([\s\S]*?)<\/a>/gi;
for (const match of html.matchAll(re)) {
if (match[2].split('-')[0] !== sourceId) continue;
const label = decode(match[3]);
const number = label.match(/^Chapter\s+([0-9]+(?:\.[0-9]+)?)(?:\b|$)/i)?.[1];
if (!number) continue;
chapters.push({ id: match[2], number, title: '', url: new URL(match[1], url).href, raw: label });
}
return { title, chapters: [...new Map(chapters.map(ref => [ref.id, ref])).values()],
chapterLinks: [...html.matchAll(/href="\/chapters\/\d+-\d+\//gi)].length };
}
function pages({ html }) {
const images = [];
for (const match of html.matchAll(/<img\b[^>]*class="[^"]*\bjs-page\b[^"]*"[^>]*>/gi)) {
const url = match[0].match(/\bdata-src="([^"]+)"/i)?.[1];
if (url) images.push(url.replace(/&amp;/g, '&'));
}
return { pages: [...new Set(images)] };
}
module.exports = { catalog, pages };
scraper/plugins/manifest.js
+6
View File
@@ -17,6 +17,12 @@ const manifests = Object.freeze({
provenance: 'MangaCircus existing MangaDex adapter; no Uchiyomi code copied',
operations: Object.freeze(['metadata', 'feed', 'catalog', 'atHome']),
}),
mangapill: Object.freeze({
id: 'mangapill', version: 1, label: 'MangaPill', status: 'in-process',
sourceHosts: Object.freeze(['mangapill.com']), implementation: 'first-party',
provenance: 'MangaCircus implementation; upstream extension reviewed, no code copied',
operations: Object.freeze(['catalog', 'pages']),
}),
});
module.exports = { manifests };
scraper/plugins/registry.js
+1
View File
@@ -6,6 +6,7 @@ const { validate } = require('./validation');
const plugins = Object.freeze({
weebcentral: require('./weebcentral'),
mangadex: require('./mangadex'),
mangapill: require('./mangapill'),
});
function parse(plugin, operation, input) {
scraper/plugins/validation.js
+28 -1
View File
@@ -33,8 +33,20 @@ function mangaDexUrl(value, kind) {
return url.href;
}
function mangaPillUrl(value, kind) {
const url = new URL(value);
if (url.protocol !== 'https:' || url.username || url.password || url.port) throw new Error('Invalid MangaPill plugin URL');
if (kind === 'chapter' && (url.hostname !== 'mangapill.com' || !/^\/chapters\/\d+-\d+\//.test(url.pathname))) {
throw new Error('Unexpected MangaPill chapter URL');
}
if (kind === 'page' && (url.hostname !== 'cdn.readdetectiveconan.com' || !url.pathname.startsWith('/file/mangap/'))) {
throw new Error('Unexpected MangaPill page host');
}
return url.href;
}
function validate(plugin, operation, output) {
if (!['weebcentral', 'mangadex'].includes(plugin) || !output || typeof output !== 'object' || Array.isArray(output)) {
if (!['weebcentral', 'mangadex', 'mangapill'].includes(plugin) || !output || typeof output !== 'object' || Array.isArray(output)) {
throw new Error('Invalid source plugin output');
}
if (plugin === 'mangadex') {
@@ -51,6 +63,21 @@ function validate(plugin, operation, output) {
} else throw new Error('Unknown source plugin operation');
return output;
}
if (plugin === 'mangapill') {
if (operation === 'catalog') {
if (typeof output.title !== 'string' || !Array.isArray(output.chapters)
|| !Number.isInteger(output.chapterLinks) || output.chapterLinks < 0) throw new Error('Invalid MangaPill catalog');
for (const chapter of output.chapters) {
if (typeof chapter?.id !== 'string' || !/^\d+-\d+$/.test(chapter.id)
|| !/^\d+(?:\.\d+)?$/.test(chapter.number)) throw new Error('Invalid MangaPill chapter');
mangaPillUrl(chapter.url, 'chapter');
}
} else if (operation === 'pages') {
if (!Array.isArray(output.pages)) throw new Error('Invalid MangaPill pages');
for (const page of output.pages) mangaPillUrl(page, 'page');
} else throw new Error('Unknown MangaPill operation');
return output;
}
if (operation === 'series') {
if (typeof output.title !== 'string' || typeof output.listUrl !== 'string' && output.listUrl !== null) throw new Error('Invalid series parse');
if (output.listUrl !== null) sourceUrl(output.listUrl, '/series/');
scraper/plugins/weebcentral.js
+8
View File
@@ -1,4 +1,5 @@
'use strict';
const { volumeKey } = require('../../chapter-identity');
// First-party, pure parsing functions. This module never fetches remote content.
const decode = value => value.replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/<[^>]+>/g, '').trim();
@@ -17,6 +18,7 @@ function catalog({ html, url }) {
const text = decode(match[3]).replace(/\s+/g, ' ');
const season = Number(text.match(/^S(\d+)\s*-/i)?.[1] || 0) || null;
const regular = text.match(/\b(?:Chapter|Rating|Episode|Night|Log|Act|Scene|Kapitel|File|Battle)\s*[.:]?\s*([0-9]+(?:\.[0-9]+)?)/i)?.[1];
const volume = !regular ? text.match(/^Volume\s+([0-9]+(?:\.[0-9]+)?)(?!\d)/i)?.[1] : null;
const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1];
const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null);
if (number) found.push({
@@ -24,6 +26,12 @@ function catalog({ html, url }) {
...(season ? { book: { number: season, title: `Season ${season}` } } : {}),
url: absolute(url, match[1]), raw: text,
});
else if (volume) found.push({
id: match[2], number: volumeKey(match[2]), title: '', special: true,
label: `Volume ${volume}`, releaseKind: 'volume', volume,
book: { number: Number(volume), title: `Volume ${volume}` },
url: absolute(url, match[1]), raw: text,
});
}
const chapters = [...new Map(found.map(chapter => [chapter.id, chapter])).values()];
const chapterLinks = [...html.matchAll(/<a\b[^>]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length;
scraper/service.js
+1
View File
@@ -21,6 +21,7 @@ const FAST_RATE_SETTINGS = {
const DEFAULT_SOURCE_RATES = {
weebcentral: { ...FAST_RATE_SETTINGS },
mangadex: { ...FAST_RATE_SETTINGS },
mangapill: { ...FAST_RATE_SETTINGS },
};
const DEFAULT_SETTINGS = {
...FAST_RATE_SETTINGS,
scraper/sources/mangapill.js
+29
View File
@@ -0,0 +1,29 @@
'use strict';
const { createHostHttp } = require('../plugins/broker');
const { mangaPillInfo, discoverMangaPillCatalog, discoverMangaPillChapter } = require('../adapters/mangapill');
const { runArchive } = require('../jobs/weebcentral-archive');
const { controlledHttp } = require('./common');
const createHttp = options => createHostHttp('mangapill.com', options);
module.exports = {
id: 'mangapill', label: 'MangaPill', match: mangaPillInfo,
async discover(item, options) {
const http = createHttp({ minMs: options.settings.minMs, maxMs: options.settings.maxMs,
maxRetries: 2, onEvent: options.onEvent });
return discoverMangaPillCatalog(item.url, controlledHttp(http, options.shouldStop));
},
run(item, options) {
return runArchive({
source: 'mangapill', sourceLabel: 'MangaPill', sourceId: mangaPillInfo(item.url)?.id,
discoverCatalog: discoverMangaPillCatalog, discoverChapter: discoverMangaPillChapter, createHttp,
seriesUrl: item.url, libraryRoot: options.libraryRoot,
metadataSearch: item.metadataSearch || item.title,
minMs: options.settings.minMs, maxMs: options.settings.maxMs,
chapterMinMs: options.settings.chapterMinMs, chapterMaxMs: options.settings.chapterMaxMs,
shouldStop: options.shouldStop, scope: item.scope, existingPolicy: item.existingPolicy,
libraryMangaId: item.libraryMangaId, onPublished: options.onPublished, onCatalog: options.onCatalog,
});
},
};
scraper/test/mangapill-source.test.js
+61
View File
@@ -0,0 +1,61 @@
'use strict';
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const { adapterFor } = require('../adapters');
const { parse } = require('../plugins/registry');
const { discoverMangaPillCatalog, discoverMangaPillChapter } = require('../adapters/mangapill');
const { validateHostRequest } = require('../plugins/broker');
const { runArchive } = require('../jobs/weebcentral-archive');
const { Canvas } = require('../../tools/png');
const { scanLibrary } = require('../lib/library-inventory');
const seriesUrl = 'https://mangapill.com/manga/552/bleach';
const chapterUrl = 'https://mangapill.com/chapters/552-10001000/bleach-chapter-1';
const imageUrl = 'https://cdn.readdetectiveconan.com/file/mangap/552/10001000/1.jpg';
const catalogHtml = `<h1 class="font-bold">Bleach</h1><div id="chapters"><a href="/chapters/552-10002000/bleach-chapter-2">Chapter 2</a><a href="/chapters/552-10001000/bleach-chapter-1">Chapter 1</a></div>`;
const chapterHtml = `<picture><img class="js-page" data-src="${imageUrl}"/></picture>`;
test('MangaPill discovery stays in its host and keeps ordered source chapter IDs', async () => {
assert.equal(adapterFor(seriesUrl).adapter.id, 'mangapill');
assert.equal(adapterFor('http://mangapill.com/manga/552/bleach'), null);
const http = { async get(url) {
assert.ok([seriesUrl, chapterUrl].includes(url));
return { async text() { return url === seriesUrl ? catalogHtml : chapterHtml; } };
} };
const catalog = await discoverMangaPillCatalog(seriesUrl, http);
assert.deepEqual(catalog.chapters.map(chapter => [chapter.id, chapter.number]),
[['552-10001000', '1'], ['552-10002000', '2']]);
const chapter = await discoverMangaPillChapter(catalog.chapters[0], http);
assert.deepEqual(chapter.pages, [{ url: imageUrl, headers: { referer: chapterUrl } }]);
assert.throws(() => parse('mangapill', 'pages', {
html: '<img class="js-page" data-src="https://other.example/file/mangap/1.jpg">', url: chapterUrl,
}), /page host/);
assert.throws(() => validateHostRequest('https://other.example/manga/552', 'mangapill.com', { accept: 'text/html' }), /left its source host/);
});
test('MangaPill uses the shared checkpoint-safe publisher and skips an existing chapter', async t => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-mangapill-'));
t.after(() => fs.rm(root, { recursive: true, force: true }));
const png = new Canvas(10, 20, [1, 2, 3]).png();
const createHttp = () => ({ async get(url) {
if (url === seriesUrl) return { async text() { return catalogHtml.replace(/<a[^>]*>Chapter 2<\/a>/, ''); } };
if (url === chapterUrl) return { async text() { return chapterHtml; } };
if (url === imageUrl) return { async arrayBuffer() { return png; } };
throw new Error(`Unexpected source request: ${url}`);
} });
const options = { source: 'mangapill', sourceLabel: 'MangaPill', sourceId: '552',
discoverCatalog: discoverMangaPillCatalog, discoverChapter: discoverMangaPillChapter, createHttp,
seriesUrl, libraryRoot: root, scope: { mode: 'missing' },
minMs: 1000, maxMs: 1000, chapterMinMs: 5000, chapterMaxMs: 5000 };
const first = await runArchive(options);
assert.equal(first.publishedCount, 1);
const manga = (await scanLibrary(root)).mangas[0];
assert.equal(manga.ids.mangapill, '552');
assert.equal(manga.chapters[0].sourceKey, 'mangapill');
const second = await runArchive(options);
assert.equal(second.publishedCount, 0);
assert.equal(second.selectedCount, 0);
});
scraper/test/source-plugin.test.js
+1 -1
View File
@@ -50,7 +50,7 @@ test('in-process parser preserves WeebCentral catalog and page discovery', async
});
test('a new source is an explicit adapter, not another container or remote module', () => {
assert.deepEqual(adapters.map(adapter => adapter.id), ['weebcentral', 'mangadex']);
assert.deepEqual(adapters.map(adapter => adapter.id), ['weebcentral', 'mangadex', 'mangapill']);
for (const adapter of adapters) {
assert.equal(typeof adapter.match, 'function');
assert.equal(typeof adapter.discover, 'function');
scraper/test/weebcentral-volumes.test.js
+69
View File
@@ -0,0 +1,69 @@
'use strict';
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const { parseChapters, discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral');
const { parseSpecialFolder } = require('../../chapter-identity');
const { selectByScope } = require('../lib/scope');
const { publishManifest } = require('../lib/publisher');
const { scanLibrary, shouldDownload } = require('../lib/library-inventory');
const reader = require('../../library');
const { Canvas } = require('../../tools/png');
const ids = ['01JHVPNS8PNMSNF4H18ERX3SMK', '01JHVPNS8PNSDS66BEPVHH536F'];
const seriesUrl = 'https://weebcentral.com/series/01J76XY8CF6MA6W5VJWZDNWESZ/Akira';
test('whole-volume releases keep their source IDs and do not masquerade as chapters', async t => {
const html = `<a href="/chapters/${ids[0]}">Volume 1</a><a href="/chapters/${ids[1]}">Volume 2</a>`;
const refs = parseChapters(html, seriesUrl);
assert.equal(refs.length, 2);
assert.deepEqual(refs.map(ref => ref.volume), ['1', '2']);
assert.ok(refs.every(ref => ref.special && ref.releaseKind === 'volume'));
assert.equal(parseSpecialFolder(refs[0].number + ' [WeebCentral]').releaseKind, 'volume');
assert.equal(selectByScope(refs, { mode: 'volume', volume: '2' })[0].id, ids[1]);
assert.equal(selectByScope(refs, { mode: 'chapter', chapter: '1' }).length, 0);
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-volume-'));
t.after(() => fs.rm(root, { recursive: true, force: true }));
const page = path.join(root, '.fixture.png');
await fs.writeFile(page, new Canvas(10, 20, [1, 2, 3]).png());
const manifest = { version: 1, series: { title: 'Volume fixture', direction: 'rtl', sources: [{ name: 'WeebCentral', url: seriesUrl }] },
chapters: [refs[0], { number: '1', source: { name: 'Other', url: 'https://example.org/1' } }].map(ref => ({
...ref, source: ref.source || { name: 'WeebCentral', url: ref.url }, pages: [{ path: page }],
})) };
await publishManifest(manifest, { libraryRoot: root });
const scanned = (await reader.scanLibrary(root)).index.mangas[0];
assert.equal(scanned.chapters.length, 2);
assert.ok(scanned.chapters.some(ch => ch.label === 'Volume 1'));
assert.ok(scanned.chapters.some(ch => ch.label === 'Ch. 1'));
const manga = (await scanLibrary(root)).mangas[0];
assert.equal(shouldDownload(manga, refs[0], 'weebcentral'), false);
assert.equal(shouldDownload(manga, refs[1], 'weebcentral'), true);
assert.equal(shouldDownload(manga, refs[0], 'weebcentral', 'add-source'), false);
const numberedOnly = { ...manga, chapters: manga.chapters.filter(chapter => chapter.releaseKind !== 'volume') };
assert.equal(shouldDownload(numberedOnly, refs[0], 'weebcentral'), false);
assert.equal(shouldDownload(numberedOnly, refs[0], 'weebcentral', 'add-source'), true);
});
test('catalog ordering and chapter discovery preserve whole-volume metadata', async () => {
const html = `<h1>Volume fixture</h1><a href="/chapters/${ids[0]}">Volume 1</a>`
+ `<a href="/chapters/${ids[1]}">Chapter 1</a>`;
const http = { async get(url) {
const body = url === seriesUrl ? html
: url === `https://weebcentral.com/chapters/${ids[0]}` ? '<time datetime="2026-01-01"></time>'
: '<img src="https://images.example/1.jpg">';
return { async text() { return body; } };
} };
const catalog = await discoverWeebCatalog(seriesUrl, http);
assert.equal(catalog.chapters.length, 2);
assert.equal(catalog.chapters[0].number, '1');
assert.equal(catalog.chapters[1].label, 'Volume 1');
const chapter = await discoverWeebChapter(catalog.chapters[1], http);
assert.equal(chapter.releaseKind, 'volume');
assert.equal(chapter.sourceId, ids[0]);
assert.equal(chapter.sourceNumber, '1');
assert.equal(chapter.book.number, 1);
assert.equal(chapter.pages.length, 1);
});