Checkpoint opt-in MangaDex parser worker

This commit is contained in:
Codex committed 2026-10-13 21:57:59 +02:00
1 parent 93f07e838e
commit 1c61686c9b
11 files changed
+298 -24

No files matched your search

+30
View File
@@ -76,6 +76,7 @@ services:
SCRAPER_PORT: "8091" SCRAPER_PORT: "8091"
SCRAPER_TOKEN_FILE: /run/secrets/scraper_token SCRAPER_TOKEN_FILE: /run/secrets/scraper_token
SOURCE_PLUGIN_URL: ${SOURCE_PLUGIN_URL:-} SOURCE_PLUGIN_URL: ${SOURCE_PLUGIN_URL:-}
MANGADEX_PLUGIN_URL: ${MANGADEX_PLUGIN_URL:-}
networks: [backend, egress, plugin_backend] networks: [backend, egress, plugin_backend]
healthcheck: healthcheck:
test: [CMD, node, tools/healthcheck.js, scraper] test: [CMD, node, tools/healthcheck.js, scraper]
@@ -126,6 +127,35 @@ services:
pids_limit: 64 pids_limit: 64
mem_limit: 256m mem_limit: 256m
environment: environment:
SOURCE_PLUGIN_ID: weebcentral
SOURCE_PLUGIN_HOST: 0.0.0.0
SOURCE_PLUGIN_PORT: "8094"
networks: [plugin_backend]
healthcheck:
test: [CMD, node, tools/healthcheck-source-plugin.js]
interval: 30s
timeout: 5s
start_period: 15s
retries: 3
source-plugin-mangadex:
image: ${APP_IMAGE:-mangacircus:local}
build:
context: .
args:
VCS_REF: ${VCS_REF:-development}
profiles: [source-plugins]
command: [node, scraper/plugins/worker.js]
init: true
restart: unless-stopped
read_only: true
tmpfs: [/tmp]
security_opt: [no-new-privileges:true]
cap_drop: [ALL]
pids_limit: 64
mem_limit: 256m
environment:
SOURCE_PLUGIN_ID: mangadex
SOURCE_PLUGIN_HOST: 0.0.0.0 SOURCE_PLUGIN_HOST: 0.0.0.0
SOURCE_PLUGIN_PORT: "8094" SOURCE_PLUGIN_PORT: "8094"
networks: [plugin_backend] networks: [plugin_backend]
+12 -12
View File
@@ -7,23 +7,23 @@ proxy commands to its private HTTP API.
## First-party source parser pilot ## First-party source parser pilot
The WeebCentral parser is now a reviewed first-party module in The WeebCentral and MangaDex parsers have reviewed first-party modules in
`scraper/plugins/weebcentral.js`. By default it runs in the scraper process, `scraper/plugins/`. By default their existing in-process paths remain in use.
preserving the existing deployment. To test process isolation in Compose, start To test process isolation in Compose, start the `source-plugins` profile and
the `source-plugins` profile and set `SOURCE_PLUGIN_URL` for the scraper to set `SOURCE_PLUGIN_URL=http://source-plugin-weebcentral:8094` and/or
`http://source-plugin-weebcentral:8094` (for example via the Compose environment). `MANGADEX_PLUGIN_URL=http://source-plugin-mangadex:8094` for the scraper.
Both settings are required; enabling the URL without a healthy worker causes Each URL selects its source independently. A selected worker that is unavailable
WeebCentral discovery to fail closed and retain its checkpoint. Remove the URL causes that source's discovery to fail closed and retain its checkpoint. Remove
to roll back. Do not restart a live scraper merely to try this option; use the its URL to roll back. Do not restart a live scraper merely to try this option;
normal checkpoint-safe deployment procedure. use the normal checkpoint-safe deployment procedure.
The worker has no published port, egress network, library/data mount, or token. Each worker has no published port, egress network, library/data mount, or token.
Only the existing rate-limited scraper HTTP client fetches remote HTML/images. Only the existing rate-limited scraper HTTP client fetches remote HTML/images.
The worker accepts bounded HTML for four parsing steps over its internal-only The worker accepts bounded HTML for four parsing steps over its internal-only
network. It loads only modules listed in the static first-party registry; there network. It loads only modules listed in the static first-party registry; there
is no runtime plugin download or execution of Uchiyomi packages. This is a is no runtime plugin download or execution of Uchiyomi packages. This is a
parser-isolation pilot, **not yet** a complete generic source-plugin contract, parser-isolation pilot, **not yet** a complete generic source-plugin contract
or migration of the MangaDex runner. Parsed catalog and image URLs are checked or migration of either archive runner. Parsed catalog and image URLs are checked
before use, and parser requests/responses are size-bounded. When the worker is before use, and parser requests/responses are size-bounded. When the worker is
enabled, WeebCentral requests use a pinned-DNS HTTPS transport that validates enabled, WeebCentral requests use a pinned-DNS HTTPS transport that validates
each redirect and stops on access errors; the ordinary in-process path is not each redirect and stops on access errors; the ordinary in-process path is not
+25 -8
View File
@@ -2,6 +2,7 @@
const API = 'https://api.mangadex.org'; const API = 'https://api.mangadex.org';
const { specialKey } = require('../../chapter-identity'); const { specialKey } = require('../../chapter-identity');
const { parseSource } = require('../plugins/client');
function mangaDexInfo(value) { function mangaDexInfo(value) {
let url; let url;
@@ -94,6 +95,10 @@ async function discoverMangaDexCatalog(seriesUrl, http) {
if (!info) throw new Error('Invalid MangaDex title URL'); if (!info) throw new Error('Invalid MangaDex title URL');
const manga = (await jsonGet(http, `${API}/manga/${info.id}`)).data; const manga = (await jsonGet(http, `${API}/manga/${info.id}`)).data;
if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found'); if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
const isolated = !!process.env.MANGADEX_PLUGIN_URL;
const pluginMetadata = isolated
? await parseSource('mangadex', 'metadata', { html: JSON.stringify(manga), url: `${API}/manga/${info.id}` })
: null;
const releases = []; const releases = [];
let offset = 0; let offset = 0;
@@ -103,28 +108,37 @@ async function discoverMangaDexCatalog(seriesUrl, http) {
params.append('includes[]', 'scanlation_group'); params.append('includes[]', 'scanlation_group');
params.append('order[chapter]', 'asc'); params.append('order[chapter]', 'asc');
const page = await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`); const page = await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`);
releases.push(...(page.data || [])); if (isolated) {
const parsed = await parseSource('mangadex', 'feed', { html: JSON.stringify(page.data || []), url: `${API}/manga/${info.id}/feed` });
releases.push(...parsed.releases);
} else releases.push(...(page.data || []));
offset += page.data?.length || 0; offset += page.data?.length || 0;
if (!page.data?.length || offset >= Number(page.total || 0)) break; if (!page.data?.length || offset >= Number(page.total || 0)) break;
} }
const title = localizedTitle(manga.attributes); const title = isolated ? pluginMetadata.title : localizedTitle(manga.attributes);
const specialLabel = /pre.serialization|pilot/i.test(title) ? 'Pilot' const specialLabel = isolated ? pluginMetadata.specialLabel : /pre.serialization|pilot/i.test(title) ? 'Pilot'
: (manga.attributes?.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special'; : (manga.attributes?.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
const chapters = selectChapterReleases(releases, specialLabel); const pluginCatalog = async externalReleases => parseSource('mangadex', 'catalog', {
html: JSON.stringify({ attributes: pluginMetadata.availabilityAttributes, releases, specialLabel, externalReleases }),
url: `${API}/manga/${info.id}`,
});
let parsed = isolated ? await pluginCatalog(null) : null;
const chapters = isolated ? parsed.chapters : selectChapterReleases(releases, specialLabel);
// External-only is a separate feed mode, not a combined hosted+external catalog. // External-only is a separate feed mode, not a combined hosted+external catalog.
let externalReleases = null; let externalReleases = null;
if (!chapters.length) { if (!chapters.length) {
const params = new URLSearchParams({ limit: '1', includeFutureUpdates: '0', includeExternalUrl: '1' }); const params = new URLSearchParams({ limit: '1', includeFutureUpdates: '0', includeExternalUrl: '1' });
params.append('translatedLanguage[]', 'en'); params.append('translatedLanguage[]', 'en');
externalReleases = Number((await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`)).total || 0); externalReleases = Number((await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`)).total || 0);
if (isolated) parsed = await pluginCatalog(externalReleases);
} }
return { return {
id: info.id, id: info.id,
title, title,
originalLanguage: manga.attributes?.originalLanguage || null, originalLanguage: isolated ? pluginMetadata.originalLanguage : manga.attributes?.originalLanguage || null,
links: manga.attributes?.links || {}, links: isolated ? pluginMetadata.links : manga.attributes?.links || {},
availability: availabilitySummary(manga.attributes || {}, releases, chapters, externalReleases), availability: isolated ? parsed.availability : availabilitySummary(manga.attributes || {}, releases, chapters, externalReleases),
chapters, chapters,
}; };
} }
@@ -147,11 +161,14 @@ function pageUrls(atHome, chapterUrl, expectedPages) {
async function discoverMangaDexChapter(ref, http) { async function discoverMangaDexChapter(ref, http) {
const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`); const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`);
const groupName = ref.group?.name ? ` · ${ref.group.name}` : ''; const groupName = ref.group?.name ? ` · ${ref.group.name}` : '';
const parsed = process.env.MANGADEX_PLUGIN_URL
? await parseSource('mangadex', 'atHome', { html: JSON.stringify({ response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }), url: `${API}/at-home/server/${ref.id}` })
: null;
return { return {
number: ref.number, title: ref.title || '', added: ref.added, number: ref.number, title: ref.title || '', added: ref.added,
...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}), ...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}),
source: { name: `MangaDex${groupName}`, url: ref.url }, source: { name: `MangaDex${groupName}`, url: ref.url },
pages: pageUrls(atHome, ref.url, ref.pages), pages: parsed ? parsed.pages : pageUrls(atHome, ref.url, ref.pages),
}; };
} }
+1 -1
View File
@@ -16,7 +16,7 @@ async function boundedJson(response) {
} }
async function parseSource(plugin, operation, input) { async function parseSource(plugin, operation, input) {
const endpoint = process.env.SOURCE_PLUGIN_URL; const endpoint = plugin === 'mangadex' ? process.env.MANGADEX_PLUGIN_URL : process.env.SOURCE_PLUGIN_URL;
if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input)); if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input));
const url = new URL('/v1/parse', endpoint); const url = new URL('/v1/parse', endpoint);
if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint'); if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint');
+132
View File
@@ -0,0 +1,132 @@
'use strict';
const { specialKey } = require('../../chapter-identity');
function localizedTitle(attributes = {}) {
const titles = attributes.title || {};
return titles.en || titles['ja-ro'] || titles.ja || Object.values(titles).find(Boolean) || 'Untitled';
}
function chapterNumber(value) {
const raw = String(value ?? '').trim();
if (!/^\d+(?:\.\d+)?$/.test(raw)) return null;
const [whole, decimal = ''] = raw.split('.');
const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0';
const normalizedDecimal = decimal.replace(/0+$/, '');
return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole;
}
function groupFor(resource) {
const relation = (resource.relationships || []).find(item => item.type === 'scanlation_group');
return relation ? { id: relation.id, name: relation.attributes?.name || 'Unknown group' } : null;
}
function releaseRank(resource) {
const group = groupFor(resource)?.name || '';
const credited = /official|license/i.test(group) ? 0 : group ? 1 : 2;
const date = Date.parse(resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt || '') || Number.MAX_SAFE_INTEGER;
return [credited, date, resource.id];
}
function compareRank(left, right) {
const a = releaseRank(left), b = releaseRank(right);
return a[0] - b[0] || a[1] - b[1] || String(a[2]).localeCompare(String(b[2]));
}
function selectChapterReleases(resources = [], specialLabel = 'Special') {
const selected = new Map();
for (const resource of resources) {
if (resource.attributes?.externalUrl || Number(resource.attributes?.pages || 0) < 1) continue;
const number = chapterNumber(resource.attributes?.chapter) || specialKey(resource.id);
const prior = selected.get(number);
if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource);
}
return [...selected.entries()].map(([number, resource]) => {
const group = groupFor(resource);
return {
id: resource.id, number, title: resource.attributes?.title || '',
...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}),
...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}),
pages: Number(resource.attributes?.pages || 0),
added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt,
group,
url: `https://mangadex.org/chapter/${resource.id}`,
};
}).sort((a, b) => Number(!!a.special) - Number(!!b.special) || (a.special ? a.number.localeCompare(b.number) : Number(a.number) - Number(b.number) || a.number.localeCompare(b.number)));
}
function availabilitySummary(attributes, releases, chapters, externalReleases = null) {
const numbered = chapters.filter(ref => !ref.special);
const expected = chapterNumber(attributes.lastChapter);
const maximum = numbered.length ? Math.max(...numbered.map(ref => Number(ref.number))) : null;
const expectedCount = expected && /^\d+$/.test(expected) ? Number(expected) : null;
const sparse = expectedCount && numbered.length < expectedCount;
const reason = chapters.length ? null : externalReleases > 0 ? 'external-only'
: Array.isArray(attributes.availableTranslatedLanguages) && !attributes.availableTranslatedLanguages.includes('en') ? 'no-english' : 'no-hosted-pages';
const message = reason === 'external-only' ? `MangaDex lists ${externalReleases} English external-link releases, but no hosted English pages. External links are not downloaded.`
: reason === 'no-english' ? 'This MangaDex edition has no English releases listed.'
: reason ? 'This MangaDex edition has no hosted English releases with pages.' : null;
return { language: 'en', hostedReleases: releases.filter(r => !r.attributes?.externalUrl && Number(r.attributes?.pages) > 0).length,
selectedChapters: chapters.length, numberedChapters: numbered.length, specialReleases: chapters.length - numbered.length,
externalReleases, expectedLastChapter: expected, highestHostedChapter: maximum,
coverage: expected && (maximum === null || maximum < Number(expected) || sparse) ? 'partial' : 'unknown', reason, message };
}
function pageUrls(atHome, chapterUrl, expectedPages) {
const base = new URL(atHome.baseUrl);
if (base.protocol !== 'https:' || !(base.hostname === 'uploads.mangadex.org' || base.hostname.endsWith('.mangadex.network'))) {
throw new Error(`Unexpected MangaDex@Home host: ${base.hostname}`);
}
const hash = String(atHome.chapter?.hash || '');
const files = atHome.chapter?.data;
if (!hash || !Array.isArray(files) || !files.length) throw new Error('MangaDex@Home returned no pages');
if (expectedPages && files.length !== expectedPages) throw new Error(`MangaDex page count changed: expected ${expectedPages}, received ${files.length}`);
return files.map(file => ({
url: new URL(`/data/${encodeURIComponent(hash)}/${encodeURIComponent(file)}`, base).href,
headers: { referer: chapterUrl },
}));
}
function metadata({ html }) {
const manga = JSON.parse(html);
if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
const attributes = manga.attributes || {};
const title = localizedTitle(attributes);
const specialLabel = /pre.serialization|pilot/i.test(title) ? 'Pilot'
: (attributes.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
return { title, originalLanguage: attributes.originalLanguage || null, links: attributes.links || {},
specialLabel, availabilityAttributes: {
lastChapter: attributes.lastChapter ?? null,
availableTranslatedLanguages: attributes.availableTranslatedLanguages ?? null,
} };
}
function feed({ html }) {
const releases = JSON.parse(html);
if (!Array.isArray(releases)) throw new Error('Invalid MangaDex feed');
return { releases: releases.map(resource => ({
id: resource.id,
attributes: {
chapter: resource.attributes?.chapter, volume: resource.attributes?.volume,
title: resource.attributes?.title, pages: resource.attributes?.pages,
publishAt: resource.attributes?.publishAt, readableAt: resource.attributes?.readableAt,
createdAt: resource.attributes?.createdAt, externalUrl: resource.attributes?.externalUrl,
},
relationships: (resource.relationships || []).filter(relation => relation.type === 'scanlation_group'),
})) };
}
function catalog({ html }) {
const { attributes, releases, specialLabel, externalReleases } = JSON.parse(html);
if (!attributes || !Array.isArray(releases)) throw new Error('Invalid MangaDex catalog input');
const chapters = selectChapterReleases(releases, specialLabel);
return { chapters, availability: availabilitySummary(attributes, releases, chapters, externalReleases) };
}
function atHome({ html }) {
const { response, chapterUrl, expectedPages } = JSON.parse(html);
return { pages: pageUrls(response, chapterUrl, expectedPages) };
}
module.exports = { localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary,
pageUrls, metadata, feed, catalog, atHome };
+7
View File
@@ -10,6 +10,13 @@ const manifests = Object.freeze({
provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied', provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied',
operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']), operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']),
}), }),
mangadex: Object.freeze({
id: 'mangadex', version: 1, label: 'MangaDex', status: 'pilot',
sourceHosts: Object.freeze(['api.mangadex.org']),
implementation: 'first-party',
provenance: 'MangaCircus existing MangaDex adapter; no Uchiyomi code copied',
operations: Object.freeze(['metadata', 'feed', 'catalog', 'atHome']),
}),
}); });
module.exports = { manifests }; module.exports = { manifests };
+4 -1
View File
@@ -2,7 +2,10 @@
// Reviewed first-party modules only. No filesystem scanning or runtime installs. // Reviewed first-party modules only. No filesystem scanning or runtime installs.
const { manifests } = require('./manifest'); const { manifests } = require('./manifest');
const plugins = Object.freeze({ weebcentral: require('./weebcentral') }); const plugins = Object.freeze({
weebcentral: require('./weebcentral'),
mangadex: require('./mangadex'),
});
function parse(plugin, operation, input) { function parse(plugin, operation, input) {
const manifest = manifests[plugin]; const manifest = manifests[plugin];
+27 -1
View File
@@ -21,10 +21,36 @@ function imageUrl(value) {
return url.href; return url.href;
} }
function mangaDexUrl(value, kind) {
const url = new URL(value);
if (url.protocol !== 'https:' || url.username || url.password || url.port) throw new Error('Invalid MangaDex plugin URL');
if (kind === 'chapter' && (url.hostname !== 'mangadex.org' || !url.pathname.startsWith('/chapter/'))) {
throw new Error('Unexpected MangaDex chapter URL');
}
if (kind === 'page' && !(url.hostname === 'uploads.mangadex.org' || url.hostname.endsWith('.mangadex.network'))) {
throw new Error('Unexpected MangaDex page host');
}
return url.href;
}
function validate(plugin, operation, output) { function validate(plugin, operation, output) {
if (plugin !== 'weebcentral' || !output || typeof output !== 'object' || Array.isArray(output)) { if (!['weebcentral', 'mangadex'].includes(plugin) || !output || typeof output !== 'object' || Array.isArray(output)) {
throw new Error('Invalid source plugin output'); throw new Error('Invalid source plugin output');
} }
if (plugin === 'mangadex') {
if (operation === 'metadata') {
if (typeof output.title !== 'string' || !output.availabilityAttributes || typeof output.specialLabel !== 'string') throw new Error('Invalid MangaDex metadata');
} else if (operation === 'feed') {
if (!Array.isArray(output.releases) || output.releases.some(ref => typeof ref?.id !== 'string' || !ref.attributes)) throw new Error('Invalid MangaDex feed');
} else if (operation === 'catalog') {
if (!Array.isArray(output.chapters) || !output.availability) throw new Error('Invalid MangaDex catalog');
for (const chapter of output.chapters) mangaDexUrl(chapter.url, 'chapter');
} else if (operation === 'atHome') {
if (!Array.isArray(output.pages) || !output.pages.length) throw new Error('Invalid MangaDex pages');
for (const page of output.pages) mangaDexUrl(page.url, 'page');
} else throw new Error('Unknown source plugin operation');
return output;
}
if (operation === 'series') { if (operation === 'series') {
if (typeof output.title !== 'string' || typeof output.listUrl !== 'string' && output.listUrl !== null) throw new Error('Invalid series parse'); if (typeof output.title !== 'string' || typeof output.listUrl !== 'string' && output.listUrl !== null) throw new Error('Invalid series parse');
if (output.listUrl !== null) sourceUrl(output.listUrl, '/series/'); if (output.listUrl !== null) sourceUrl(output.listUrl, '/series/');
+2 -1
View File
@@ -4,7 +4,7 @@ const http = require('node:http');
const { parse } = require('./registry'); const { parse } = require('./registry');
const MAX_BODY_BYTES = 2 * 1024 * 1024; const MAX_BODY_BYTES = 2 * 1024 * 1024;
function createServer() { function createServer(pluginId = process.env.SOURCE_PLUGIN_ID || null) {
return http.createServer((request, response) => { return http.createServer((request, response) => {
if (request.method === 'GET' && request.url === '/health') { if (request.method === 'GET' && request.url === '/health') {
response.writeHead(200, { 'content-type': 'application/json' }); response.writeHead(200, { 'content-type': 'application/json' });
@@ -30,6 +30,7 @@ function createServer() {
if (bytes > MAX_BODY_BYTES) return; if (bytes > MAX_BODY_BYTES) return;
try { try {
const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8')); const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8'));
if (pluginId && plugin !== pluginId) throw new Error('Plugin worker does not serve this source');
const output = parse(plugin, operation, input); const output = parse(plugin, operation, input);
response.writeHead(200, { 'content-type': 'application/json' }); response.writeHead(200, { 'content-type': 'application/json' });
response.end(JSON.stringify(output)); response.end(JSON.stringify(output));
@@ -0,0 +1,55 @@
'use strict';
const test = require('node:test');
const assert = require('node:assert/strict');
const { createServer } = require('../plugins/worker');
const { discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex');
const mangaId = '123e4567-e89b-42d3-a456-426614174000';
const chapterIds = [
'123e4567-e89b-42d3-a456-426614174001',
'123e4567-e89b-42d3-a456-426614174002',
];
const seriesUrl = `https://mangadex.org/title/${mangaId}/Fixture`;
const manga = { data: { type: 'manga', attributes: {
title: { en: 'Fixture' }, originalLanguage: 'ja', lastChapter: '2',
availableTranslatedLanguages: ['en'], links: {}, tags: [],
} } };
const feed = { total: 2, data: chapterIds.map((id, index) => ({
id, attributes: { chapter: String(index + 1), title: `Part ${index + 1}`, pages: 2,
publishAt: `2026-01-0${index + 1}T00:00:00Z` }, relationships: [],
})) };
function fakeHttp() {
return { async get(url) {
const body = url.includes('/at-home/server/') ? {
baseUrl: 'https://uploads.mangadex.org', chapter: { hash: 'HASH', data: ['1.jpg', '2.jpg'] },
} : url.includes('/feed?') ? feed : manga;
return { json: async () => body };
} };
}
test('MangaDex isolated parser matches the existing catalog and page behavior', async () => {
const server = createServer('mangadex');
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const previous = process.env.MANGADEX_PLUGIN_URL;
delete process.env.MANGADEX_PLUGIN_URL;
try {
const baseline = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
const baselineChapter = await discoverMangaDexChapter(baseline.chapters[0], fakeHttp());
process.env.MANGADEX_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
const isolated = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
const isolatedChapter = await discoverMangaDexChapter(isolated.chapters[0], fakeHttp());
assert.deepEqual(isolated, baseline);
assert.deepEqual(isolatedChapter, baselineChapter);
const rejected = await fetch(`${process.env.MANGADEX_PLUGIN_URL}/v1/parse`, {
method: 'POST', headers: { 'content-type': 'application/json' },
body: JSON.stringify({ plugin: 'weebcentral', operation: 'series', input: { html: '<h1>X</h1>', url: 'https://weebcentral.com/series/X' } }),
});
assert.equal(rejected.status, 400);
} finally {
if (previous === undefined) delete process.env.MANGADEX_PLUGIN_URL;
else process.env.MANGADEX_PLUGIN_URL = previous;
await new Promise(resolve => server.close(resolve));
}
});
+3
View File
@@ -9,6 +9,9 @@ to do; the diagram describes the target rather than the current implementation.
Parser-result URL validation and bounded worker messages were added next. The Parser-result URL validation and bounded worker messages were added next. The
opt-in WeebCentral path now validates redirect hops and pins public DNS answers; opt-in WeebCentral path now validates redirect hops and pins public DNS answers;
source-specific image CDN allowlists and generic broker integration remain pending. source-specific image CDN allowlists and generic broker integration remain pending.
MangaDex metadata, feed selection, catalog availability, and at-home page parsing
now have a second opt-in worker path with fixture parity; its transport still
uses the existing API client and runner.
## Decision ## Decision