Checkpoint opt-in MangaDex parser worker
This commit is contained in:
1 parent
93f07e838e
commit
1c61686c9b
11 files changed
+298
-24
No files matched your search
@@ -76,6 +76,7 @@ services:
|
|||||||
SCRAPER_PORT: "8091"
|
SCRAPER_PORT: "8091"
|
||||||
SCRAPER_TOKEN_FILE: /run/secrets/scraper_token
|
SCRAPER_TOKEN_FILE: /run/secrets/scraper_token
|
||||||
SOURCE_PLUGIN_URL: ${SOURCE_PLUGIN_URL:-}
|
SOURCE_PLUGIN_URL: ${SOURCE_PLUGIN_URL:-}
|
||||||
|
MANGADEX_PLUGIN_URL: ${MANGADEX_PLUGIN_URL:-}
|
||||||
networks: [backend, egress, plugin_backend]
|
networks: [backend, egress, plugin_backend]
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: [CMD, node, tools/healthcheck.js, scraper]
|
test: [CMD, node, tools/healthcheck.js, scraper]
|
||||||
@@ -126,6 +127,35 @@ services:
|
|||||||
pids_limit: 64
|
pids_limit: 64
|
||||||
mem_limit: 256m
|
mem_limit: 256m
|
||||||
environment:
|
environment:
|
||||||
|
SOURCE_PLUGIN_ID: weebcentral
|
||||||
|
SOURCE_PLUGIN_HOST: 0.0.0.0
|
||||||
|
SOURCE_PLUGIN_PORT: "8094"
|
||||||
|
networks: [plugin_backend]
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD, node, tools/healthcheck-source-plugin.js]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
start_period: 15s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
source-plugin-mangadex:
|
||||||
|
image: ${APP_IMAGE:-mangacircus:local}
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
args:
|
||||||
|
VCS_REF: ${VCS_REF:-development}
|
||||||
|
profiles: [source-plugins]
|
||||||
|
command: [node, scraper/plugins/worker.js]
|
||||||
|
init: true
|
||||||
|
restart: unless-stopped
|
||||||
|
read_only: true
|
||||||
|
tmpfs: [/tmp]
|
||||||
|
security_opt: [no-new-privileges:true]
|
||||||
|
cap_drop: [ALL]
|
||||||
|
pids_limit: 64
|
||||||
|
mem_limit: 256m
|
||||||
|
environment:
|
||||||
|
SOURCE_PLUGIN_ID: mangadex
|
||||||
SOURCE_PLUGIN_HOST: 0.0.0.0
|
SOURCE_PLUGIN_HOST: 0.0.0.0
|
||||||
SOURCE_PLUGIN_PORT: "8094"
|
SOURCE_PLUGIN_PORT: "8094"
|
||||||
networks: [plugin_backend]
|
networks: [plugin_backend]
|
||||||
|
|||||||
+12
-12
@@ -7,23 +7,23 @@ proxy commands to its private HTTP API.
|
|||||||
|
|
||||||
## First-party source parser pilot
|
## First-party source parser pilot
|
||||||
|
|
||||||
The WeebCentral parser is now a reviewed first-party module in
|
The WeebCentral and MangaDex parsers have reviewed first-party modules in
|
||||||
`scraper/plugins/weebcentral.js`. By default it runs in the scraper process,
|
`scraper/plugins/`. By default their existing in-process paths remain in use.
|
||||||
preserving the existing deployment. To test process isolation in Compose, start
|
To test process isolation in Compose, start the `source-plugins` profile and
|
||||||
the `source-plugins` profile and set `SOURCE_PLUGIN_URL` for the scraper to
|
set `SOURCE_PLUGIN_URL=http://source-plugin-weebcentral:8094` and/or
|
||||||
`http://source-plugin-weebcentral:8094` (for example via the Compose environment).
|
`MANGADEX_PLUGIN_URL=http://source-plugin-mangadex:8094` for the scraper.
|
||||||
Both settings are required; enabling the URL without a healthy worker causes
|
Each URL selects its source independently. A selected worker that is unavailable
|
||||||
WeebCentral discovery to fail closed and retain its checkpoint. Remove the URL
|
causes that source's discovery to fail closed and retain its checkpoint. Remove
|
||||||
to roll back. Do not restart a live scraper merely to try this option; use the
|
its URL to roll back. Do not restart a live scraper merely to try this option;
|
||||||
normal checkpoint-safe deployment procedure.
|
use the normal checkpoint-safe deployment procedure.
|
||||||
|
|
||||||
The worker has no published port, egress network, library/data mount, or token.
|
Each worker has no published port, egress network, library/data mount, or token.
|
||||||
Only the existing rate-limited scraper HTTP client fetches remote HTML/images.
|
Only the existing rate-limited scraper HTTP client fetches remote HTML/images.
|
||||||
The worker accepts bounded HTML for four parsing steps over its internal-only
|
The worker accepts bounded HTML for four parsing steps over its internal-only
|
||||||
network. It loads only modules listed in the static first-party registry; there
|
network. It loads only modules listed in the static first-party registry; there
|
||||||
is no runtime plugin download or execution of Uchiyomi packages. This is a
|
is no runtime plugin download or execution of Uchiyomi packages. This is a
|
||||||
parser-isolation pilot, **not yet** a complete generic source-plugin contract,
|
parser-isolation pilot, **not yet** a complete generic source-plugin contract
|
||||||
or migration of the MangaDex runner. Parsed catalog and image URLs are checked
|
or migration of either archive runner. Parsed catalog and image URLs are checked
|
||||||
before use, and parser requests/responses are size-bounded. When the worker is
|
before use, and parser requests/responses are size-bounded. When the worker is
|
||||||
enabled, WeebCentral requests use a pinned-DNS HTTPS transport that validates
|
enabled, WeebCentral requests use a pinned-DNS HTTPS transport that validates
|
||||||
each redirect and stops on access errors; the ordinary in-process path is not
|
each redirect and stops on access errors; the ordinary in-process path is not
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
const API = 'https://api.mangadex.org';
|
const API = 'https://api.mangadex.org';
|
||||||
const { specialKey } = require('../../chapter-identity');
|
const { specialKey } = require('../../chapter-identity');
|
||||||
|
const { parseSource } = require('../plugins/client');
|
||||||
|
|
||||||
function mangaDexInfo(value) {
|
function mangaDexInfo(value) {
|
||||||
let url;
|
let url;
|
||||||
@@ -94,6 +95,10 @@ async function discoverMangaDexCatalog(seriesUrl, http) {
|
|||||||
if (!info) throw new Error('Invalid MangaDex title URL');
|
if (!info) throw new Error('Invalid MangaDex title URL');
|
||||||
const manga = (await jsonGet(http, `${API}/manga/${info.id}`)).data;
|
const manga = (await jsonGet(http, `${API}/manga/${info.id}`)).data;
|
||||||
if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
|
if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
|
||||||
|
const isolated = !!process.env.MANGADEX_PLUGIN_URL;
|
||||||
|
const pluginMetadata = isolated
|
||||||
|
? await parseSource('mangadex', 'metadata', { html: JSON.stringify(manga), url: `${API}/manga/${info.id}` })
|
||||||
|
: null;
|
||||||
|
|
||||||
const releases = [];
|
const releases = [];
|
||||||
let offset = 0;
|
let offset = 0;
|
||||||
@@ -103,28 +108,37 @@ async function discoverMangaDexCatalog(seriesUrl, http) {
|
|||||||
params.append('includes[]', 'scanlation_group');
|
params.append('includes[]', 'scanlation_group');
|
||||||
params.append('order[chapter]', 'asc');
|
params.append('order[chapter]', 'asc');
|
||||||
const page = await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`);
|
const page = await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`);
|
||||||
releases.push(...(page.data || []));
|
if (isolated) {
|
||||||
|
const parsed = await parseSource('mangadex', 'feed', { html: JSON.stringify(page.data || []), url: `${API}/manga/${info.id}/feed` });
|
||||||
|
releases.push(...parsed.releases);
|
||||||
|
} else releases.push(...(page.data || []));
|
||||||
offset += page.data?.length || 0;
|
offset += page.data?.length || 0;
|
||||||
if (!page.data?.length || offset >= Number(page.total || 0)) break;
|
if (!page.data?.length || offset >= Number(page.total || 0)) break;
|
||||||
}
|
}
|
||||||
|
|
||||||
const title = localizedTitle(manga.attributes);
|
const title = isolated ? pluginMetadata.title : localizedTitle(manga.attributes);
|
||||||
const specialLabel = /pre.serialization|pilot/i.test(title) ? 'Pilot'
|
const specialLabel = isolated ? pluginMetadata.specialLabel : /pre.serialization|pilot/i.test(title) ? 'Pilot'
|
||||||
: (manga.attributes?.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
|
: (manga.attributes?.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
|
||||||
const chapters = selectChapterReleases(releases, specialLabel);
|
const pluginCatalog = async externalReleases => parseSource('mangadex', 'catalog', {
|
||||||
|
html: JSON.stringify({ attributes: pluginMetadata.availabilityAttributes, releases, specialLabel, externalReleases }),
|
||||||
|
url: `${API}/manga/${info.id}`,
|
||||||
|
});
|
||||||
|
let parsed = isolated ? await pluginCatalog(null) : null;
|
||||||
|
const chapters = isolated ? parsed.chapters : selectChapterReleases(releases, specialLabel);
|
||||||
// External-only is a separate feed mode, not a combined hosted+external catalog.
|
// External-only is a separate feed mode, not a combined hosted+external catalog.
|
||||||
let externalReleases = null;
|
let externalReleases = null;
|
||||||
if (!chapters.length) {
|
if (!chapters.length) {
|
||||||
const params = new URLSearchParams({ limit: '1', includeFutureUpdates: '0', includeExternalUrl: '1' });
|
const params = new URLSearchParams({ limit: '1', includeFutureUpdates: '0', includeExternalUrl: '1' });
|
||||||
params.append('translatedLanguage[]', 'en');
|
params.append('translatedLanguage[]', 'en');
|
||||||
externalReleases = Number((await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`)).total || 0);
|
externalReleases = Number((await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`)).total || 0);
|
||||||
|
if (isolated) parsed = await pluginCatalog(externalReleases);
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
id: info.id,
|
id: info.id,
|
||||||
title,
|
title,
|
||||||
originalLanguage: manga.attributes?.originalLanguage || null,
|
originalLanguage: isolated ? pluginMetadata.originalLanguage : manga.attributes?.originalLanguage || null,
|
||||||
links: manga.attributes?.links || {},
|
links: isolated ? pluginMetadata.links : manga.attributes?.links || {},
|
||||||
availability: availabilitySummary(manga.attributes || {}, releases, chapters, externalReleases),
|
availability: isolated ? parsed.availability : availabilitySummary(manga.attributes || {}, releases, chapters, externalReleases),
|
||||||
chapters,
|
chapters,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -147,11 +161,14 @@ function pageUrls(atHome, chapterUrl, expectedPages) {
|
|||||||
async function discoverMangaDexChapter(ref, http) {
|
async function discoverMangaDexChapter(ref, http) {
|
||||||
const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`);
|
const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`);
|
||||||
const groupName = ref.group?.name ? ` · ${ref.group.name}` : '';
|
const groupName = ref.group?.name ? ` · ${ref.group.name}` : '';
|
||||||
|
const parsed = process.env.MANGADEX_PLUGIN_URL
|
||||||
|
? await parseSource('mangadex', 'atHome', { html: JSON.stringify({ response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }), url: `${API}/at-home/server/${ref.id}` })
|
||||||
|
: null;
|
||||||
return {
|
return {
|
||||||
number: ref.number, title: ref.title || '', added: ref.added,
|
number: ref.number, title: ref.title || '', added: ref.added,
|
||||||
...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}),
|
...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}),
|
||||||
source: { name: `MangaDex${groupName}`, url: ref.url },
|
source: { name: `MangaDex${groupName}`, url: ref.url },
|
||||||
pages: pageUrls(atHome, ref.url, ref.pages),
|
pages: parsed ? parsed.pages : pageUrls(atHome, ref.url, ref.pages),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ async function boundedJson(response) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function parseSource(plugin, operation, input) {
|
async function parseSource(plugin, operation, input) {
|
||||||
const endpoint = process.env.SOURCE_PLUGIN_URL;
|
const endpoint = plugin === 'mangadex' ? process.env.MANGADEX_PLUGIN_URL : process.env.SOURCE_PLUGIN_URL;
|
||||||
if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input));
|
if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input));
|
||||||
const url = new URL('/v1/parse', endpoint);
|
const url = new URL('/v1/parse', endpoint);
|
||||||
if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint');
|
if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint');
|
||||||
|
|||||||
@@ -0,0 +1,132 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
|
const { specialKey } = require('../../chapter-identity');
|
||||||
|
|
||||||
|
function localizedTitle(attributes = {}) {
|
||||||
|
const titles = attributes.title || {};
|
||||||
|
return titles.en || titles['ja-ro'] || titles.ja || Object.values(titles).find(Boolean) || 'Untitled';
|
||||||
|
}
|
||||||
|
|
||||||
|
function chapterNumber(value) {
|
||||||
|
const raw = String(value ?? '').trim();
|
||||||
|
if (!/^\d+(?:\.\d+)?$/.test(raw)) return null;
|
||||||
|
const [whole, decimal = ''] = raw.split('.');
|
||||||
|
const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0';
|
||||||
|
const normalizedDecimal = decimal.replace(/0+$/, '');
|
||||||
|
return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole;
|
||||||
|
}
|
||||||
|
|
||||||
|
function groupFor(resource) {
|
||||||
|
const relation = (resource.relationships || []).find(item => item.type === 'scanlation_group');
|
||||||
|
return relation ? { id: relation.id, name: relation.attributes?.name || 'Unknown group' } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function releaseRank(resource) {
|
||||||
|
const group = groupFor(resource)?.name || '';
|
||||||
|
const credited = /official|license/i.test(group) ? 0 : group ? 1 : 2;
|
||||||
|
const date = Date.parse(resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt || '') || Number.MAX_SAFE_INTEGER;
|
||||||
|
return [credited, date, resource.id];
|
||||||
|
}
|
||||||
|
|
||||||
|
function compareRank(left, right) {
|
||||||
|
const a = releaseRank(left), b = releaseRank(right);
|
||||||
|
return a[0] - b[0] || a[1] - b[1] || String(a[2]).localeCompare(String(b[2]));
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectChapterReleases(resources = [], specialLabel = 'Special') {
|
||||||
|
const selected = new Map();
|
||||||
|
for (const resource of resources) {
|
||||||
|
if (resource.attributes?.externalUrl || Number(resource.attributes?.pages || 0) < 1) continue;
|
||||||
|
const number = chapterNumber(resource.attributes?.chapter) || specialKey(resource.id);
|
||||||
|
const prior = selected.get(number);
|
||||||
|
if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource);
|
||||||
|
}
|
||||||
|
return [...selected.entries()].map(([number, resource]) => {
|
||||||
|
const group = groupFor(resource);
|
||||||
|
return {
|
||||||
|
id: resource.id, number, title: resource.attributes?.title || '',
|
||||||
|
...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}),
|
||||||
|
...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}),
|
||||||
|
pages: Number(resource.attributes?.pages || 0),
|
||||||
|
added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt,
|
||||||
|
group,
|
||||||
|
url: `https://mangadex.org/chapter/${resource.id}`,
|
||||||
|
};
|
||||||
|
}).sort((a, b) => Number(!!a.special) - Number(!!b.special) || (a.special ? a.number.localeCompare(b.number) : Number(a.number) - Number(b.number) || a.number.localeCompare(b.number)));
|
||||||
|
}
|
||||||
|
|
||||||
|
function availabilitySummary(attributes, releases, chapters, externalReleases = null) {
|
||||||
|
const numbered = chapters.filter(ref => !ref.special);
|
||||||
|
const expected = chapterNumber(attributes.lastChapter);
|
||||||
|
const maximum = numbered.length ? Math.max(...numbered.map(ref => Number(ref.number))) : null;
|
||||||
|
const expectedCount = expected && /^\d+$/.test(expected) ? Number(expected) : null;
|
||||||
|
const sparse = expectedCount && numbered.length < expectedCount;
|
||||||
|
const reason = chapters.length ? null : externalReleases > 0 ? 'external-only'
|
||||||
|
: Array.isArray(attributes.availableTranslatedLanguages) && !attributes.availableTranslatedLanguages.includes('en') ? 'no-english' : 'no-hosted-pages';
|
||||||
|
const message = reason === 'external-only' ? `MangaDex lists ${externalReleases} English external-link releases, but no hosted English pages. External links are not downloaded.`
|
||||||
|
: reason === 'no-english' ? 'This MangaDex edition has no English releases listed.'
|
||||||
|
: reason ? 'This MangaDex edition has no hosted English releases with pages.' : null;
|
||||||
|
return { language: 'en', hostedReleases: releases.filter(r => !r.attributes?.externalUrl && Number(r.attributes?.pages) > 0).length,
|
||||||
|
selectedChapters: chapters.length, numberedChapters: numbered.length, specialReleases: chapters.length - numbered.length,
|
||||||
|
externalReleases, expectedLastChapter: expected, highestHostedChapter: maximum,
|
||||||
|
coverage: expected && (maximum === null || maximum < Number(expected) || sparse) ? 'partial' : 'unknown', reason, message };
|
||||||
|
}
|
||||||
|
|
||||||
|
function pageUrls(atHome, chapterUrl, expectedPages) {
|
||||||
|
const base = new URL(atHome.baseUrl);
|
||||||
|
if (base.protocol !== 'https:' || !(base.hostname === 'uploads.mangadex.org' || base.hostname.endsWith('.mangadex.network'))) {
|
||||||
|
throw new Error(`Unexpected MangaDex@Home host: ${base.hostname}`);
|
||||||
|
}
|
||||||
|
const hash = String(atHome.chapter?.hash || '');
|
||||||
|
const files = atHome.chapter?.data;
|
||||||
|
if (!hash || !Array.isArray(files) || !files.length) throw new Error('MangaDex@Home returned no pages');
|
||||||
|
if (expectedPages && files.length !== expectedPages) throw new Error(`MangaDex page count changed: expected ${expectedPages}, received ${files.length}`);
|
||||||
|
return files.map(file => ({
|
||||||
|
url: new URL(`/data/${encodeURIComponent(hash)}/${encodeURIComponent(file)}`, base).href,
|
||||||
|
headers: { referer: chapterUrl },
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function metadata({ html }) {
|
||||||
|
const manga = JSON.parse(html);
|
||||||
|
if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
|
||||||
|
const attributes = manga.attributes || {};
|
||||||
|
const title = localizedTitle(attributes);
|
||||||
|
const specialLabel = /pre.serialization|pilot/i.test(title) ? 'Pilot'
|
||||||
|
: (attributes.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
|
||||||
|
return { title, originalLanguage: attributes.originalLanguage || null, links: attributes.links || {},
|
||||||
|
specialLabel, availabilityAttributes: {
|
||||||
|
lastChapter: attributes.lastChapter ?? null,
|
||||||
|
availableTranslatedLanguages: attributes.availableTranslatedLanguages ?? null,
|
||||||
|
} };
|
||||||
|
}
|
||||||
|
|
||||||
|
function feed({ html }) {
|
||||||
|
const releases = JSON.parse(html);
|
||||||
|
if (!Array.isArray(releases)) throw new Error('Invalid MangaDex feed');
|
||||||
|
return { releases: releases.map(resource => ({
|
||||||
|
id: resource.id,
|
||||||
|
attributes: {
|
||||||
|
chapter: resource.attributes?.chapter, volume: resource.attributes?.volume,
|
||||||
|
title: resource.attributes?.title, pages: resource.attributes?.pages,
|
||||||
|
publishAt: resource.attributes?.publishAt, readableAt: resource.attributes?.readableAt,
|
||||||
|
createdAt: resource.attributes?.createdAt, externalUrl: resource.attributes?.externalUrl,
|
||||||
|
},
|
||||||
|
relationships: (resource.relationships || []).filter(relation => relation.type === 'scanlation_group'),
|
||||||
|
})) };
|
||||||
|
}
|
||||||
|
|
||||||
|
function catalog({ html }) {
|
||||||
|
const { attributes, releases, specialLabel, externalReleases } = JSON.parse(html);
|
||||||
|
if (!attributes || !Array.isArray(releases)) throw new Error('Invalid MangaDex catalog input');
|
||||||
|
const chapters = selectChapterReleases(releases, specialLabel);
|
||||||
|
return { chapters, availability: availabilitySummary(attributes, releases, chapters, externalReleases) };
|
||||||
|
}
|
||||||
|
|
||||||
|
function atHome({ html }) {
|
||||||
|
const { response, chapterUrl, expectedPages } = JSON.parse(html);
|
||||||
|
return { pages: pageUrls(response, chapterUrl, expectedPages) };
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary,
|
||||||
|
pageUrls, metadata, feed, catalog, atHome };
|
||||||
@@ -10,6 +10,13 @@ const manifests = Object.freeze({
|
|||||||
provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied',
|
provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied',
|
||||||
operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']),
|
operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']),
|
||||||
}),
|
}),
|
||||||
|
mangadex: Object.freeze({
|
||||||
|
id: 'mangadex', version: 1, label: 'MangaDex', status: 'pilot',
|
||||||
|
sourceHosts: Object.freeze(['api.mangadex.org']),
|
||||||
|
implementation: 'first-party',
|
||||||
|
provenance: 'MangaCircus existing MangaDex adapter; no Uchiyomi code copied',
|
||||||
|
operations: Object.freeze(['metadata', 'feed', 'catalog', 'atHome']),
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
module.exports = { manifests };
|
module.exports = { manifests };
|
||||||
@@ -2,7 +2,10 @@
|
|||||||
|
|
||||||
// Reviewed first-party modules only. No filesystem scanning or runtime installs.
|
// Reviewed first-party modules only. No filesystem scanning or runtime installs.
|
||||||
const { manifests } = require('./manifest');
|
const { manifests } = require('./manifest');
|
||||||
const plugins = Object.freeze({ weebcentral: require('./weebcentral') });
|
const plugins = Object.freeze({
|
||||||
|
weebcentral: require('./weebcentral'),
|
||||||
|
mangadex: require('./mangadex'),
|
||||||
|
});
|
||||||
|
|
||||||
function parse(plugin, operation, input) {
|
function parse(plugin, operation, input) {
|
||||||
const manifest = manifests[plugin];
|
const manifest = manifests[plugin];
|
||||||
|
|||||||
@@ -21,10 +21,36 @@ function imageUrl(value) {
|
|||||||
return url.href;
|
return url.href;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function mangaDexUrl(value, kind) {
|
||||||
|
const url = new URL(value);
|
||||||
|
if (url.protocol !== 'https:' || url.username || url.password || url.port) throw new Error('Invalid MangaDex plugin URL');
|
||||||
|
if (kind === 'chapter' && (url.hostname !== 'mangadex.org' || !url.pathname.startsWith('/chapter/'))) {
|
||||||
|
throw new Error('Unexpected MangaDex chapter URL');
|
||||||
|
}
|
||||||
|
if (kind === 'page' && !(url.hostname === 'uploads.mangadex.org' || url.hostname.endsWith('.mangadex.network'))) {
|
||||||
|
throw new Error('Unexpected MangaDex page host');
|
||||||
|
}
|
||||||
|
return url.href;
|
||||||
|
}
|
||||||
|
|
||||||
function validate(plugin, operation, output) {
|
function validate(plugin, operation, output) {
|
||||||
if (plugin !== 'weebcentral' || !output || typeof output !== 'object' || Array.isArray(output)) {
|
if (!['weebcentral', 'mangadex'].includes(plugin) || !output || typeof output !== 'object' || Array.isArray(output)) {
|
||||||
throw new Error('Invalid source plugin output');
|
throw new Error('Invalid source plugin output');
|
||||||
}
|
}
|
||||||
|
if (plugin === 'mangadex') {
|
||||||
|
if (operation === 'metadata') {
|
||||||
|
if (typeof output.title !== 'string' || !output.availabilityAttributes || typeof output.specialLabel !== 'string') throw new Error('Invalid MangaDex metadata');
|
||||||
|
} else if (operation === 'feed') {
|
||||||
|
if (!Array.isArray(output.releases) || output.releases.some(ref => typeof ref?.id !== 'string' || !ref.attributes)) throw new Error('Invalid MangaDex feed');
|
||||||
|
} else if (operation === 'catalog') {
|
||||||
|
if (!Array.isArray(output.chapters) || !output.availability) throw new Error('Invalid MangaDex catalog');
|
||||||
|
for (const chapter of output.chapters) mangaDexUrl(chapter.url, 'chapter');
|
||||||
|
} else if (operation === 'atHome') {
|
||||||
|
if (!Array.isArray(output.pages) || !output.pages.length) throw new Error('Invalid MangaDex pages');
|
||||||
|
for (const page of output.pages) mangaDexUrl(page.url, 'page');
|
||||||
|
} else throw new Error('Unknown source plugin operation');
|
||||||
|
return output;
|
||||||
|
}
|
||||||
if (operation === 'series') {
|
if (operation === 'series') {
|
||||||
if (typeof output.title !== 'string' || typeof output.listUrl !== 'string' && output.listUrl !== null) throw new Error('Invalid series parse');
|
if (typeof output.title !== 'string' || typeof output.listUrl !== 'string' && output.listUrl !== null) throw new Error('Invalid series parse');
|
||||||
if (output.listUrl !== null) sourceUrl(output.listUrl, '/series/');
|
if (output.listUrl !== null) sourceUrl(output.listUrl, '/series/');
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ const http = require('node:http');
|
|||||||
const { parse } = require('./registry');
|
const { parse } = require('./registry');
|
||||||
const MAX_BODY_BYTES = 2 * 1024 * 1024;
|
const MAX_BODY_BYTES = 2 * 1024 * 1024;
|
||||||
|
|
||||||
function createServer() {
|
function createServer(pluginId = process.env.SOURCE_PLUGIN_ID || null) {
|
||||||
return http.createServer((request, response) => {
|
return http.createServer((request, response) => {
|
||||||
if (request.method === 'GET' && request.url === '/health') {
|
if (request.method === 'GET' && request.url === '/health') {
|
||||||
response.writeHead(200, { 'content-type': 'application/json' });
|
response.writeHead(200, { 'content-type': 'application/json' });
|
||||||
@@ -30,6 +30,7 @@ function createServer() {
|
|||||||
if (bytes > MAX_BODY_BYTES) return;
|
if (bytes > MAX_BODY_BYTES) return;
|
||||||
try {
|
try {
|
||||||
const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
||||||
|
if (pluginId && plugin !== pluginId) throw new Error('Plugin worker does not serve this source');
|
||||||
const output = parse(plugin, operation, input);
|
const output = parse(plugin, operation, input);
|
||||||
response.writeHead(200, { 'content-type': 'application/json' });
|
response.writeHead(200, { 'content-type': 'application/json' });
|
||||||
response.end(JSON.stringify(output));
|
response.end(JSON.stringify(output));
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
|
const test = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { createServer } = require('../plugins/worker');
|
||||||
|
const { discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex');
|
||||||
|
|
||||||
|
const mangaId = '123e4567-e89b-42d3-a456-426614174000';
|
||||||
|
const chapterIds = [
|
||||||
|
'123e4567-e89b-42d3-a456-426614174001',
|
||||||
|
'123e4567-e89b-42d3-a456-426614174002',
|
||||||
|
];
|
||||||
|
const seriesUrl = `https://mangadex.org/title/${mangaId}/Fixture`;
|
||||||
|
const manga = { data: { type: 'manga', attributes: {
|
||||||
|
title: { en: 'Fixture' }, originalLanguage: 'ja', lastChapter: '2',
|
||||||
|
availableTranslatedLanguages: ['en'], links: {}, tags: [],
|
||||||
|
} } };
|
||||||
|
const feed = { total: 2, data: chapterIds.map((id, index) => ({
|
||||||
|
id, attributes: { chapter: String(index + 1), title: `Part ${index + 1}`, pages: 2,
|
||||||
|
publishAt: `2026-01-0${index + 1}T00:00:00Z` }, relationships: [],
|
||||||
|
})) };
|
||||||
|
|
||||||
|
function fakeHttp() {
|
||||||
|
return { async get(url) {
|
||||||
|
const body = url.includes('/at-home/server/') ? {
|
||||||
|
baseUrl: 'https://uploads.mangadex.org', chapter: { hash: 'HASH', data: ['1.jpg', '2.jpg'] },
|
||||||
|
} : url.includes('/feed?') ? feed : manga;
|
||||||
|
return { json: async () => body };
|
||||||
|
} };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('MangaDex isolated parser matches the existing catalog and page behavior', async () => {
|
||||||
|
const server = createServer('mangadex');
|
||||||
|
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
||||||
|
const previous = process.env.MANGADEX_PLUGIN_URL;
|
||||||
|
delete process.env.MANGADEX_PLUGIN_URL;
|
||||||
|
try {
|
||||||
|
const baseline = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
|
||||||
|
const baselineChapter = await discoverMangaDexChapter(baseline.chapters[0], fakeHttp());
|
||||||
|
process.env.MANGADEX_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
|
||||||
|
const isolated = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
|
||||||
|
const isolatedChapter = await discoverMangaDexChapter(isolated.chapters[0], fakeHttp());
|
||||||
|
assert.deepEqual(isolated, baseline);
|
||||||
|
assert.deepEqual(isolatedChapter, baselineChapter);
|
||||||
|
const rejected = await fetch(`${process.env.MANGADEX_PLUGIN_URL}/v1/parse`, {
|
||||||
|
method: 'POST', headers: { 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify({ plugin: 'weebcentral', operation: 'series', input: { html: '<h1>X</h1>', url: 'https://weebcentral.com/series/X' } }),
|
||||||
|
});
|
||||||
|
assert.equal(rejected.status, 400);
|
||||||
|
} finally {
|
||||||
|
if (previous === undefined) delete process.env.MANGADEX_PLUGIN_URL;
|
||||||
|
else process.env.MANGADEX_PLUGIN_URL = previous;
|
||||||
|
await new Promise(resolve => server.close(resolve));
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -9,6 +9,9 @@ to do; the diagram describes the target rather than the current implementation.
|
|||||||
Parser-result URL validation and bounded worker messages were added next. The
|
Parser-result URL validation and bounded worker messages were added next. The
|
||||||
opt-in WeebCentral path now validates redirect hops and pins public DNS answers;
|
opt-in WeebCentral path now validates redirect hops and pins public DNS answers;
|
||||||
source-specific image CDN allowlists and generic broker integration remain pending.
|
source-specific image CDN allowlists and generic broker integration remain pending.
|
||||||
|
MangaDex metadata, feed selection, catalog availability, and at-home page parsing
|
||||||
|
now have a second opt-in worker path with fixture parity; its transport still
|
||||||
|
uses the existing API client and runner.
|
||||||
|
|
||||||
## Decision
|
## Decision
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user