Checkpoint opt-in MangaDex parser worker

scraper/plugins/client.js

@@ -16,7 +16,7 @@ async function boundedJson(response) {
 }
 
 async function parseSource(plugin, operation, input) {
-  const endpoint = process.env.SOURCE_PLUGIN_URL;
+  const endpoint = plugin === 'mangadex' ? process.env.MANGADEX_PLUGIN_URL : process.env.SOURCE_PLUGIN_URL;
   if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input));
   const url = new URL('/v1/parse', endpoint);
   if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint');

scraper/plugins/mangadex.js

@@ -0,0 +1,132 @@
+'use strict';
+
+const { specialKey } = require('../../chapter-identity');
+
+function localizedTitle(attributes = {}) {
+  const titles = attributes.title || {};
+  return titles.en || titles['ja-ro'] || titles.ja || Object.values(titles).find(Boolean) || 'Untitled';
+}
+
+function chapterNumber(value) {
+  const raw = String(value ?? '').trim();
+  if (!/^\d+(?:\.\d+)?$/.test(raw)) return null;
+  const [whole, decimal = ''] = raw.split('.');
+  const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0';
+  const normalizedDecimal = decimal.replace(/0+$/, '');
+  return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole;
+}
+
+function groupFor(resource) {
+  const relation = (resource.relationships || []).find(item => item.type === 'scanlation_group');
+  return relation ? { id: relation.id, name: relation.attributes?.name || 'Unknown group' } : null;
+}
+
+function releaseRank(resource) {
+  const group = groupFor(resource)?.name || '';
+  const credited = /official|license/i.test(group) ? 0 : group ? 1 : 2;
+  const date = Date.parse(resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt || '') || Number.MAX_SAFE_INTEGER;
+  return [credited, date, resource.id];
+}
+
+function compareRank(left, right) {
+  const a = releaseRank(left), b = releaseRank(right);
+  return a[0] - b[0] || a[1] - b[1] || String(a[2]).localeCompare(String(b[2]));
+}
+
+function selectChapterReleases(resources = [], specialLabel = 'Special') {
+  const selected = new Map();
+  for (const resource of resources) {
+    if (resource.attributes?.externalUrl || Number(resource.attributes?.pages || 0) < 1) continue;
+    const number = chapterNumber(resource.attributes?.chapter) || specialKey(resource.id);
+    const prior = selected.get(number);
+    if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource);
+  }
+  return [...selected.entries()].map(([number, resource]) => {
+    const group = groupFor(resource);
+    return {
+      id: resource.id, number, title: resource.attributes?.title || '',
+      ...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}),
+      ...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}),
+      pages: Number(resource.attributes?.pages || 0),
+      added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt,
+      group,
+      url: `https://mangadex.org/chapter/${resource.id}`,
+    };
+  }).sort((a, b) => Number(!!a.special) - Number(!!b.special) || (a.special ? a.number.localeCompare(b.number) : Number(a.number) - Number(b.number) || a.number.localeCompare(b.number)));
+}
+
+function availabilitySummary(attributes, releases, chapters, externalReleases = null) {
+  const numbered = chapters.filter(ref => !ref.special);
+  const expected = chapterNumber(attributes.lastChapter);
+  const maximum = numbered.length ? Math.max(...numbered.map(ref => Number(ref.number))) : null;
+  const expectedCount = expected && /^\d+$/.test(expected) ? Number(expected) : null;
+  const sparse = expectedCount && numbered.length < expectedCount;
+  const reason = chapters.length ? null : externalReleases > 0 ? 'external-only'
+    : Array.isArray(attributes.availableTranslatedLanguages) && !attributes.availableTranslatedLanguages.includes('en') ? 'no-english' : 'no-hosted-pages';
+  const message = reason === 'external-only' ? `MangaDex lists ${externalReleases} English external-link releases, but no hosted English pages. External links are not downloaded.`
+    : reason === 'no-english' ? 'This MangaDex edition has no English releases listed.'
+      : reason ? 'This MangaDex edition has no hosted English releases with pages.' : null;
+  return { language: 'en', hostedReleases: releases.filter(r => !r.attributes?.externalUrl && Number(r.attributes?.pages) > 0).length,
+    selectedChapters: chapters.length, numberedChapters: numbered.length, specialReleases: chapters.length - numbered.length,
+    externalReleases, expectedLastChapter: expected, highestHostedChapter: maximum,
+    coverage: expected && (maximum === null || maximum < Number(expected) || sparse) ? 'partial' : 'unknown', reason, message };
+}
+
+function pageUrls(atHome, chapterUrl, expectedPages) {
+  const base = new URL(atHome.baseUrl);
+  if (base.protocol !== 'https:' || !(base.hostname === 'uploads.mangadex.org' || base.hostname.endsWith('.mangadex.network'))) {
+    throw new Error(`Unexpected MangaDex@Home host: ${base.hostname}`);
+  }
+  const hash = String(atHome.chapter?.hash || '');
+  const files = atHome.chapter?.data;
+  if (!hash || !Array.isArray(files) || !files.length) throw new Error('MangaDex@Home returned no pages');
+  if (expectedPages && files.length !== expectedPages) throw new Error(`MangaDex page count changed: expected ${expectedPages}, received ${files.length}`);
+  return files.map(file => ({
+    url: new URL(`/data/${encodeURIComponent(hash)}/${encodeURIComponent(file)}`, base).href,
+    headers: { referer: chapterUrl },
+  }));
+}
+
+function metadata({ html }) {
+  const manga = JSON.parse(html);
+  if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
+  const attributes = manga.attributes || {};
+  const title = localizedTitle(attributes);
+  const specialLabel = /pre.serialization|pilot/i.test(title) ? 'Pilot'
+    : (attributes.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
+  return { title, originalLanguage: attributes.originalLanguage || null, links: attributes.links || {},
+    specialLabel, availabilityAttributes: {
+      lastChapter: attributes.lastChapter ?? null,
+      availableTranslatedLanguages: attributes.availableTranslatedLanguages ?? null,
+    } };
+}
+
+function feed({ html }) {
+  const releases = JSON.parse(html);
+  if (!Array.isArray(releases)) throw new Error('Invalid MangaDex feed');
+  return { releases: releases.map(resource => ({
+    id: resource.id,
+    attributes: {
+      chapter: resource.attributes?.chapter, volume: resource.attributes?.volume,
+      title: resource.attributes?.title, pages: resource.attributes?.pages,
+      publishAt: resource.attributes?.publishAt, readableAt: resource.attributes?.readableAt,
+      createdAt: resource.attributes?.createdAt, externalUrl: resource.attributes?.externalUrl,
+    },
+    relationships: (resource.relationships || []).filter(relation => relation.type === 'scanlation_group'),
+  })) };
+}
+
+function catalog({ html }) {
+  const { attributes, releases, specialLabel, externalReleases } = JSON.parse(html);
+  if (!attributes || !Array.isArray(releases)) throw new Error('Invalid MangaDex catalog input');
+  const chapters = selectChapterReleases(releases, specialLabel);
+  return { chapters, availability: availabilitySummary(attributes, releases, chapters, externalReleases) };
+}
+
+function atHome({ html }) {
+  const { response, chapterUrl, expectedPages } = JSON.parse(html);
+  return { pages: pageUrls(response, chapterUrl, expectedPages) };
+}
+
+module.exports = { localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary,
+  pageUrls, metadata, feed, catalog, atHome };

scraper/plugins/manifest.js

@@ -10,6 +10,13 @@ const manifests = Object.freeze({
     provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied',
     operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']),
   }),
+  mangadex: Object.freeze({
+    id: 'mangadex', version: 1, label: 'MangaDex', status: 'pilot',
+    sourceHosts: Object.freeze(['api.mangadex.org']),
+    implementation: 'first-party',
+    provenance: 'MangaCircus existing MangaDex adapter; no Uchiyomi code copied',
+    operations: Object.freeze(['metadata', 'feed', 'catalog', 'atHome']),
+  }),
 });
 
 module.exports = { manifests };

scraper/plugins/registry.js

@@ -2,7 +2,10 @@
 
 // Reviewed first-party modules only. No filesystem scanning or runtime installs.
 const { manifests } = require('./manifest');
-const plugins = Object.freeze({ weebcentral: require('./weebcentral') });
+const plugins = Object.freeze({
+  weebcentral: require('./weebcentral'),
+  mangadex: require('./mangadex'),
+});
 
 function parse(plugin, operation, input) {
   const manifest = manifests[plugin];

scraper/plugins/validation.js

@@ -21,10 +21,36 @@ function imageUrl(value) {
   return url.href;
 }
 
+function mangaDexUrl(value, kind) {
+  const url = new URL(value);
+  if (url.protocol !== 'https:' || url.username || url.password || url.port) throw new Error('Invalid MangaDex plugin URL');
+  if (kind === 'chapter' && (url.hostname !== 'mangadex.org' || !url.pathname.startsWith('/chapter/'))) {
+    throw new Error('Unexpected MangaDex chapter URL');
+  }
+  if (kind === 'page' && !(url.hostname === 'uploads.mangadex.org' || url.hostname.endsWith('.mangadex.network'))) {
+    throw new Error('Unexpected MangaDex page host');
+  }
+  return url.href;
+}
+
 function validate(plugin, operation, output) {
-  if (plugin !== 'weebcentral' || !output || typeof output !== 'object' || Array.isArray(output)) {
+  if (!['weebcentral', 'mangadex'].includes(plugin) || !output || typeof output !== 'object' || Array.isArray(output)) {
     throw new Error('Invalid source plugin output');
   }
+  if (plugin === 'mangadex') {
+    if (operation === 'metadata') {
+      if (typeof output.title !== 'string' || !output.availabilityAttributes || typeof output.specialLabel !== 'string') throw new Error('Invalid MangaDex metadata');
+    } else if (operation === 'feed') {
+      if (!Array.isArray(output.releases) || output.releases.some(ref => typeof ref?.id !== 'string' || !ref.attributes)) throw new Error('Invalid MangaDex feed');
+    } else if (operation === 'catalog') {
+      if (!Array.isArray(output.chapters) || !output.availability) throw new Error('Invalid MangaDex catalog');
+      for (const chapter of output.chapters) mangaDexUrl(chapter.url, 'chapter');
+    } else if (operation === 'atHome') {
+      if (!Array.isArray(output.pages) || !output.pages.length) throw new Error('Invalid MangaDex pages');
+      for (const page of output.pages) mangaDexUrl(page.url, 'page');
+    } else throw new Error('Unknown source plugin operation');
+    return output;
+  }
   if (operation === 'series') {
     if (typeof output.title !== 'string' || typeof output.listUrl !== 'string' && output.listUrl !== null) throw new Error('Invalid series parse');
     if (output.listUrl !== null) sourceUrl(output.listUrl, '/series/');

scraper/plugins/worker.js

@@ -4,7 +4,7 @@ const http = require('node:http');
 const { parse } = require('./registry');
 const MAX_BODY_BYTES = 2 * 1024 * 1024;
 
-function createServer() {
+function createServer(pluginId = process.env.SOURCE_PLUGIN_ID || null) {
   return http.createServer((request, response) => {
     if (request.method === 'GET' && request.url === '/health') {
       response.writeHead(200, { 'content-type': 'application/json' });
@@ -30,6 +30,7 @@ function createServer() {
       if (bytes > MAX_BODY_BYTES) return;
       try {
         const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+        if (pluginId && plugin !== pluginId) throw new Error('Plugin worker does not serve this source');
         const output = parse(plugin, operation, input);
         response.writeHead(200, { 'content-type': 'application/json' });
         response.end(JSON.stringify(output));