Consolidate MangaCircus app, scraper, and Compose baseline

scraper/adapters/index.js

@@ -0,0 +1,95 @@
+'use strict';
+
+const { runArchive } = require('../jobs/weebcentral-archive');
+const { runMangaDexArchive } = require('../jobs/mangadex-archive');
+const { SlowHttp } = require('../lib/http');
+const { discoverWeebCatalog } = require('./weebcentral');
+const { mangaDexInfo, discoverMangaDexCatalog } = require('./mangadex');
+const { ArchiveControlError } = require('../lib/control');
+
+function controlledHttp(http, shouldStop) {
+  return { async get(...args) {
+    const check = () => { const reason = shouldStop?.(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); };
+    check(); const response = await http.get(...args); check(); return response;
+  } };
+}
+
+function weebCentralInfo(value) {
+  let url;
+  try { url = new URL(String(value)); } catch { return null; }
+  if (url.protocol !== 'https:' || url.hostname !== 'weebcentral.com') return null;
+  const match = /^\/series\/([A-Z0-9]+)(?:\/([^/?#]+))?\/?$/i.exec(url.pathname);
+  if (!match) return null;
+  const title = decodeURIComponent(match[2] || match[1]).replace(/[-_]+/g, ' ').trim();
+  return { source: 'weebcentral', id: match[1].toUpperCase(), title, url: `https://weebcentral.com/series/${match[1]}${match[2] ? `/${match[2]}` : ''}` };
+}
+
+const adapters = [{
+  id: 'weebcentral',
+  label: 'WeebCentral',
+  match: weebCentralInfo,
+  async discover(item, options) {
+    const http = new SlowHttp({
+      minMs: options.settings.minMs, maxMs: options.settings.maxMs, maxRetries: 2,
+      onEvent: options.onEvent,
+    });
+    return discoverWeebCatalog(item.url, controlledHttp(http, options.shouldStop));
+  },
+  run(item, options) {
+    return runArchive({
+      seriesUrl: item.url,
+      libraryRoot: options.libraryRoot,
+      metadataSearch: item.metadataSearch || item.title,
+      minMs: options.settings.minMs,
+      maxMs: options.settings.maxMs,
+      chapterMinMs: options.settings.chapterMinMs,
+      chapterMaxMs: options.settings.chapterMaxMs,
+      shouldStop: options.shouldStop,
+      scope: item.scope,
+      existingPolicy: item.existingPolicy,
+      libraryMangaId: item.libraryMangaId,
+      onPublished: options.onPublished,
+      onCatalog: options.onCatalog,
+    });
+  },
+}, {
+  id: 'mangadex',
+  label: 'MangaDex',
+  match: mangaDexInfo,
+  async discover(item, options) {
+    const http = new SlowHttp({
+      minMs: options.settings.minMs, maxMs: options.settings.maxMs, maxRetries: 2,
+      userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)',
+      onEvent: options.onEvent,
+    });
+    return discoverMangaDexCatalog(item.url, controlledHttp(http, options.shouldStop));
+  },
+  run(item, options) {
+    return runMangaDexArchive({
+      seriesUrl: item.url,
+      title: item.title,
+      libraryRoot: options.libraryRoot,
+      metadataSearch: item.metadataSearch || item.title,
+      minMs: options.settings.minMs,
+      maxMs: options.settings.maxMs,
+      chapterMinMs: options.settings.chapterMinMs,
+      chapterMaxMs: options.settings.chapterMaxMs,
+      shouldStop: options.shouldStop,
+      scope: item.scope,
+      existingPolicy: item.existingPolicy,
+      libraryMangaId: item.libraryMangaId,
+      onPublished: options.onPublished,
+      onCatalog: options.onCatalog,
+    });
+  },
+}];
+
+function adapterFor(url) {
+  for (const adapter of adapters) {
+    const info = adapter.match(url);
+    if (info) return { adapter, info };
+  }
+  return null;
+}
+
+module.exports = { adapters, adapterFor, weebCentralInfo, mangaDexInfo };

scraper/adapters/mangadex.js

@@ -0,0 +1,161 @@
+'use strict';
+
+const API = 'https://api.mangadex.org';
+const { specialKey } = require('../../chapter-identity');
+
+function mangaDexInfo(value) {
+  let url;
+  try { url = new URL(String(value)); } catch { return null; }
+  if (url.protocol !== 'https:' || !['mangadex.org', 'www.mangadex.org'].includes(url.hostname)) return null;
+  const match = /^\/title\/([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})(?:\/([^/?#]+))?\/?$/i.exec(url.pathname);
+  if (!match) return null;
+  const id = match[1].toLowerCase();
+  const title = decodeURIComponent(match[2] || id).replace(/[-_]+/g, ' ').trim();
+  return { source: 'mangadex', id, title, url: `https://mangadex.org/title/${id}${match[2] ? `/${match[2]}` : ''}` };
+}
+
+function localizedTitle(attributes = {}) {
+  const titles = attributes.title || {};
+  return titles.en || titles['ja-ro'] || titles.ja || Object.values(titles).find(Boolean) || 'Untitled';
+}
+
+function chapterNumber(value) {
+  const raw = String(value ?? '').trim();
+  if (!/^\d+(?:\.\d+)?$/.test(raw)) return null;
+  const [whole, decimal = ''] = raw.split('.');
+  const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0';
+  const normalizedDecimal = decimal.replace(/0+$/, '');
+  return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole;
+}
+
+function groupFor(resource) {
+  const relation = (resource.relationships || []).find(item => item.type === 'scanlation_group');
+  return relation ? { id: relation.id, name: relation.attributes?.name || 'Unknown group' } : null;
+}
+
+function releaseRank(resource) {
+  const group = groupFor(resource)?.name || '';
+  const credited = /official|license/i.test(group) ? 0 : group ? 1 : 2;
+  const date = Date.parse(resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt || '') || Number.MAX_SAFE_INTEGER;
+  return [credited, date, resource.id];
+}
+
+function compareRank(left, right) {
+  const a = releaseRank(left), b = releaseRank(right);
+  return a[0] - b[0] || a[1] - b[1] || String(a[2]).localeCompare(String(b[2]));
+}
+
+function selectChapterReleases(resources = [], specialLabel = 'Special') {
+  const selected = new Map();
+  for (const resource of resources) {
+    if (resource.attributes?.externalUrl || Number(resource.attributes?.pages || 0) < 1) continue;
+    const number = chapterNumber(resource.attributes?.chapter) || specialKey(resource.id);
+    const prior = selected.get(number);
+    if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource);
+  }
+  return [...selected.entries()].map(([number, resource]) => {
+    const group = groupFor(resource);
+    return {
+      id: resource.id, number, title: resource.attributes?.title || '',
+      ...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}),
+      ...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}),
+      pages: Number(resource.attributes?.pages || 0),
+      added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt,
+      group,
+      url: `https://mangadex.org/chapter/${resource.id}`,
+    };
+  }).sort((a, b) => Number(!!a.special) - Number(!!b.special) || (a.special ? a.number.localeCompare(b.number) : Number(a.number) - Number(b.number) || a.number.localeCompare(b.number)));
+}
+
+function availabilitySummary(attributes, releases, chapters, externalReleases = null) {
+  const numbered = chapters.filter(ref => !ref.special);
+  const expected = chapterNumber(attributes.lastChapter);
+  const maximum = numbered.length ? Math.max(...numbered.map(ref => Number(ref.number))) : null;
+  const reason = chapters.length ? null : externalReleases > 0 ? 'external-only'
+    : Array.isArray(attributes.availableTranslatedLanguages) && !attributes.availableTranslatedLanguages.includes('en') ? 'no-english' : 'no-hosted-pages';
+  const message = reason === 'external-only' ? `MangaDex lists ${externalReleases} English external-link releases, but no hosted English pages. External links are not downloaded.`
+    : reason === 'no-english' ? 'This MangaDex edition has no English releases listed.'
+      : reason ? 'This MangaDex edition has no hosted English releases with pages.' : null;
+  return { language: 'en', hostedReleases: releases.filter(r => !r.attributes?.externalUrl && Number(r.attributes?.pages) > 0).length,
+    selectedChapters: chapters.length, numberedChapters: numbered.length, specialReleases: chapters.length - numbered.length,
+    externalReleases, expectedLastChapter: expected, highestHostedChapter: maximum,
+    coverage: expected && (maximum === null || maximum < Number(expected)) ? 'partial' : 'unknown', reason, message };
+}
+
+async function jsonGet(http, url) {
+  const response = await http.get(url, { accept: 'application/json' });
+  const body = await response.json();
+  if (body.result && body.result !== 'ok') throw new Error(`MangaDex API result: ${body.result}`);
+  return body;
+}
+
+async function discoverMangaDexCatalog(seriesUrl, http) {
+  const info = mangaDexInfo(seriesUrl);
+  if (!info) throw new Error('Invalid MangaDex title URL');
+  const manga = (await jsonGet(http, `${API}/manga/${info.id}`)).data;
+  if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
+
+  const releases = [];
+  let offset = 0;
+  for (;;) {
+    const params = new URLSearchParams({ limit: '100', offset: String(offset), includeFutureUpdates: '0', includeExternalUrl: '0' });
+    params.append('translatedLanguage[]', 'en');
+    params.append('includes[]', 'scanlation_group');
+    params.append('order[chapter]', 'asc');
+    const page = await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`);
+    releases.push(...(page.data || []));
+    offset += page.data?.length || 0;
+    if (!page.data?.length || offset >= Number(page.total || 0)) break;
+  }
+
+  const title = localizedTitle(manga.attributes);
+  const specialLabel = /pre.serialization|pilot/i.test(title) ? 'Pilot'
+    : (manga.attributes?.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
+  const chapters = selectChapterReleases(releases, specialLabel);
+  // External-only is a separate feed mode, not a combined hosted+external catalog.
+  let externalReleases = null;
+  if (!chapters.length) {
+    const params = new URLSearchParams({ limit: '1', includeFutureUpdates: '0', includeExternalUrl: '1' });
+    params.append('translatedLanguage[]', 'en');
+    externalReleases = Number((await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`)).total || 0);
+  }
+  return {
+    id: info.id,
+    title,
+    originalLanguage: manga.attributes?.originalLanguage || null,
+    links: manga.attributes?.links || {},
+    availability: availabilitySummary(manga.attributes || {}, releases, chapters, externalReleases),
+    chapters,
+  };
+}
+
+function pageUrls(atHome, chapterUrl, expectedPages) {
+  const base = new URL(atHome.baseUrl);
+  if (base.protocol !== 'https:' || !(base.hostname === 'uploads.mangadex.org' || base.hostname.endsWith('.mangadex.network'))) {
+    throw new Error(`Unexpected MangaDex@Home host: ${base.hostname}`);
+  }
+  const hash = String(atHome.chapter?.hash || '');
+  const files = atHome.chapter?.data;
+  if (!hash || !Array.isArray(files) || !files.length) throw new Error('MangaDex@Home returned no pages');
+  if (expectedPages && files.length !== expectedPages) throw new Error(`MangaDex page count changed: expected ${expectedPages}, received ${files.length}`);
+  return files.map(file => ({
+    url: new URL(`/data/${encodeURIComponent(hash)}/${encodeURIComponent(file)}`, base).href,
+    headers: { referer: chapterUrl },
+  }));
+}
+
+async function discoverMangaDexChapter(ref, http) {
+  const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`);
+  const groupName = ref.group?.name ? ` · ${ref.group.name}` : '';
+  return {
+    number: ref.number, title: ref.title || '', added: ref.added,
+    ...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}),
+    source: { name: `MangaDex${groupName}`, url: ref.url },
+    pages: pageUrls(atHome, ref.url, ref.pages),
+  };
+}
+
+module.exports = {
+  API, mangaDexInfo, localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary,
+  discoverMangaDexCatalog, discoverMangaDexChapter, pageUrls,
+};

scraper/adapters/weebcentral.js

@@ -0,0 +1,88 @@
+'use strict';
+
+const { SlowHttp } = require('../lib/http');
+const { fallbackDirection } = require('../../reading-direction');
+
+const decode = value => value.replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/<[^>]+>/g, '').trim();
+
+function absolute(base, value) { return new URL(value, base).href; }
+
+function parseChapters(html, baseUrl) {
+  const found = [];
+  const re = /<a\b[^>]*href="([^"]*\/chapters\/([A-Z0-9]+)[^"]*)"[^>]*>([\s\S]*?)<\/a>/gi;
+  for (const match of html.matchAll(re)) {
+    const text = decode(match[3]).replace(/\s+/g, ' ');
+    const season = Number(text.match(/^S(\d+)\s*-/i)?.[1] || 0) || null;
+    // A few WeebCentral catalogs use a series-specific label (for example
+    // "Rating 442", "Episode 128", "Night 182", "Log. 65", or "Act 60.6") in the same place where most catalogs
+    // say "Chapter 442". Treat those labels as chapter numbering instead of
+    // discarding the entire catalog.
+    const regular = text.match(/\b(?:Chapter|Rating|Episode|Night|Log|Act)\s*[.:]?\s*([0-9]+(?:\.[0-9]+)?)/i)?.[1];
+    const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1];
+    const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null);
+    if (number) found.push({
+      id: match[2], number, title: regular ? '' : `Prologue ${prologue || ''}`.trim(),
+      ...(season ? { book: { number: season, title: `Season ${season}` } } : {}),
+      url: absolute(baseUrl, match[1]), raw: text,
+    });
+  }
+  return [...new Map(found.map(chapter => [chapter.id, chapter])).values()];
+}
+
+async function discoverWeebCatalog(seriesUrl, http) {
+  const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text();
+  const title = decode(seriesHtml.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || 'Untitled');
+  const listUrl = seriesHtml.match(/hx-get="([^"]+\/full-chapter-list)"/i)?.[1];
+  const listHtml = listUrl ? await (await http.get(absolute(seriesUrl, listUrl), { accept: 'text/html' })).text() : seriesHtml;
+  const chapterLinks = [...listHtml.matchAll(/<a\b[^>]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length;
+  const chapters = parseChapters(listHtml, seriesUrl).sort((a, b) =>
+    Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number));
+  return { title, chapters, chapterLinks,
+    ...(chapterLinks && !chapters.length ? { availability: { reason: 'unrecognized-labels',
+      message: `WeebCentral lists ${chapterLinks} chapter links, but their numbering labels are not recognized by the parser.` } } : {}),
+  };
+}
+
+async function discoverWeebChapter(chapter, http) {
+  const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text();
+  const manifestUrl = pageHtml.match(/['"](https?:\/\/weebcentral\.com\/chapters\/[^'"]+\/images\?is_prev=[^'"]+)['"]/i)?.[1]
+    || `${chapter.url}/images?is_prev=False`;
+  const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip&current_page=1`, { accept: 'text/html' })).text();
+  const pages = parsePageUrls(fragment, manifestUrl).map(url => ({ url, headers: { referer: chapter.url } }));
+  if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`);
+  return {
+    number: chapter.number, title: chapter.title || '',
+    ...(chapter.book ? { book: chapter.book } : {}),
+    added: pageHtml.match(/datetime="([^"]+)"/i)?.[1],
+    source: { name: 'WeebCentral', url: chapter.url }, pages,
+  };
+}
+
+function parsePageUrls(html, baseUrl) {
+  const urls = [];
+  for (const match of html.matchAll(/<img\b[^>]*(?:src|data-src)="([^"]+)"/gi)) {
+    const url = absolute(baseUrl, match[1].replace(/&amp;/g, '&'));
+    if (!url.startsWith('data:')) urls.push(url);
+  }
+  return [...new Set(urls)];
+}
+
+async function discoverWeebCentral(seriesUrl, options = {}) {
+  const http = new SlowHttp(options.http || {});
+  const catalog = await discoverWeebCatalog(seriesUrl, http);
+  const chapters = catalog.chapters
+    .sort((a, b) => Number(b.number) - Number(a.number))
+    .slice(0, options.limit || 3)
+    .sort((a, b) => Number(a.number) - Number(b.number));
+  const output = [];
+  for (const chapter of chapters) {
+    output.push(await discoverWeebChapter(chapter, http));
+  }
+  return {
+    version: 1,
+    series: { title: catalog.title, ...fallbackDirection(), sources: [{ name: 'WeebCentral', url: seriesUrl }], scraper: { ids: { weebcentral: seriesUrl.split('/series/')[1]?.split('/')[0] } } },
+    http: options.http || {}, chapters: output,
+  };
+}
+
+module.exports = { discoverWeebCentral, discoverWeebCatalog, discoverWeebChapter, parseChapters, parsePageUrls };