diff --git a/docker-compose.yml b/docker-compose.yml index 7aeac0b..4ffba26 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -80,9 +80,7 @@ services: SCRAPER_PORT: "8091" SCRAPER_TOKEN_FILE: /run/secrets/scraper_token PEER_DATA: /data/peering - SOURCE_PLUGIN_URL: ${SOURCE_PLUGIN_URL:-} - MANGADEX_PLUGIN_URL: ${MANGADEX_PLUGIN_URL:-} - networks: [backend, egress, plugin_backend] + networks: [backend, egress] healthcheck: test: [CMD, node, tools/healthcheck.js, scraper] interval: 30s @@ -148,67 +146,9 @@ services: start_period: 30s retries: 3 - source-plugin-weebcentral: - image: ${APP_IMAGE:-mangacircus:local} - build: - context: . - args: - VCS_REF: ${VCS_REF:-development} - profiles: [source-plugins] - command: [node, scraper/plugins/worker.js] - init: true - restart: unless-stopped - read_only: true - tmpfs: [/tmp] - security_opt: [no-new-privileges:true] - cap_drop: [ALL] - pids_limit: 64 - mem_limit: 256m - environment: - SOURCE_PLUGIN_ID: weebcentral - SOURCE_PLUGIN_HOST: 0.0.0.0 - SOURCE_PLUGIN_PORT: "8094" - networks: [plugin_backend] - healthcheck: - test: [CMD, node, tools/healthcheck-source-plugin.js] - interval: 30s - timeout: 5s - start_period: 15s - retries: 3 - - source-plugin-mangadex: - image: ${APP_IMAGE:-mangacircus:local} - build: - context: . - args: - VCS_REF: ${VCS_REF:-development} - profiles: [source-plugins] - command: [node, scraper/plugins/worker.js] - init: true - restart: unless-stopped - read_only: true - tmpfs: [/tmp] - security_opt: [no-new-privileges:true] - cap_drop: [ALL] - pids_limit: 64 - mem_limit: 256m - environment: - SOURCE_PLUGIN_ID: mangadex - SOURCE_PLUGIN_HOST: 0.0.0.0 - SOURCE_PLUGIN_PORT: "8094" - networks: [plugin_backend] - healthcheck: - test: [CMD, node, tools/healthcheck-source-plugin.js] - interval: 30s - timeout: 5s - start_period: 15s - retries: 3 - networks: backend: internal: true - plugin_backend: - internal: true egress: {} secrets: diff --git a/docs/architecture.md b/docs/architecture.md index 979c984..8dc1384 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,6 +17,7 @@ Related documents: - [settings-navigation.md](settings-navigation.md): User/Admin workspace, scraper tabs and deep links. - [peering.md](peering.md): optional private server-to-server catalog and transfer. - [discovery.md](discovery.md): reader-owned, opt-in deterministic discovery feed and its limits. +- [source-modules.md](source-modules.md): adding and maintaining first-party hosters inside the single scraper process. ## 1. Principles diff --git a/docs/scraper-service.md b/docs/scraper-service.md index 64a3330..aff3de2 100644 --- a/docs/scraper-service.md +++ b/docs/scraper-service.md @@ -5,31 +5,23 @@ It owns the durable queue and is the only process allowed to run a queued archiv The browser never contacts it directly: authenticated admin routes in `server.js` proxy commands to its private HTTP API. -## First-party source parser pilot +## In-process source modules -The WeebCentral and MangaDex parsers have reviewed first-party modules in -`scraper/plugins/`. By default their existing in-process paths remain in use. -To test process isolation in Compose, start the `source-plugins` profile and -set `SOURCE_PLUGIN_URL=http://source-plugin-weebcentral:8094` and/or -`MANGADEX_PLUGIN_URL=http://source-plugin-mangadex:8094` for the scraper. -Each URL selects its source independently. A selected worker that is unavailable -causes that source's discovery to fail closed and retain its checkpoint. Remove -its URL to roll back. Do not restart a live scraper merely to try this option; -use the normal checkpoint-safe deployment procedure. +There is one scraper daemon/container and one durable queue. WeebCentral and +MangaDex source modules are registered explicitly in `scraper/adapters/index.js`. +Their descriptors live in `scraper/sources/`; they call source-specific catalog +and archive code in the same process. Pure parsing functions in +`scraper/plugins/` handle source page/API shapes. No source parser has a separate +container, HTTP endpoint, runtime download, or independent queue. Only reviewed +first-party modules ship in the image; Uchiyomi packages are not executed. -Each worker has no published port, egress network, library/data mount, or token. -Only the existing rate-limited scraper HTTP client fetches remote HTML/images. -The worker accepts bounded HTML for four parsing steps over its internal-only -network. It loads only modules listed in the static first-party registry; there -is no runtime plugin download or execution of Uchiyomi packages. This is a -parser-isolation pilot, **not yet** a complete generic source-plugin contract -or migration of either archive runner. Parsed catalog and image URLs are checked -before use, and parser requests/responses are size-bounded. When the worker is -enabled, WeebCentral requests use a pinned-DNS HTTPS transport that validates -each redirect and stops on access errors; the ordinary in-process path is not -changed. Public image hosts are not yet restricted to a reviewed CDN allowlist. -Keep the profile off in production until a separate deployment validation is -completed. +The scraper's paced HTTP client fetches source content. Parsing is an ordinary +function call. The registry checks source URLs and validates parsed output before +the scraper follows links. WeebCentral catalog requests use a pinned-DNS HTTPS +transport that validates redirects and stops on access errors. Image hosts are +not restricted to a reviewed CDN allowlist yet. Existing queue state, source +delays, chapter delays, and partial-download checkpoints are unchanged by this +code organization. See [adding a source](source-modules.md) for the full path. ## Start and storage @@ -218,7 +210,7 @@ the shared token. ## Adapter contract -Each source plugin is registered in `scraper/adapters/index.js` with: +Each first-party source module is registered in `scraper/adapters/index.js` with: ```js { diff --git a/docs/source-modules.md b/docs/source-modules.md new file mode 100644 index 0000000..771ed9d --- /dev/null +++ b/docs/source-modules.md @@ -0,0 +1,42 @@ +# Adding a source to the single scraper + +MangaCircus uses **one scraper process**. A source module is reviewed first-party +code bundled into that process, not a container or a downloaded extension. +`scraper/adapters/index.js` is the explicit registration point. The library, +queue, pacing settings, checkpointing, source-error policy, and publication +remain shared infrastructure. Do not add another queue writer for a new hoster. + +## Source boundary + +| Concern | Location | New-source responsibility | +| --- | --- | --- | +| URL recognition and adapter contract | `scraper/sources/.js` | `id`, `label`, `match`, `discover`, `run` | +| Catalog and chapter discovery | `scraper/adapters/.js` | Map source responses to stable chapter IDs, metadata and page URLs | +| Source HTML/API parsing | `scraper/plugins/.js` | Pure functions; no network or filesystem writes | +| Parser operation/URL validation | `scraper/plugins/manifest.js`, `registry.js`, `validation.js` | Allow only known operations and expected hosts/links | +| Resumable archive workflow | `scraper/jobs/-archive.js` | Use the common publisher, inventory, pacing and checkpoints | +| Registration | `scraper/adapters/index.js` | Add one explicit `require` to `adapters` | + +The module contract is shown in [scraper-service.md](scraper-service.md#adapter-contract). +Start from the closest existing hoster. `match(url)` returns `null` for unrelated +URLs and `{ source, id, title, url }` for an exact supported series URL. `discover` +must be read-only and return a catalog. `run` must honor scope, existing-content +policy, `shouldStop`, source-specific delays, and publication callbacks. Stable +source series/chapter IDs are essential for retries and duplicate detection. + +## Before enabling a hoster + +1. Document its permitted access method, terms and source-specific pacing. Do + not work around 401, 403, 429, CAPTCHA, challenges or source protections. +2. Add fixture-backed tests for URL matching, catalog, empty/partial catalogs, + chapter pages, unsafe links, and duplicate release handling. No test should + contact the live hoster by default. +3. Test pause/resume from a partial chapter and an existing library target with + the common publisher; verify one source lane and no parallel image workers. +4. Run `npm run test:scraper` and an isolated Compose smoke test. Review and deploy + the single scraper image with a checkpoint-safe procedure; never start another + daemon against the live queue. + +Removing a hoster from registration stops new matches; it does **not** delete its +library chapters or queued/checkpointed work. Inspect affected queue items and +preserve checkpoints before disabling a source in a release. diff --git a/scraper/adapters/index.js b/scraper/adapters/index.js index c12a5cd..00b7d37 100644 --- a/scraper/adapters/index.js +++ b/scraper/adapters/index.js @@ -1,92 +1,13 @@ 'use strict'; -const { runArchive } = require('../jobs/weebcentral-archive'); -const { runMangaDexArchive } = require('../jobs/mangadex-archive'); -const { SlowHttp } = require('../lib/http'); -const { createWeebHttp } = require('../plugins/broker'); -const { discoverWeebCatalog } = require('./weebcentral'); -const { mangaDexInfo, discoverMangaDexCatalog } = require('./mangadex'); -const { ArchiveControlError } = require('../lib/control'); +// First-party registration only; never load executable source code from a +// remote catalog or scan a writable directory for modules. +const adapters = Object.freeze([ + require('../sources/weebcentral'), + require('../sources/mangadex'), +]); const { adapterForPeer } = require('../../peering/adapter'); -function controlledHttp(http, shouldStop) { - return { async get(...args) { - const check = () => { const reason = shouldStop?.(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); }; - check(); const response = await http.get(...args); check(); return response; - } }; -} - -function weebCentralInfo(value) { - let url; - try { url = new URL(String(value)); } catch { return null; } - if (url.protocol !== 'https:' || url.hostname !== 'weebcentral.com') return null; - const match = /^\/series\/([A-Z0-9]+)(?:\/([^/?#]+))?\/?$/i.exec(url.pathname); - if (!match) return null; - const title = decodeURIComponent(match[2] || match[1]).replace(/[-_]+/g, ' ').trim(); - return { source: 'weebcentral', id: match[1].toUpperCase(), title, url: `https://weebcentral.com/series/${match[1]}${match[2] ? `/${match[2]}` : ''}` }; -} - -const adapters = [{ - id: 'weebcentral', - label: 'WeebCentral', - match: weebCentralInfo, - async discover(item, options) { - const httpOptions = { - minMs: options.settings.minMs, maxMs: options.settings.maxMs, maxRetries: 2, - onEvent: options.onEvent, - }; - const http = process.env.SOURCE_PLUGIN_URL ? createWeebHttp(httpOptions) : new SlowHttp(httpOptions); - return discoverWeebCatalog(item.url, controlledHttp(http, options.shouldStop)); - }, - run(item, options) { - return runArchive({ - seriesUrl: item.url, - libraryRoot: options.libraryRoot, - metadataSearch: item.metadataSearch || item.title, - minMs: options.settings.minMs, - maxMs: options.settings.maxMs, - chapterMinMs: options.settings.chapterMinMs, - chapterMaxMs: options.settings.chapterMaxMs, - shouldStop: options.shouldStop, - scope: item.scope, - existingPolicy: item.existingPolicy, - libraryMangaId: item.libraryMangaId, - onPublished: options.onPublished, - onCatalog: options.onCatalog, - }); - }, -}, { - id: 'mangadex', - label: 'MangaDex', - match: mangaDexInfo, - async discover(item, options) { - const http = new SlowHttp({ - minMs: options.settings.minMs, maxMs: options.settings.maxMs, maxRetries: 2, - userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)', - onEvent: options.onEvent, - }); - return discoverMangaDexCatalog(item.url, controlledHttp(http, options.shouldStop)); - }, - run(item, options) { - return runMangaDexArchive({ - seriesUrl: item.url, - title: item.title, - libraryRoot: options.libraryRoot, - metadataSearch: item.metadataSearch || item.title, - minMs: options.settings.minMs, - maxMs: options.settings.maxMs, - chapterMinMs: options.settings.chapterMinMs, - chapterMaxMs: options.settings.chapterMaxMs, - shouldStop: options.shouldStop, - scope: item.scope, - existingPolicy: item.existingPolicy, - libraryMangaId: item.libraryMangaId, - onPublished: options.onPublished, - onCatalog: options.onCatalog, - }); - }, -}]; - function adapterFor(url) { const peer = adapterForPeer(url); if (peer) return peer; @@ -97,4 +18,6 @@ function adapterFor(url) { return null; } -module.exports = { adapters, adapterFor, weebCentralInfo, mangaDexInfo }; +module.exports = { adapters, adapterFor, + weebCentralInfo: require('./weebcentral').weebCentralInfo, + mangaDexInfo: require('./mangadex').mangaDexInfo }; diff --git a/scraper/adapters/mangadex.js b/scraper/adapters/mangadex.js index 92638bf..147e1cd 100644 --- a/scraper/adapters/mangadex.js +++ b/scraper/adapters/mangadex.js @@ -1,8 +1,8 @@ 'use strict'; const API = 'https://api.mangadex.org'; -const { specialKey } = require('../../chapter-identity'); -const { parseSource } = require('../plugins/client'); +const { parse } = require('../plugins/registry'); +const { localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary, pageUrls } = require('../plugins/mangadex'); function mangaDexInfo(value) { let url; @@ -15,78 +15,6 @@ function mangaDexInfo(value) { return { source: 'mangadex', id, title, url: `https://mangadex.org/title/${id}${match[2] ? `/${match[2]}` : ''}` }; } -function localizedTitle(attributes = {}) { - const titles = attributes.title || {}; - return titles.en || titles['ja-ro'] || titles.ja || Object.values(titles).find(Boolean) || 'Untitled'; -} - -function chapterNumber(value) { - const raw = String(value ?? '').trim(); - if (!/^\d+(?:\.\d+)?$/.test(raw)) return null; - const [whole, decimal = ''] = raw.split('.'); - const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0'; - const normalizedDecimal = decimal.replace(/0+$/, ''); - return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole; -} - -function groupFor(resource) { - const relation = (resource.relationships || []).find(item => item.type === 'scanlation_group'); - return relation ? { id: relation.id, name: relation.attributes?.name || 'Unknown group' } : null; -} - -function releaseRank(resource) { - const group = groupFor(resource)?.name || ''; - const credited = /official|license/i.test(group) ? 0 : group ? 1 : 2; - const date = Date.parse(resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt || '') || Number.MAX_SAFE_INTEGER; - return [credited, date, resource.id]; -} - -function compareRank(left, right) { - const a = releaseRank(left), b = releaseRank(right); - return a[0] - b[0] || a[1] - b[1] || String(a[2]).localeCompare(String(b[2])); -} - -function selectChapterReleases(resources = [], specialLabel = 'Special') { - const selected = new Map(); - for (const resource of resources) { - if (resource.attributes?.externalUrl || Number(resource.attributes?.pages || 0) < 1) continue; - const number = chapterNumber(resource.attributes?.chapter) || specialKey(resource.id); - const prior = selected.get(number); - if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource); - } - return [...selected.entries()].map(([number, resource]) => { - const group = groupFor(resource); - return { - id: resource.id, number, title: resource.attributes?.title || '', - ...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}), - ...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}), - pages: Number(resource.attributes?.pages || 0), - added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt, - group, - url: `https://mangadex.org/chapter/${resource.id}`, - }; - }).sort((a, b) => Number(!!a.special) - Number(!!b.special) || (a.special ? a.number.localeCompare(b.number) : Number(a.number) - Number(b.number) || a.number.localeCompare(b.number))); -} - -function availabilitySummary(attributes, releases, chapters, externalReleases = null) { - const numbered = chapters.filter(ref => !ref.special); - const expected = chapterNumber(attributes.lastChapter); - const maximum = numbered.length ? Math.max(...numbered.map(ref => Number(ref.number))) : null; - // Reaching the final chapter number is not full coverage when the catalog - // contains fewer distinct numbered releases than the expected chapter span. - const expectedCount = expected && /^\d+$/.test(expected) ? Number(expected) : null; - const sparse = expectedCount && numbered.length < expectedCount; - const reason = chapters.length ? null : externalReleases > 0 ? 'external-only' - : Array.isArray(attributes.availableTranslatedLanguages) && !attributes.availableTranslatedLanguages.includes('en') ? 'no-english' : 'no-hosted-pages'; - const message = reason === 'external-only' ? `MangaDex lists ${externalReleases} English external-link releases, but no hosted English pages. External links are not downloaded.` - : reason === 'no-english' ? 'This MangaDex edition has no English releases listed.' - : reason ? 'This MangaDex edition has no hosted English releases with pages.' : null; - return { language: 'en', hostedReleases: releases.filter(r => !r.attributes?.externalUrl && Number(r.attributes?.pages) > 0).length, - selectedChapters: chapters.length, numberedChapters: numbered.length, specialReleases: chapters.length - numbered.length, - externalReleases, expectedLastChapter: expected, highestHostedChapter: maximum, - coverage: expected && (maximum === null || maximum < Number(expected) || sparse) ? 'partial' : 'unknown', reason, message }; -} - async function jsonGet(http, url) { const response = await http.get(url, { accept: 'application/json' }); const body = await response.json(); @@ -99,10 +27,7 @@ async function discoverMangaDexCatalog(seriesUrl, http) { if (!info) throw new Error('Invalid MangaDex title URL'); const manga = (await jsonGet(http, `${API}/manga/${info.id}`)).data; if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found'); - const isolated = !!process.env.MANGADEX_PLUGIN_URL; - const pluginMetadata = isolated - ? await parseSource('mangadex', 'metadata', { html: JSON.stringify(manga), url: `${API}/manga/${info.id}` }) - : null; + const metadata = parse('mangadex', 'metadata', { data: manga, url: `${API}/manga/${info.id}` }); const releases = []; let offset = 0; @@ -112,67 +37,45 @@ async function discoverMangaDexCatalog(seriesUrl, http) { params.append('includes[]', 'scanlation_group'); params.append('order[chapter]', 'asc'); const page = await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`); - if (isolated) { - const parsed = await parseSource('mangadex', 'feed', { html: JSON.stringify(page.data || []), url: `${API}/manga/${info.id}/feed` }); - releases.push(...parsed.releases); - } else releases.push(...(page.data || [])); + releases.push(...parse('mangadex', 'feed', { data: page.data || [], url: `${API}/manga/${info.id}/feed` }).releases); offset += page.data?.length || 0; if (!page.data?.length || offset >= Number(page.total || 0)) break; } - const title = isolated ? pluginMetadata.title : localizedTitle(manga.attributes); - const specialLabel = isolated ? pluginMetadata.specialLabel : /pre.serialization|pilot/i.test(title) ? 'Pilot' - : (manga.attributes?.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special'; - const pluginCatalog = async externalReleases => parseSource('mangadex', 'catalog', { - html: JSON.stringify({ attributes: pluginMetadata.availabilityAttributes, releases, specialLabel, externalReleases }), + const { title, specialLabel } = metadata; + const parseCatalog = externalReleases => parse('mangadex', 'catalog', { + data: { attributes: metadata.availabilityAttributes, releases, specialLabel, externalReleases }, url: `${API}/manga/${info.id}`, }); - let parsed = isolated ? await pluginCatalog(null) : null; - const chapters = isolated ? parsed.chapters : selectChapterReleases(releases, specialLabel); + let parsed = parseCatalog(null); + const chapters = parsed.chapters; // External-only is a separate feed mode, not a combined hosted+external catalog. let externalReleases = null; if (!chapters.length) { const params = new URLSearchParams({ limit: '1', includeFutureUpdates: '0', includeExternalUrl: '1' }); params.append('translatedLanguage[]', 'en'); externalReleases = Number((await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`)).total || 0); - if (isolated) parsed = await pluginCatalog(externalReleases); + parsed = parseCatalog(externalReleases); } return { id: info.id, title, - originalLanguage: isolated ? pluginMetadata.originalLanguage : manga.attributes?.originalLanguage || null, - links: isolated ? pluginMetadata.links : manga.attributes?.links || {}, - availability: isolated ? parsed.availability : availabilitySummary(manga.attributes || {}, releases, chapters, externalReleases), + originalLanguage: metadata.originalLanguage, + links: metadata.links, + availability: parsed.availability, chapters, }; } -function pageUrls(atHome, chapterUrl, expectedPages) { - const base = new URL(atHome.baseUrl); - if (base.protocol !== 'https:' || !(base.hostname === 'uploads.mangadex.org' || base.hostname.endsWith('.mangadex.network'))) { - throw new Error(`Unexpected MangaDex@Home host: ${base.hostname}`); - } - const hash = String(atHome.chapter?.hash || ''); - const files = atHome.chapter?.data; - if (!hash || !Array.isArray(files) || !files.length) throw new Error('MangaDex@Home returned no pages'); - if (expectedPages && files.length !== expectedPages) throw new Error(`MangaDex page count changed: expected ${expectedPages}, received ${files.length}`); - return files.map(file => ({ - url: new URL(`/data/${encodeURIComponent(hash)}/${encodeURIComponent(file)}`, base).href, - headers: { referer: chapterUrl }, - })); -} - async function discoverMangaDexChapter(ref, http) { const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`); const groupName = ref.group?.name ? ` ยท ${ref.group.name}` : ''; - const parsed = process.env.MANGADEX_PLUGIN_URL - ? await parseSource('mangadex', 'atHome', { html: JSON.stringify({ response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }), url: `${API}/at-home/server/${ref.id}` }) - : null; + const parsed = parse('mangadex', 'atHome', { data: { response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }, url: `${API}/at-home/server/${ref.id}` }); return { number: ref.number, title: ref.title || '', added: ref.added, ...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}), source: { name: `MangaDex${groupName}`, url: ref.url }, - pages: parsed ? parsed.pages : pageUrls(atHome, ref.url, ref.pages), + pages: parsed.pages, }; } diff --git a/scraper/adapters/weebcentral.js b/scraper/adapters/weebcentral.js index d8e8da9..ba5f1a4 100644 --- a/scraper/adapters/weebcentral.js +++ b/scraper/adapters/weebcentral.js @@ -3,16 +3,26 @@ const { SlowHttp } = require('../lib/http'); const { fallbackDirection } = require('../../reading-direction'); const parser = require('../plugins/weebcentral'); -const { parseSource } = require('../plugins/client'); +const { parse } = require('../plugins/registry'); + +function weebCentralInfo(value) { + let url; + try { url = new URL(String(value)); } catch { return null; } + if (url.protocol !== 'https:' || url.hostname !== 'weebcentral.com') return null; + const match = /^\/series\/([A-Z0-9]+)(?:\/([^/?#]+))?\/?$/i.exec(url.pathname); + if (!match) return null; + const title = decodeURIComponent(match[2] || match[1]).replace(/[-_]+/g, ' ').trim(); + return { source: 'weebcentral', id: match[1].toUpperCase(), title, url: `https://weebcentral.com/series/${match[1]}${match[2] ? `/${match[2]}` : ''}` }; +} const parseChapters = (html, baseUrl) => parser.catalog({ html, url: baseUrl }).chapters; const parsePageUrls = (html, baseUrl) => parser.pages({ html, url: baseUrl }).pages; async function discoverWeebCatalog(seriesUrl, http) { const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text(); - const { title, listUrl } = await parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl }); + const { title, listUrl } = parse('weebcentral', 'series', { html: seriesHtml, url: seriesUrl }); const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html' })).text() : seriesHtml; - const parsed = await parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }); + const parsed = parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }); const chapterLinks = parsed.chapterLinks; const chapters = parsed.chapters.sort((a, b) => Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number)); @@ -24,9 +34,9 @@ async function discoverWeebCatalog(seriesUrl, http) { async function discoverWeebChapter(chapter, http) { const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text(); - const { manifestUrl, added } = await parseSource('weebcentral', 'chapter', { html: pageHtml, url: chapter.url }); + const { manifestUrl, added } = parse('weebcentral', 'chapter', { html: pageHtml, url: chapter.url }); const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip¤t_page=1`, { accept: 'text/html' })).text(); - const parsed = await parseSource('weebcentral', 'pages', { html: fragment, url: manifestUrl }); + const parsed = parse('weebcentral', 'pages', { html: fragment, url: manifestUrl }); const pages = parsed.pages.map(url => ({ url, headers: { referer: chapter.url } })); if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`); return { @@ -55,4 +65,4 @@ async function discoverWeebCentral(seriesUrl, options = {}) { }; } -module.exports = { discoverWeebCentral, discoverWeebCatalog, discoverWeebChapter, parseChapters, parsePageUrls }; +module.exports = { weebCentralInfo, discoverWeebCentral, discoverWeebCatalog, discoverWeebChapter, parseChapters, parsePageUrls }; diff --git a/scraper/jobs/weebcentral-archive.js b/scraper/jobs/weebcentral-archive.js index f15d7d9..a4ce16c 100644 --- a/scraper/jobs/weebcentral-archive.js +++ b/scraper/jobs/weebcentral-archive.js @@ -2,7 +2,6 @@ const fs = require('fs').promises; const path = require('path'); -const { SlowHttp } = require('../lib/http'); const { createWeebHttp } = require('../plugins/broker'); const { sleep } = require('../lib/delay'); const { ArchiveControlError } = require('../lib/control'); @@ -73,7 +72,7 @@ async function runArchive(options) { minMs: options.minMs, maxMs: options.maxMs, maxRetries: 2, onEvent: item => event(item.level, item.message, { url: item.url, ...(item.status ? { status: item.status } : {}) }), }; - const http = process.env.SOURCE_PLUGIN_URL ? createWeebHttp(httpOptions) : new SlowHttp(httpOptions); + const http = createWeebHttp(httpOptions); checkControl(); console.log(`[archive] discovering catalog ${options.seriesUrl}`); diff --git a/scraper/plugins/client.js b/scraper/plugins/client.js deleted file mode 100644 index ddf83cc..0000000 --- a/scraper/plugins/client.js +++ /dev/null @@ -1,35 +0,0 @@ -'use strict'; - -const { parse } = require('./registry'); -const { validate } = require('./validation'); -const MAX_MESSAGE_BYTES = 2 * 1024 * 1024; - -async function boundedJson(response) { - const chunks = []; - let bytes = 0; - for await (const chunk of response.body) { - bytes += chunk.length; - if (bytes > MAX_MESSAGE_BYTES) throw new Error('Source plugin response exceeds limit'); - chunks.push(chunk); - } - return JSON.parse(Buffer.concat(chunks).toString('utf8')); -} - -async function parseSource(plugin, operation, input) { - const endpoint = plugin === 'mangadex' ? process.env.MANGADEX_PLUGIN_URL : process.env.SOURCE_PLUGIN_URL; - if (!endpoint) return validate(plugin, operation, parse(plugin, operation, input)); - const url = new URL('/v1/parse', endpoint); - if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint'); - const body = JSON.stringify({ plugin, operation, input }); - if (Buffer.byteLength(body) > MAX_MESSAGE_BYTES) throw new Error('Source plugin request exceeds limit'); - const response = await fetch(url, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body, - signal: AbortSignal.timeout(10000), - }); - if (!response.ok) throw new Error(`Source parser unavailable (${response.status}); checkpoint preserved`); - return validate(plugin, operation, await boundedJson(response)); -} - -module.exports = { parseSource }; diff --git a/scraper/plugins/mangadex.js b/scraper/plugins/mangadex.js index 00ddd37..0ee2209 100644 --- a/scraper/plugins/mangadex.js +++ b/scraper/plugins/mangadex.js @@ -87,8 +87,7 @@ function pageUrls(atHome, chapterUrl, expectedPages) { })); } -function metadata({ html }) { - const manga = JSON.parse(html); +function metadata({ data: manga }) { if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found'); const attributes = manga.attributes || {}; const title = localizedTitle(attributes); @@ -101,8 +100,7 @@ function metadata({ html }) { } }; } -function feed({ html }) { - const releases = JSON.parse(html); +function feed({ data: releases }) { if (!Array.isArray(releases)) throw new Error('Invalid MangaDex feed'); return { releases: releases.map(resource => ({ id: resource.id, @@ -116,15 +114,15 @@ function feed({ html }) { })) }; } -function catalog({ html }) { - const { attributes, releases, specialLabel, externalReleases } = JSON.parse(html); +function catalog({ data }) { + const { attributes, releases, specialLabel, externalReleases } = data; if (!attributes || !Array.isArray(releases)) throw new Error('Invalid MangaDex catalog input'); const chapters = selectChapterReleases(releases, specialLabel); return { chapters, availability: availabilitySummary(attributes, releases, chapters, externalReleases) }; } -function atHome({ html }) { - const { response, chapterUrl, expectedPages } = JSON.parse(html); +function atHome({ data }) { + const { response, chapterUrl, expectedPages } = data; return { pages: pageUrls(response, chapterUrl, expectedPages) }; } diff --git a/scraper/plugins/manifest.js b/scraper/plugins/manifest.js index 524acf3..d1b8de0 100644 --- a/scraper/plugins/manifest.js +++ b/scraper/plugins/manifest.js @@ -4,14 +4,14 @@ // source may add executable modules at runtime. const manifests = Object.freeze({ weebcentral: Object.freeze({ - id: 'weebcentral', version: 1, label: 'WeebCentral', status: 'pilot', + id: 'weebcentral', version: 1, label: 'WeebCentral', status: 'in-process', sourceHosts: Object.freeze(['weebcentral.com']), implementation: 'first-party', provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied', operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']), }), mangadex: Object.freeze({ - id: 'mangadex', version: 1, label: 'MangaDex', status: 'pilot', + id: 'mangadex', version: 1, label: 'MangaDex', status: 'in-process', sourceHosts: Object.freeze(['api.mangadex.org']), implementation: 'first-party', provenance: 'MangaCircus existing MangaDex adapter; no Uchiyomi code copied', diff --git a/scraper/plugins/registry.js b/scraper/plugins/registry.js index fb2a6f7..a74f230 100644 --- a/scraper/plugins/registry.js +++ b/scraper/plugins/registry.js @@ -2,6 +2,7 @@ // Reviewed first-party modules only. No filesystem scanning or runtime installs. const { manifests } = require('./manifest'); +const { validate } = require('./validation'); const plugins = Object.freeze({ weebcentral: require('./weebcentral'), mangadex: require('./mangadex'), @@ -11,13 +12,14 @@ function parse(plugin, operation, input) { const manifest = manifests[plugin]; const handler = manifest?.operations.includes(operation) ? plugins[plugin]?.[operation] : null; if (typeof handler !== 'function') throw new Error('Unknown source plugin or operation'); - if (!input || typeof input !== 'object' || typeof input.html !== 'string' || typeof input.url !== 'string') { + if (!input || typeof input !== 'object' || typeof input.url !== 'string' + || (typeof input.html !== 'string' && !(plugin === 'mangadex' && input.data && typeof input.data === 'object'))) { throw new Error('Invalid parser input'); } const url = new URL(input.url); if (!['https:', 'http:'].includes(url.protocol)) throw new Error('Invalid source URL'); if (!manifest.sourceHosts.includes(url.hostname)) throw new Error('Source URL does not belong to plugin'); - return handler({ html: input.html, url: url.href }); + return validate(plugin, operation, handler({ html: input.html, data: input.data, url: url.href })); } module.exports = { parse, manifests }; diff --git a/scraper/plugins/worker.js b/scraper/plugins/worker.js deleted file mode 100644 index aa737be..0000000 --- a/scraper/plugins/worker.js +++ /dev/null @@ -1,49 +0,0 @@ -'use strict'; - -const http = require('node:http'); -const { parse } = require('./registry'); -const MAX_BODY_BYTES = 2 * 1024 * 1024; - -function createServer(pluginId = process.env.SOURCE_PLUGIN_ID || null) { - return http.createServer((request, response) => { - if (request.method === 'GET' && request.url === '/health') { - response.writeHead(200, { 'content-type': 'application/json' }); - response.end(JSON.stringify({ ok: true })); - return; - } - if (request.method !== 'POST' || request.url !== '/v1/parse') { - response.writeHead(404).end(); - return; - } - let bytes = 0; - const chunks = []; - request.on('data', chunk => { - bytes += chunk.length; - if (bytes > MAX_BODY_BYTES) { - response.writeHead(413).end(); - request.destroy(); - return; - } - chunks.push(chunk); - }); - request.on('end', () => { - if (bytes > MAX_BODY_BYTES) return; - try { - const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8')); - if (pluginId && plugin !== pluginId) throw new Error('Plugin worker does not serve this source'); - const output = parse(plugin, operation, input); - response.writeHead(200, { 'content-type': 'application/json' }); - response.end(JSON.stringify(output)); - } catch (error) { - response.writeHead(400, { 'content-type': 'application/json' }); - response.end(JSON.stringify({ error: error.message })); - } - }); - }); -} - -if (require.main === module) { - createServer().listen(Number(process.env.SOURCE_PLUGIN_PORT || 8094), process.env.SOURCE_PLUGIN_HOST || '127.0.0.1'); -} - -module.exports = { createServer }; diff --git a/scraper/sources/common.js b/scraper/sources/common.js new file mode 100644 index 0000000..85a4118 --- /dev/null +++ b/scraper/sources/common.js @@ -0,0 +1,12 @@ +'use strict'; + +const { ArchiveControlError } = require('../lib/control'); + +function controlledHttp(http, shouldStop) { + return { async get(...args) { + const check = () => { const reason = shouldStop?.(); if (reason) throw new ArchiveControlError(reason === true ? 'paused' : String(reason)); }; + check(); const response = await http.get(...args); check(); return response; + } }; +} + +module.exports = { controlledHttp }; diff --git a/scraper/sources/mangadex.js b/scraper/sources/mangadex.js new file mode 100644 index 0000000..c49a27a --- /dev/null +++ b/scraper/sources/mangadex.js @@ -0,0 +1,26 @@ +'use strict'; + +const { SlowHttp } = require('../lib/http'); +const { mangaDexInfo, discoverMangaDexCatalog } = require('../adapters/mangadex'); +const { runMangaDexArchive } = require('../jobs/mangadex-archive'); +const { controlledHttp } = require('./common'); + +module.exports = { + id: 'mangadex', label: 'MangaDex', match: mangaDexInfo, + async discover(item, options) { + const http = new SlowHttp({ minMs: options.settings.minMs, maxMs: options.settings.maxMs, + maxRetries: 2, userAgent: 'MangaCircus-Scraper/1.0 (personal archival; sequential MangaDex requests)', + onEvent: options.onEvent }); + return discoverMangaDexCatalog(item.url, controlledHttp(http, options.shouldStop)); + }, + run(item, options) { + return runMangaDexArchive({ + seriesUrl: item.url, title: item.title, libraryRoot: options.libraryRoot, + metadataSearch: item.metadataSearch || item.title, + minMs: options.settings.minMs, maxMs: options.settings.maxMs, + chapterMinMs: options.settings.chapterMinMs, chapterMaxMs: options.settings.chapterMaxMs, + shouldStop: options.shouldStop, scope: item.scope, existingPolicy: item.existingPolicy, + libraryMangaId: item.libraryMangaId, onPublished: options.onPublished, onCatalog: options.onCatalog, + }); + }, +}; diff --git a/scraper/sources/weebcentral.js b/scraper/sources/weebcentral.js new file mode 100644 index 0000000..ae621ec --- /dev/null +++ b/scraper/sources/weebcentral.js @@ -0,0 +1,25 @@ +'use strict'; + +const { createWeebHttp } = require('../plugins/broker'); +const { discoverWeebCatalog, weebCentralInfo } = require('../adapters/weebcentral'); +const { runArchive } = require('../jobs/weebcentral-archive'); +const { controlledHttp } = require('./common'); + +module.exports = { + id: 'weebcentral', label: 'WeebCentral', match: weebCentralInfo, + async discover(item, options) { + const http = createWeebHttp({ minMs: options.settings.minMs, maxMs: options.settings.maxMs, + maxRetries: 2, onEvent: options.onEvent }); + return discoverWeebCatalog(item.url, controlledHttp(http, options.shouldStop)); + }, + run(item, options) { + return runArchive({ + seriesUrl: item.url, libraryRoot: options.libraryRoot, + metadataSearch: item.metadataSearch || item.title, + minMs: options.settings.minMs, maxMs: options.settings.maxMs, + chapterMinMs: options.settings.chapterMinMs, chapterMaxMs: options.settings.chapterMaxMs, + shouldStop: options.shouldStop, scope: item.scope, existingPolicy: item.existingPolicy, + libraryMangaId: item.libraryMangaId, onPublished: options.onPublished, onCatalog: options.onCatalog, + }); + }, +}; diff --git a/scraper/test/source-mangadex-plugin.test.js b/scraper/test/source-mangadex-plugin.test.js index 0584b2b..f98f1a6 100644 --- a/scraper/test/source-mangadex-plugin.test.js +++ b/scraper/test/source-mangadex-plugin.test.js @@ -2,8 +2,8 @@ const test = require('node:test'); const assert = require('node:assert/strict'); -const { createServer } = require('../plugins/worker'); const { discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex'); +const { parse } = require('../plugins/registry'); const mangaId = '123e4567-e89b-42d3-a456-426614174000'; const chapterIds = [ @@ -29,27 +29,12 @@ function fakeHttp() { } }; } -test('MangaDex isolated parser matches the existing catalog and page behavior', async () => { - const server = createServer('mangadex'); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const previous = process.env.MANGADEX_PLUGIN_URL; - delete process.env.MANGADEX_PLUGIN_URL; - try { - const baseline = await discoverMangaDexCatalog(seriesUrl, fakeHttp()); - const baselineChapter = await discoverMangaDexChapter(baseline.chapters[0], fakeHttp()); - process.env.MANGADEX_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`; - const isolated = await discoverMangaDexCatalog(seriesUrl, fakeHttp()); - const isolatedChapter = await discoverMangaDexChapter(isolated.chapters[0], fakeHttp()); - assert.deepEqual(isolated, baseline); - assert.deepEqual(isolatedChapter, baselineChapter); - const rejected = await fetch(`${process.env.MANGADEX_PLUGIN_URL}/v1/parse`, { - method: 'POST', headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ plugin: 'weebcentral', operation: 'series', input: { html: '

X

', url: 'https://weebcentral.com/series/X' } }), - }); - assert.equal(rejected.status, 400); - } finally { - if (previous === undefined) delete process.env.MANGADEX_PLUGIN_URL; - else process.env.MANGADEX_PLUGIN_URL = previous; - await new Promise(resolve => server.close(resolve)); - } +test('MangaDex parses a catalog and pages in the scraper process', async () => { + const catalog = await discoverMangaDexCatalog(seriesUrl, fakeHttp()); + const chapter = await discoverMangaDexChapter(catalog.chapters[0], fakeHttp()); + assert.equal(catalog.title, 'Fixture'); + assert.deepEqual(catalog.chapters.map(ref => ref.number), ['1', '2']); + assert.equal(chapter.pages.length, 2); + assert.match(chapter.pages[0].url, /^https:\/\/uploads\.mangadex\.org\/data\//); + assert.throws(() => parse('mangadex', 'metadata', { data: manga.data, url: 'https://other.example/manga/x' }), /does not belong/); }); diff --git a/scraper/test/source-plugin.test.js b/scraper/test/source-plugin.test.js index 84cf34b..7ce4db2 100644 --- a/scraper/test/source-plugin.test.js +++ b/scraper/test/source-plugin.test.js @@ -2,12 +2,10 @@ const test = require('node:test'); const assert = require('node:assert/strict'); -const httpModule = require('node:http'); -const { createServer } = require('../plugins/worker'); const { parse } = require('../plugins/registry'); -const { parseSource } = require('../plugins/client'); const { validate } = require('../plugins/validation'); const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral'); +const { adapters, adapterFor } = require('../adapters'); const seriesUrl = 'https://weebcentral.com/series/ABC/title'; const chapterUrl = 'https://weebcentral.com/chapters/XYZ'; @@ -32,11 +30,7 @@ test('parser output cannot turn catalog and image links into local or unrelated assert.doesNotThrow(() => validate('weebcentral', 'pages', { pages: ['https://img.example/1.jpg'] })); }); -test('isolated parser preserves WeebCentral catalog and page discovery', async () => { - const server = createServer(); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const previous = process.env.SOURCE_PLUGIN_URL; - process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`; +test('in-process parser preserves WeebCentral catalog and page discovery', async () => { const requested = []; const http = { async get(url) { requested.push(url); @@ -46,50 +40,22 @@ test('isolated parser preserves WeebCentral catalog and page discovery', async ( : imagesHtml; return { text: async () => body }; } }; - try { - const catalog = await discoverWeebCatalog(seriesUrl, http); - assert.equal(catalog.title, 'Test Manga'); - assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']); - const chapter = await discoverWeebChapter(catalog.chapters[0], http); - assert.equal(chapter.pages.length, 2); - assert.equal(chapter.added, '2026-01-01'); - assert.equal(requested.length, 4); - const response = await fetch(`${process.env.SOURCE_PLUGIN_URL}/v1/parse`, { - method: 'POST', headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ plugin: 'unapproved', operation: 'catalog', input: { html: listHtml, url: seriesUrl } }), - }); - assert.equal(response.status, 400); - } finally { - if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL; - else process.env.SOURCE_PLUGIN_URL = previous; - await new Promise(resolve => server.close(resolve)); - } + const catalog = await discoverWeebCatalog(seriesUrl, http); + assert.equal(catalog.title, 'Test Manga'); + assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']); + const chapter = await discoverWeebChapter(catalog.chapters[0], http); + assert.equal(chapter.pages.length, 2); + assert.equal(chapter.added, '2026-01-01'); + assert.equal(requested.length, 4); }); -test('configured worker failure does not fall back to local parsing', async () => { - const previous = process.env.SOURCE_PLUGIN_URL; - process.env.SOURCE_PLUGIN_URL = 'http://127.0.0.1:1'; - try { - await assert.rejects(parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl })); - } finally { - if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL; - else process.env.SOURCE_PLUGIN_URL = previous; - } -}); - -test('scraper rejects unsafe responses from a configured worker', async () => { - const server = httpModule.createServer((_request, response) => { - response.writeHead(200, { 'content-type': 'application/json' }); - response.end(JSON.stringify({ title: 'X', listUrl: 'http://127.0.0.1:8091/api/state' })); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const previous = process.env.SOURCE_PLUGIN_URL; - process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`; - try { - await assert.rejects(parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl }), /unexpected/); - } finally { - if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL; - else process.env.SOURCE_PLUGIN_URL = previous; - await new Promise(resolve => server.close(resolve)); +test('a new source is an explicit adapter, not another container or remote module', () => { + assert.deepEqual(adapters.map(adapter => adapter.id), ['weebcentral', 'mangadex']); + for (const adapter of adapters) { + assert.equal(typeof adapter.match, 'function'); + assert.equal(typeof adapter.discover, 'function'); + assert.equal(typeof adapter.run, 'function'); } + assert.equal(adapterFor(seriesUrl)?.adapter.id, 'weebcentral'); + assert.equal(adapterFor('https://unknown.example/series/ABC'), null); }); diff --git a/tools/healthcheck-source-plugin.js b/tools/healthcheck-source-plugin.js deleted file mode 100644 index 0c097cd..0000000 --- a/tools/healthcheck-source-plugin.js +++ /dev/null @@ -1,5 +0,0 @@ -'use strict'; - -fetch('http://127.0.0.1:8094/health', { signal: AbortSignal.timeout(3000) }) - .then(response => { if (!response.ok) process.exitCode = 1; }) - .catch(() => { process.exitCode = 1; });