diff --git a/scraper/adapters/weebcentral.js b/scraper/adapters/weebcentral.js index 5cd2ad6..d8e8da9 100644 --- a/scraper/adapters/weebcentral.js +++ b/scraper/adapters/weebcentral.js @@ -2,40 +2,19 @@ const { SlowHttp } = require('../lib/http'); const { fallbackDirection } = require('../../reading-direction'); +const parser = require('../plugins/weebcentral'); +const { parseSource } = require('../plugins/client'); -const decode = value => value.replace(/&/g, '&').replace(/'/g, "'").replace(/"/g, '"').replace(/<[^>]+>/g, '').trim(); - -function absolute(base, value) { return new URL(value, base).href; } - -function parseChapters(html, baseUrl) { - const found = []; - const re = /]*href="([^"]*\/chapters\/([A-Z0-9]+)[^"]*)"[^>]*>([\s\S]*?)<\/a>/gi; - for (const match of html.matchAll(re)) { - const text = decode(match[3]).replace(/\s+/g, ' '); - const season = Number(text.match(/^S(\d+)\s*-/i)?.[1] || 0) || null; - // A few WeebCentral catalogs use a series-specific label (for example - // "Rating 442", "Episode 128", "Night 182", "Log. 65", or "Act 60.6") in the same place where most catalogs - // say "Chapter 442". Treat those labels as chapter numbering instead of - // discarding the entire catalog. - const regular = text.match(/\b(?:Chapter|Rating|Episode|Night|Log|Act)\s*[.:]?\s*([0-9]+(?:\.[0-9]+)?)/i)?.[1]; - const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1]; - const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null); - if (number) found.push({ - id: match[2], number, title: regular ? '' : `Prologue ${prologue || ''}`.trim(), - ...(season ? { book: { number: season, title: `Season ${season}` } } : {}), - url: absolute(baseUrl, match[1]), raw: text, - }); - } - return [...new Map(found.map(chapter => [chapter.id, chapter])).values()]; -} +const parseChapters = (html, baseUrl) => parser.catalog({ html, url: baseUrl }).chapters; +const parsePageUrls = (html, baseUrl) => parser.pages({ html, url: baseUrl }).pages; async function discoverWeebCatalog(seriesUrl, http) { const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text(); - const title = decode(seriesHtml.match(/]*>([\s\S]*?)<\/h1>/i)?.[1] || 'Untitled'); - const listUrl = seriesHtml.match(/hx-get="([^"]+\/full-chapter-list)"/i)?.[1]; - const listHtml = listUrl ? await (await http.get(absolute(seriesUrl, listUrl), { accept: 'text/html' })).text() : seriesHtml; - const chapterLinks = [...listHtml.matchAll(/]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length; - const chapters = parseChapters(listHtml, seriesUrl).sort((a, b) => + const { title, listUrl } = await parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl }); + const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html' })).text() : seriesHtml; + const parsed = await parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }); + const chapterLinks = parsed.chapterLinks; + const chapters = parsed.chapters.sort((a, b) => Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number)); return { title, chapters, chapterLinks, ...(chapterLinks && !chapters.length ? { availability: { reason: 'unrecognized-labels', @@ -45,28 +24,19 @@ async function discoverWeebCatalog(seriesUrl, http) { async function discoverWeebChapter(chapter, http) { const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text(); - const manifestUrl = pageHtml.match(/['"](https?:\/\/weebcentral\.com\/chapters\/[^'"]+\/images\?is_prev=[^'"]+)['"]/i)?.[1] - || `${chapter.url}/images?is_prev=False`; + const { manifestUrl, added } = await parseSource('weebcentral', 'chapter', { html: pageHtml, url: chapter.url }); const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip¤t_page=1`, { accept: 'text/html' })).text(); - const pages = parsePageUrls(fragment, manifestUrl).map(url => ({ url, headers: { referer: chapter.url } })); + const parsed = await parseSource('weebcentral', 'pages', { html: fragment, url: manifestUrl }); + const pages = parsed.pages.map(url => ({ url, headers: { referer: chapter.url } })); if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`); return { number: chapter.number, title: chapter.title || '', ...(chapter.book ? { book: chapter.book } : {}), - added: pageHtml.match(/datetime="([^"]+)"/i)?.[1], + added, source: { name: 'WeebCentral', url: chapter.url }, pages, }; } -function parsePageUrls(html, baseUrl) { - const urls = []; - for (const match of html.matchAll(/]*(?:src|data-src)="([^"]+)"/gi)) { - const url = absolute(baseUrl, match[1].replace(/&/g, '&')); - if (!url.startsWith('data:')) urls.push(url); - } - return [...new Set(urls)]; -} - async function discoverWeebCentral(seriesUrl, options = {}) { const http = new SlowHttp(options.http || {}); const catalog = await discoverWeebCatalog(seriesUrl, http); diff --git a/scraper/plugins/client.js b/scraper/plugins/client.js new file mode 100644 index 0000000..9d3e966 --- /dev/null +++ b/scraper/plugins/client.js @@ -0,0 +1,20 @@ +'use strict'; + +const { parse } = require('./registry'); + +async function parseSource(plugin, operation, input) { + const endpoint = process.env.SOURCE_PLUGIN_URL; + if (!endpoint) return parse(plugin, operation, input); + const url = new URL('/v1/parse', endpoint); + if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint'); + const response = await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ plugin, operation, input }), + signal: AbortSignal.timeout(10000), + }); + if (!response.ok) throw new Error(`Source parser unavailable (${response.status}); checkpoint preserved`); + return response.json(); +} + +module.exports = { parseSource }; diff --git a/scraper/plugins/manifest.js b/scraper/plugins/manifest.js new file mode 100644 index 0000000..e547079 --- /dev/null +++ b/scraper/plugins/manifest.js @@ -0,0 +1,15 @@ +'use strict'; + +// This list is shipped with MangaCircus. Neither the admin UI nor a remote +// source may add executable modules at runtime. +const manifests = Object.freeze({ + weebcentral: Object.freeze({ + id: 'weebcentral', version: 1, label: 'WeebCentral', status: 'pilot', + sourceHosts: Object.freeze(['weebcentral.com']), + implementation: 'first-party', + provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied', + operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']), + }), +}); + +module.exports = { manifests }; diff --git a/scraper/plugins/registry.js b/scraper/plugins/registry.js new file mode 100644 index 0000000..eca7382 --- /dev/null +++ b/scraper/plugins/registry.js @@ -0,0 +1,20 @@ +'use strict'; + +// Reviewed first-party modules only. No filesystem scanning or runtime installs. +const { manifests } = require('./manifest'); +const plugins = Object.freeze({ weebcentral: require('./weebcentral') }); + +function parse(plugin, operation, input) { + const manifest = manifests[plugin]; + const handler = manifest?.operations.includes(operation) ? plugins[plugin]?.[operation] : null; + if (typeof handler !== 'function') throw new Error('Unknown source plugin or operation'); + if (!input || typeof input !== 'object' || typeof input.html !== 'string' || typeof input.url !== 'string') { + throw new Error('Invalid parser input'); + } + const url = new URL(input.url); + if (!['https:', 'http:'].includes(url.protocol)) throw new Error('Invalid source URL'); + if (!manifest.sourceHosts.includes(url.hostname)) throw new Error('Source URL does not belong to plugin'); + return handler({ html: input.html, url: url.href }); +} + +module.exports = { parse, manifests }; diff --git a/scraper/plugins/weebcentral.js b/scraper/plugins/weebcentral.js new file mode 100644 index 0000000..7e83b0e --- /dev/null +++ b/scraper/plugins/weebcentral.js @@ -0,0 +1,50 @@ +'use strict'; + +// First-party, pure parsing functions. This module never fetches remote content. +const decode = value => value.replace(/&/g, '&').replace(/'/g, "'").replace(/"/g, '"').replace(/<[^>]+>/g, '').trim(); +const absolute = (base, value) => new URL(value, base).href; + +function series({ html, url }) { + const title = decode(html.match(/]*>([\s\S]*?)<\/h1>/i)?.[1] || 'Untitled'); + const listUrl = html.match(/hx-get="([^"]+\/full-chapter-list)"/i)?.[1]; + return { title, listUrl: listUrl ? absolute(url, listUrl) : null }; +} + +function catalog({ html, url }) { + const found = []; + const re = /]*href="([^"]*\/chapters\/([A-Z0-9]+)[^"]*)"[^>]*>([\s\S]*?)<\/a>/gi; + for (const match of html.matchAll(re)) { + const text = decode(match[3]).replace(/\s+/g, ' '); + const season = Number(text.match(/^S(\d+)\s*-/i)?.[1] || 0) || null; + const regular = text.match(/\b(?:Chapter|Rating|Episode|Night|Log|Act)\s*[.:]?\s*([0-9]+(?:\.[0-9]+)?)/i)?.[1]; + const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1]; + const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null); + if (number) found.push({ + id: match[2], number, title: regular ? '' : `Prologue ${prologue || ''}`.trim(), + ...(season ? { book: { number: season, title: `Season ${season}` } } : {}), + url: absolute(url, match[1]), raw: text, + }); + } + const chapters = [...new Map(found.map(chapter => [chapter.id, chapter])).values()]; + const chapterLinks = [...html.matchAll(/]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length; + return { chapters, chapterLinks }; +} + +function chapter({ html, url }) { + return { + manifestUrl: html.match(/['"](https?:\/\/weebcentral\.com\/chapters\/[^'"]+\/images\?is_prev=[^'"]+)['"]/i)?.[1] + || `${url}/images?is_prev=False`, + added: html.match(/datetime="([^"]+)"/i)?.[1], + }; +} + +function pages({ html, url }) { + const urls = []; + for (const match of html.matchAll(/]*(?:src|data-src)="([^"]+)"/gi)) { + const pageUrl = absolute(url, match[1].replace(/&/g, '&')); + if (!pageUrl.startsWith('data:')) urls.push(pageUrl); + } + return { pages: [...new Set(urls)] }; +} + +module.exports = { series, catalog, chapter, pages }; diff --git a/scraper/plugins/worker.js b/scraper/plugins/worker.js new file mode 100644 index 0000000..7858910 --- /dev/null +++ b/scraper/plugins/worker.js @@ -0,0 +1,48 @@ +'use strict'; + +const http = require('node:http'); +const { parse } = require('./registry'); +const MAX_BODY_BYTES = 2 * 1024 * 1024; + +function createServer() { + return http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/health') { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ ok: true })); + return; + } + if (request.method !== 'POST' || request.url !== '/v1/parse') { + response.writeHead(404).end(); + return; + } + let bytes = 0; + const chunks = []; + request.on('data', chunk => { + bytes += chunk.length; + if (bytes > MAX_BODY_BYTES) { + response.writeHead(413).end(); + request.destroy(); + return; + } + chunks.push(chunk); + }); + request.on('end', () => { + if (bytes > MAX_BODY_BYTES) return; + try { + const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8')); + const output = parse(plugin, operation, input); + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify(output)); + } catch (error) { + response.writeHead(400, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ error: error.message })); + } + }); + }); +} + +if (require.main === module) { + createServer().listen(Number(process.env.SOURCE_PLUGIN_PORT || 8094), process.env.SOURCE_PLUGIN_HOST || '127.0.0.1'); +} + +module.exports = { createServer }; diff --git a/scraper/test/source-plugin.test.js b/scraper/test/source-plugin.test.js new file mode 100644 index 0000000..c38af47 --- /dev/null +++ b/scraper/test/source-plugin.test.js @@ -0,0 +1,67 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { createServer } = require('../plugins/worker'); +const { parse } = require('../plugins/registry'); +const { parseSource } = require('../plugins/client'); +const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral'); + +const seriesUrl = 'https://weebcentral.com/series/ABC/title'; +const chapterUrl = 'https://weebcentral.com/chapters/XYZ'; +const seriesHtml = '

Test Manga

'; +const listHtml = 'Chapter 1Log. 2'; +const chapterHtml = ''; +const imagesHtml = ''; + +test('first-party registry rejects unknown plugins and malformed input', () => { + assert.throws(() => parse('remote-module', 'catalog', { html: listHtml, url: seriesUrl }), /Unknown/); + assert.throws(() => parse('weebcentral', 'catalog', { html: listHtml, url: 'file:///tmp/x' }), /Invalid source URL/); + assert.throws(() => parse('weebcentral', 'catalog', { html: listHtml, url: 'https://other.example/series/a' }), /does not belong/); + assert.equal(parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }).chapters.length, 2); +}); + +test('isolated parser preserves WeebCentral catalog and page discovery', async () => { + const server = createServer(); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const previous = process.env.SOURCE_PLUGIN_URL; + process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`; + const requested = []; + const http = { async get(url) { + requested.push(url); + const body = url === seriesUrl ? seriesHtml + : url.endsWith('/full-chapter-list') ? listHtml + : url === chapterUrl ? chapterHtml + : imagesHtml; + return { text: async () => body }; + } }; + try { + const catalog = await discoverWeebCatalog(seriesUrl, http); + assert.equal(catalog.title, 'Test Manga'); + assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']); + const chapter = await discoverWeebChapter(catalog.chapters[0], http); + assert.equal(chapter.pages.length, 2); + assert.equal(chapter.added, '2026-01-01'); + assert.equal(requested.length, 4); + const response = await fetch(`${process.env.SOURCE_PLUGIN_URL}/v1/parse`, { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ plugin: 'unapproved', operation: 'catalog', input: { html: listHtml, url: seriesUrl } }), + }); + assert.equal(response.status, 400); + } finally { + if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL; + else process.env.SOURCE_PLUGIN_URL = previous; + await new Promise(resolve => server.close(resolve)); + } +}); + +test('configured worker failure does not fall back to local parsing', async () => { + const previous = process.env.SOURCE_PLUGIN_URL; + process.env.SOURCE_PLUGIN_URL = 'http://127.0.0.1:1'; + try { + await assert.rejects(parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl })); + } finally { + if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL; + else process.env.SOURCE_PLUGIN_URL = previous; + } +}); diff --git a/tools/healthcheck-source-plugin.js b/tools/healthcheck-source-plugin.js new file mode 100644 index 0000000..0c097cd --- /dev/null +++ b/tools/healthcheck-source-plugin.js @@ -0,0 +1,5 @@ +'use strict'; + +fetch('http://127.0.0.1:8094/health', { signal: AbortSignal.timeout(3000) }) + .then(response => { if (!response.ok) process.exitCode = 1; }) + .catch(() => { process.exitCode = 1; }); diff --git a/vision/first-party-source-plugins.md b/vision/first-party-source-plugins.md new file mode 100644 index 0000000..bae36ed --- /dev/null +++ b/vision/first-party-source-plugins.md @@ -0,0 +1,82 @@ +# First-party source plugins for MangaCircus + +Prepared 2026-10-13. The first implementation slice extracts the WeebCentral +parser into a static first-party registry and adds an opt-in, network-isolated +Compose worker. No external plugin was installed, no host was probed, and no +live queue, delay, or container was changed. The broker, generic runner, new +hoster pilot, admin controls, and full deployment validation below remain work +to do; the diagram describes the target rather than the current implementation. + +## Decision + +Keep MangaCircus's no-database library, source-aware queue, checkpoints, source ledger, and chapter publisher. Extract source-specific discovery/parsing into versioned, first-party plugins. Review and build every plugin from **our repository**; never install an arbitrary repository, APK, JavaScript package, or remotely supplied parser at runtime. An admin may enable/disable a reviewed plugin, not supply executable code. A catalog match remains distinct from adapter compatibility, actual chapter coverage, and download rights. + +The upstream model is Uchiyomi v0.61.1, commit [`726bd9fa6fe0090fd4b5de1cdb2b6926e5382127`](https://github.com/AngeloSha/uchiyomi/tree/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127). Uchiyomi registers a common `search/getSeries/listChapters/getPageUrls` interface across built-in MangaDex, generic site engines, mounted JS source packs, and Suwayomi-backed Mihon/Tachiyomi extensions. We want the interface idea and selected reviewed parsers, **not** its executable-code loader or Suwayomi/FlareSolverr runtime. + +## Existing seam and problem + +- `scraper/adapters/index.js` already dispatches `match/discover/run`; `scraper/service.js` schedules one job at a time per source, applies scopes and source settings, records evidence, and owns the durable queue. +- `scraper/adapters/{weebcentral,mangadex}.js` parse source catalogs and pages. `scraper/jobs/{weebcentral,mangadex}-archive.js` repeat transport, checkpoint, metadata, and publication steps. `scraper/lib/http.js` stops on 401/403/429. `scraper/lib/publisher.js` makes chapters visible atomically. +- Adding another hoster today means copying a whole archive runner. The plugin work must eliminate that repetition **without** allowing plugin code to write the library or bypass request controls. + +## Boundary + +```text +Admin/UI -> queue + source ledger -> scraper scheduler -> first-party plugin worker + | ^ + v | request/response IPC + HTTP broker ----------+ + | + v + host / approved image CDN + +scraper scheduler -> library inventory + scope/dedupe -> common downloader + -> existing checkpoint/publisher -> library +``` + +The worker describes requests and parses replies. It has no library or state mount and no internet-facing network. The scraper-owned broker makes **every** source request, including image requests. It enforces one source lane, configured request/chapter delays, declared hostname allowlists, redirect and resolved-address checks, response size/time limits, and stop-on-401/403/429/challenge behavior. HTTP 5xx and network retry remain centrally bounded. No proxy/identity/user-agent rotation and no challenge solver. A source that requires a protection bypass is reported as unsupported/blocked, not silently routed elsewhere. + +For production Compose, run one small `plugin-worker` instance per enabled plugin from the same pinned first-party image. Give each a dedicated internal-only network shared with the scraper (not the general backend or `egress` network); mount no library/data/secrets; use a read-only filesystem, dropped capabilities, no-new-privileges, and memory/PID limits. Do **not** inherit Compose's existing `x-app` anchor, which mounts the scraper token. Only the scraper has egress and library write access. On Windows local development, use the same Compose worker: an ordinary Node child process is crash isolation but **not** a network sandbox. No exposed plugin port or browser-to-worker route. + +### Proposed v1 IPC contract + +Transport: versioned JSON messages over a private internal HTTP endpoint or stdio; bounded payloads and request IDs. The exact transport is an implementation choice, not exposed to users. A worker is pinned to one `pluginId` and code revision. The host validates every returned object. + +```json +{ "v": 1, "id": "req-1", "op": "catalog", "pluginId": "example", "seriesUrl": "https://allowed.example/series/123" } +{ "v": 1, "id": "req-2", "op": "fetch", "url": "https://allowed.example/series/123", "accept": "text/html" } +{ "v": 1, "id": "req-2", "ok": true, "status": 200, "contentType": "text/html", "body": "...bounded..." } +{ "v": 1, "id": "req-1", "ok": true, "title": "...", "chapters": [{ "id": "source-chapter-id", "number": "1", "volume": "1", "url": "https://allowed.example/chapter/1" }], "coverage": "unknown" } +``` + +Operations: `match` from a declarative manifest, `catalog`, `chapterPages`, optional `search`/`seriesMetadata` later. `chapterPages` returns ordered image descriptors; **only** the common downloader fetches bytes. Plugin output may report `unrecognized-labels`, `external-only`, and confidence/evidence, but may not mark rights verified or override the library's work identity. Plugin failures are typed (`parser`, `catalog-empty`, `network`, `access-block`, `rate-limit`) and visible in the source ledger. Version/schema mismatch disables that plugin lane without losing checkpoints. + +Manifest, reviewed with code: stable plugin ID/version, exact series URL matcher, primary hostnames, separately reviewed image/CDN hostnames, optional language, parser fixture set, upstream provenance, license, and support state (`experimental`/`supported`/`disabled`). Hostnames are **not** administrator-provided at runtime in v1. Redirects and DNS answers must be checked again by the broker; declared hostnames alone do not protect against SSRF. + +## Uchiyomi code triage + +| Upstream implementation | Use in MangaCircus | Rationale | +| --- | --- | --- | +| [`sources/types.ts`](https://github.com/AngeloSha/uchiyomi/blob/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127/bff/src/lib/sources/types.ts) | Adapt **concept**, write our own contract | Its narrow catalog/page operations are valuable; its Cloudflare/image-header capabilities and database-specific assumptions are not our policy. | +| [`sources/engines/madara.ts`](https://github.com/AngeloSha/uchiyomi/blob/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127/bff/src/lib/sources/engines/madara.ts) | Candidate for pure parser extraction | Broad WordPress/Madara markup; separate parsing from `cfGet/cfPost`, test with fixtures, then trial only on a specifically reviewed accessible host. Not a blanket license to scrape every Madara site. | +| [`sources/engines/mangathemesia.ts`](https://github.com/AngeloSha/uchiyomi/blob/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127/bff/src/lib/sources/engines/mangathemesia.ts) | Later candidate | Similar, but source-specific page and image hosts need review; currently solver-coupled. | +| [`sources/engines/manganato.ts`](https://github.com/AngeloSha/uchiyomi/blob/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127/bff/src/lib/sources/engines/manganato.ts) | Later candidate, not first pilot | More variant/fallback logic and an explicit challenge dependency; higher maintenance and source-policy risk. | +| [`sources/mangadex.ts`](https://github.com/AngeloSha/uchiyomi/blob/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127/bff/src/lib/sources/mangadex.ts) | Compare behavior/tests only | MangaCircus already has a MangaDex adapter. Evaluate its language and rate-limit edge cases without replacing working code wholesale. | +| [`sources/suwayomi/*`](https://github.com/AngeloSha/uchiyomi/tree/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127/bff/src/lib/sources/suwayomi), [`loader.ts`](https://github.com/AngeloSha/uchiyomi/blob/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127/bff/src/lib/sources/loader.ts) | Do not port | Designed to run operator-supplied APKs/JS. Conflicts with the first-party-only requirement. | +| [`downloader.ts`](https://github.com/AngeloSha/uchiyomi/blob/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127/bff/src/lib/downloader.ts) | Learn from tests/failure cases, do not transplant | CBZ/database assumptions differ from MangaCircus's folder/checkpoint publisher; its challenge session handling does not fit our policy. | + +The upstream `sources/` directory contains no bundled plugin implementations beyond a placeholder; the generic engines above live in its main server. Do not mistake its extension catalog for code licensed by Uchiyomi: separately authored APKs have separately determined provenance and licenses. + +### Code provenance and license gate + +Uchiyomi is [MPL-2.0](https://github.com/AngeloSha/uchiyomi/blob/726bd9fa6fe0090fd4b5de1cdb2b6926e5382127/LICENSE). **No verbatim or adapted upstream code has been copied into MangaCircus in this preparation.** Before copying any implementation, identify the exact upstream file/revision and third-party notices, preserve attribution/notices, add the MPL license and a `THIRD_PARTY_SOURCES.md` ledger, and keep copied/modified MPL-covered code in clearly marked files. If distributing a Docker image with those files, provide the corresponding source for the covered files as the license requires. Mozilla describes MPL as file-level copyleft; it does not require unrelated MangaCircus files to become MPL. This is an implementation checklist, not legal advice; review the final distribution arrangement before release. Reimplementing an idea from documented behavior is distinct from copying its expressive code, but provenance should still be documented. + +## Incremental implementation passes + +1. **Contract and broker, no new host.** Add manifest validation and IPC schema, central fetch broker, disabled-by-default worker, fixture-driven contract tests, worker crash/timeouts, SSRF/redirect tests, 401/403/429 stop tests, and checkpoint-preservation tests. Existing adapters continue unchanged. +2. **First-party compatibility plugins.** Migrate MangaDex and WeebCentral catalog/page parsing one at a time behind the contract, with old-vs-new fixture parity. Keep existing job IDs, source IDs, queue order, delays, and work directory paths. The common runner calls the existing inventory/scope/publisher. Rollback is a plugin enable switch, not a queue rewrite. +3. **One reviewed generic parser pilot.** Start with a pure Madara-family parser based on upstream lessons. Choose a single source only after checking accessibility, terms, stable hostnames, and parser fixtures. Inspection/catalog-only first; no auto-queue. Record source evidence, compare with library identity, then test a small chapter in isolated A/B fixtures. Add further families only if they solve a concrete catalog gap. +4. **Admin UI.** Under Scraper → Sources, show installed first-party plugins, version/provenance, enabled state, supported hosts, last probe stage, observed catalog/coverage, rights state, and blocked reason. Enabling requires admin confirmation; no repository URL/APK upload/install button. Source candidates stay leads until reviewed and explicitly queued. +5. **Deployment.** Add the worker to local and VM Compose with internal-only networking and resource limits; run compose checks and end-to-end fixture tests, then stage on a test instance. Leave the live stack untouched until jobs are checkpoint-safe and rollback verified. Document plugin upgrades and recovery. + +Acceptance: no plugin has direct internet or library-write capability; one worker per plugin may crash without taking down the reader/scraper; no source request can evade host limits or 401/403/429 handling; existing jobs resume unchanged across a rollback; the library still needs no database; source provenance and MPL obligations are traceable; no dynamic third-party code execution is introduced.