Maashkinoozhe/MangaCircus
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Checkpoint first-party source parser pilot

Browse Source
This commit is contained in:
Codex committed 2026-10-13 21:45:27 +02:00
1 parent 605fa5b9d8
commit c5a92282a9
9 files changed
+320 -43
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scraper/adapters/weebcentral.js
+13 -43
View File
@@ -2,40 +2,19 @@
const { SlowHttp } = require('../lib/http');
const { fallbackDirection } = require('../../reading-direction');
const parser = require('../plugins/weebcentral');
const { parseSource } = require('../plugins/client');
const decode = value => value.replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/<[^>]+>/g, '').trim();
function absolute(base, value) { return new URL(value, base).href; }
function parseChapters(html, baseUrl) {
const found = [];
const re = /<a\b[^>]*href="([^"]*\/chapters\/([A-Z0-9]+)[^"]*)"[^>]*>([\s\S]*?)<\/a>/gi;
for (const match of html.matchAll(re)) {
const text = decode(match[3]).replace(/\s+/g, ' ');
const season = Number(text.match(/^S(\d+)\s*-/i)?.[1] || 0) || null;
// A few WeebCentral catalogs use a series-specific label (for example
// "Rating 442", "Episode 128", "Night 182", "Log. 65", or "Act 60.6") in the same place where most catalogs
// say "Chapter 442". Treat those labels as chapter numbering instead of
// discarding the entire catalog.
const regular = text.match(/\b(?:Chapter|Rating|Episode|Night|Log|Act)\s*[.:]?\s*([0-9]+(?:\.[0-9]+)?)/i)?.[1];
const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1];
const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null);
if (number) found.push({
id: match[2], number, title: regular ? '' : `Prologue ${prologue || ''}`.trim(),
...(season ? { book: { number: season, title: `Season ${season}` } } : {}),
url: absolute(baseUrl, match[1]), raw: text,
});
}
return [...new Map(found.map(chapter => [chapter.id, chapter])).values()];
}
const parseChapters = (html, baseUrl) => parser.catalog({ html, url: baseUrl }).chapters;
const parsePageUrls = (html, baseUrl) => parser.pages({ html, url: baseUrl }).pages;
async function discoverWeebCatalog(seriesUrl, http) {
const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text();
const title = decode(seriesHtml.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || 'Untitled');
const listUrl = seriesHtml.match(/hx-get="([^"]+\/full-chapter-list)"/i)?.[1];
const listHtml = listUrl ? await (await http.get(absolute(seriesUrl, listUrl), { accept: 'text/html' })).text() : seriesHtml;
const chapterLinks = [...listHtml.matchAll(/<a\b[^>]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length;
const chapters = parseChapters(listHtml, seriesUrl).sort((a, b) =>
const { title, listUrl } = await parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl });
const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html' })).text() : seriesHtml;
const parsed = await parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl });
const chapterLinks = parsed.chapterLinks;
const chapters = parsed.chapters.sort((a, b) =>
Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number));
return { title, chapters, chapterLinks,
...(chapterLinks && !chapters.length ? { availability: { reason: 'unrecognized-labels',
@@ -45,28 +24,19 @@ async function discoverWeebCatalog(seriesUrl, http) {
async function discoverWeebChapter(chapter, http) {
const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text();
const manifestUrl = pageHtml.match(/['"](https?:\/\/weebcentral\.com\/chapters\/[^'"]+\/images\?is_prev=[^'"]+)['"]/i)?.[1]
|| `${chapter.url}/images?is_prev=False`;
const { manifestUrl, added } = await parseSource('weebcentral', 'chapter', { html: pageHtml, url: chapter.url });
const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip&current_page=1`, { accept: 'text/html' })).text();
const pages = parsePageUrls(fragment, manifestUrl).map(url => ({ url, headers: { referer: chapter.url } }));
const parsed = await parseSource('weebcentral', 'pages', { html: fragment, url: manifestUrl });
const pages = parsed.pages.map(url => ({ url, headers: { referer: chapter.url } }));
if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`);
return {
number: chapter.number, title: chapter.title || '',
...(chapter.book ? { book: chapter.book } : {}),
added: pageHtml.match(/datetime="([^"]+)"/i)?.[1],
added,
source: { name: 'WeebCentral', url: chapter.url }, pages,
};
}
function parsePageUrls(html, baseUrl) {
const urls = [];
for (const match of html.matchAll(/<img\b[^>]*(?:src|data-src)="([^"]+)"/gi)) {
const url = absolute(baseUrl, match[1].replace(/&amp;/g, '&'));
if (!url.startsWith('data:')) urls.push(url);
}
return [...new Set(urls)];
}
async function discoverWeebCentral(seriesUrl, options = {}) {
const http = new SlowHttp(options.http || {});
const catalog = await discoverWeebCatalog(seriesUrl, http);
scraper/plugins/client.js
+20
View File
@@ -0,0 +1,20 @@
'use strict';
const { parse } = require('./registry');
async function parseSource(plugin, operation, input) {
const endpoint = process.env.SOURCE_PLUGIN_URL;
if (!endpoint) return parse(plugin, operation, input);
const url = new URL('/v1/parse', endpoint);
if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid source plugin endpoint');
const response = await fetch(url, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ plugin, operation, input }),
signal: AbortSignal.timeout(10000),
});
if (!response.ok) throw new Error(`Source parser unavailable (${response.status}); checkpoint preserved`);
return response.json();
}
module.exports = { parseSource };
scraper/plugins/manifest.js
+15
View File
@@ -0,0 +1,15 @@
'use strict';
// This list is shipped with MangaCircus. Neither the admin UI nor a remote
// source may add executable modules at runtime.
const manifests = Object.freeze({
weebcentral: Object.freeze({
id: 'weebcentral', version: 1, label: 'WeebCentral', status: 'pilot',
sourceHosts: Object.freeze(['weebcentral.com']),
implementation: 'first-party',
provenance: 'MangaCircus existing WeebCentral adapter; no Uchiyomi code copied',
operations: Object.freeze(['series', 'catalog', 'chapter', 'pages']),
}),
});
module.exports = { manifests };
scraper/plugins/registry.js
+20
View File
@@ -0,0 +1,20 @@
'use strict';
// Reviewed first-party modules only. No filesystem scanning or runtime installs.
const { manifests } = require('./manifest');
const plugins = Object.freeze({ weebcentral: require('./weebcentral') });
function parse(plugin, operation, input) {
const manifest = manifests[plugin];
const handler = manifest?.operations.includes(operation) ? plugins[plugin]?.[operation] : null;
if (typeof handler !== 'function') throw new Error('Unknown source plugin or operation');
if (!input || typeof input !== 'object' || typeof input.html !== 'string' || typeof input.url !== 'string') {
throw new Error('Invalid parser input');
}
const url = new URL(input.url);
if (!['https:', 'http:'].includes(url.protocol)) throw new Error('Invalid source URL');
if (!manifest.sourceHosts.includes(url.hostname)) throw new Error('Source URL does not belong to plugin');
return handler({ html: input.html, url: url.href });
}
module.exports = { parse, manifests };
scraper/plugins/weebcentral.js
+50
View File
@@ -0,0 +1,50 @@
'use strict';
// First-party, pure parsing functions. This module never fetches remote content.
const decode = value => value.replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/<[^>]+>/g, '').trim();
const absolute = (base, value) => new URL(value, base).href;
function series({ html, url }) {
const title = decode(html.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || 'Untitled');
const listUrl = html.match(/hx-get="([^"]+\/full-chapter-list)"/i)?.[1];
return { title, listUrl: listUrl ? absolute(url, listUrl) : null };
}
function catalog({ html, url }) {
const found = [];
const re = /<a\b[^>]*href="([^"]*\/chapters\/([A-Z0-9]+)[^"]*)"[^>]*>([\s\S]*?)<\/a>/gi;
for (const match of html.matchAll(re)) {
const text = decode(match[3]).replace(/\s+/g, ' ');
const season = Number(text.match(/^S(\d+)\s*-/i)?.[1] || 0) || null;
const regular = text.match(/\b(?:Chapter|Rating|Episode|Night|Log|Act)\s*[.:]?\s*([0-9]+(?:\.[0-9]+)?)/i)?.[1];
const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1];
const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null);
if (number) found.push({
id: match[2], number, title: regular ? '' : `Prologue ${prologue || ''}`.trim(),
...(season ? { book: { number: season, title: `Season ${season}` } } : {}),
url: absolute(url, match[1]), raw: text,
});
}
const chapters = [...new Map(found.map(chapter => [chapter.id, chapter])).values()];
const chapterLinks = [...html.matchAll(/<a\b[^>]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length;
return { chapters, chapterLinks };
}
function chapter({ html, url }) {
return {
manifestUrl: html.match(/['"](https?:\/\/weebcentral\.com\/chapters\/[^'"]+\/images\?is_prev=[^'"]+)['"]/i)?.[1]
|| `${url}/images?is_prev=False`,
added: html.match(/datetime="([^"]+)"/i)?.[1],
};
}
function pages({ html, url }) {
const urls = [];
for (const match of html.matchAll(/<img\b[^>]*(?:src|data-src)="([^"]+)"/gi)) {
const pageUrl = absolute(url, match[1].replace(/&amp;/g, '&'));
if (!pageUrl.startsWith('data:')) urls.push(pageUrl);
}
return { pages: [...new Set(urls)] };
}
module.exports = { series, catalog, chapter, pages };
scraper/plugins/worker.js
+48
View File
@@ -0,0 +1,48 @@
'use strict';
const http = require('node:http');
const { parse } = require('./registry');
const MAX_BODY_BYTES = 2 * 1024 * 1024;
function createServer() {
return http.createServer((request, response) => {
if (request.method === 'GET' && request.url === '/health') {
response.writeHead(200, { 'content-type': 'application/json' });
response.end(JSON.stringify({ ok: true }));
return;
}
if (request.method !== 'POST' || request.url !== '/v1/parse') {
response.writeHead(404).end();
return;
}
let bytes = 0;
const chunks = [];
request.on('data', chunk => {
bytes += chunk.length;
if (bytes > MAX_BODY_BYTES) {
response.writeHead(413).end();
request.destroy();
return;
}
chunks.push(chunk);
});
request.on('end', () => {
if (bytes > MAX_BODY_BYTES) return;
try {
const { plugin, operation, input } = JSON.parse(Buffer.concat(chunks).toString('utf8'));
const output = parse(plugin, operation, input);
response.writeHead(200, { 'content-type': 'application/json' });
response.end(JSON.stringify(output));
} catch (error) {
response.writeHead(400, { 'content-type': 'application/json' });
response.end(JSON.stringify({ error: error.message }));
}
});
});
}
if (require.main === module) {
createServer().listen(Number(process.env.SOURCE_PLUGIN_PORT || 8094), process.env.SOURCE_PLUGIN_HOST || '127.0.0.1');
}
module.exports = { createServer };
scraper/test/source-plugin.test.js
+67
View File
@@ -0,0 +1,67 @@
'use strict';
const test = require('node:test');
const assert = require('node:assert/strict');
const { createServer } = require('../plugins/worker');
const { parse } = require('../plugins/registry');
const { parseSource } = require('../plugins/client');
const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral');
const seriesUrl = 'https://weebcentral.com/series/ABC/title';
const chapterUrl = 'https://weebcentral.com/chapters/XYZ';
const seriesHtml = '<h1>Test Manga</h1><div hx-get="/series/ABC/full-chapter-list"></div>';
const listHtml = '<a href="/chapters/XYZ">Chapter 1</a><a href="/chapters/LMN">Log. 2</a>';
const chapterHtml = '<time datetime="2026-01-01"></time>';
const imagesHtml = '<img src="https://img.example/1.jpg"><img data-src="https://img.example/2.jpg">';
test('first-party registry rejects unknown plugins and malformed input', () => {
assert.throws(() => parse('remote-module', 'catalog', { html: listHtml, url: seriesUrl }), /Unknown/);
assert.throws(() => parse('weebcentral', 'catalog', { html: listHtml, url: 'file:///tmp/x' }), /Invalid source URL/);
assert.throws(() => parse('weebcentral', 'catalog', { html: listHtml, url: 'https://other.example/series/a' }), /does not belong/);
assert.equal(parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }).chapters.length, 2);
});
test('isolated parser preserves WeebCentral catalog and page discovery', async () => {
const server = createServer();
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const previous = process.env.SOURCE_PLUGIN_URL;
process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
const requested = [];
const http = { async get(url) {
requested.push(url);
const body = url === seriesUrl ? seriesHtml
: url.endsWith('/full-chapter-list') ? listHtml
: url === chapterUrl ? chapterHtml
: imagesHtml;
return { text: async () => body };
} };
try {
const catalog = await discoverWeebCatalog(seriesUrl, http);
assert.equal(catalog.title, 'Test Manga');
assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']);
const chapter = await discoverWeebChapter(catalog.chapters[0], http);
assert.equal(chapter.pages.length, 2);
assert.equal(chapter.added, '2026-01-01');
assert.equal(requested.length, 4);
const response = await fetch(`${process.env.SOURCE_PLUGIN_URL}/v1/parse`, {
method: 'POST', headers: { 'content-type': 'application/json' },
body: JSON.stringify({ plugin: 'unapproved', operation: 'catalog', input: { html: listHtml, url: seriesUrl } }),
});
assert.equal(response.status, 400);
} finally {
if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL;
else process.env.SOURCE_PLUGIN_URL = previous;
await new Promise(resolve => server.close(resolve));
}
});
test('configured worker failure does not fall back to local parsing', async () => {
const previous = process.env.SOURCE_PLUGIN_URL;
process.env.SOURCE_PLUGIN_URL = 'http://127.0.0.1:1';
try {
await assert.rejects(parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }));
} finally {
if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL;
else process.env.SOURCE_PLUGIN_URL = previous;
}
});