Maashkinoozhe/MangaCircus
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep source plugins in single scraper process

Browse Source
This commit is contained in:
Codex committed 2026-10-13 22:42:20 +02:00
1 parent 0fc1a7b52a
commit 6d713e0a8c
19 files changed
+202 -467
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scraper/test/source-mangadex-plugin.test.js
+9 -24
View File
@@ -2,8 +2,8 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { createServer } = require('../plugins/worker');
const { discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex');
const { parse } = require('../plugins/registry');
const mangaId = '123e4567-e89b-42d3-a456-426614174000';
const chapterIds = [
@@ -29,27 +29,12 @@ function fakeHttp() {
} };
}
test('MangaDex isolated parser matches the existing catalog and page behavior', async () => {
const server = createServer('mangadex');
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const previous = process.env.MANGADEX_PLUGIN_URL;
delete process.env.MANGADEX_PLUGIN_URL;
try {
const baseline = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
const baselineChapter = await discoverMangaDexChapter(baseline.chapters[0], fakeHttp());
process.env.MANGADEX_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
const isolated = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
const isolatedChapter = await discoverMangaDexChapter(isolated.chapters[0], fakeHttp());
assert.deepEqual(isolated, baseline);
assert.deepEqual(isolatedChapter, baselineChapter);
const rejected = await fetch(`${process.env.MANGADEX_PLUGIN_URL}/v1/parse`, {
method: 'POST', headers: { 'content-type': 'application/json' },
body: JSON.stringify({ plugin: 'weebcentral', operation: 'series', input: { html: '<h1>X</h1>', url: 'https://weebcentral.com/series/X' } }),
});
assert.equal(rejected.status, 400);
} finally {
if (previous === undefined) delete process.env.MANGADEX_PLUGIN_URL;
else process.env.MANGADEX_PLUGIN_URL = previous;
await new Promise(resolve => server.close(resolve));
}
test('MangaDex parses a catalog and pages in the scraper process', async () => {
const catalog = await discoverMangaDexCatalog(seriesUrl, fakeHttp());
const chapter = await discoverMangaDexChapter(catalog.chapters[0], fakeHttp());
assert.equal(catalog.title, 'Fixture');
assert.deepEqual(catalog.chapters.map(ref => ref.number), ['1', '2']);
assert.equal(chapter.pages.length, 2);
assert.match(chapter.pages[0].url, /^https:\/\/uploads\.mangadex\.org\/data\//);
assert.throws(() => parse('mangadex', 'metadata', { data: manga.data, url: 'https://other.example/manga/x' }), /does not belong/);
});
scraper/test/source-plugin.test.js
+17 -51
View File
@@ -2,12 +2,10 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const httpModule = require('node:http');
const { createServer } = require('../plugins/worker');
const { parse } = require('../plugins/registry');
const { parseSource } = require('../plugins/client');
const { validate } = require('../plugins/validation');
const { discoverWeebCatalog, discoverWeebChapter } = require('../adapters/weebcentral');
const { adapters, adapterFor } = require('../adapters');
const seriesUrl = 'https://weebcentral.com/series/ABC/title';
const chapterUrl = 'https://weebcentral.com/chapters/XYZ';
@@ -32,11 +30,7 @@ test('parser output cannot turn catalog and image links into local or unrelated
assert.doesNotThrow(() => validate('weebcentral', 'pages', { pages: ['https://img.example/1.jpg'] }));
});
test('isolated parser preserves WeebCentral catalog and page discovery', async () => {
const server = createServer();
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const previous = process.env.SOURCE_PLUGIN_URL;
process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
test('in-process parser preserves WeebCentral catalog and page discovery', async () => {
const requested = [];
const http = { async get(url) {
requested.push(url);
@@ -46,50 +40,22 @@ test('isolated parser preserves WeebCentral catalog and page discovery', async (
: imagesHtml;
return { text: async () => body };
} };
try {
const catalog = await discoverWeebCatalog(seriesUrl, http);
assert.equal(catalog.title, 'Test Manga');
assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']);
const chapter = await discoverWeebChapter(catalog.chapters[0], http);
assert.equal(chapter.pages.length, 2);
assert.equal(chapter.added, '2026-01-01');
assert.equal(requested.length, 4);
const response = await fetch(`${process.env.SOURCE_PLUGIN_URL}/v1/parse`, {
method: 'POST', headers: { 'content-type': 'application/json' },
body: JSON.stringify({ plugin: 'unapproved', operation: 'catalog', input: { html: listHtml, url: seriesUrl } }),
});
assert.equal(response.status, 400);
} finally {
if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL;
else process.env.SOURCE_PLUGIN_URL = previous;
await new Promise(resolve => server.close(resolve));
}
const catalog = await discoverWeebCatalog(seriesUrl, http);
assert.equal(catalog.title, 'Test Manga');
assert.deepEqual(catalog.chapters.map(chapter => chapter.number), ['1', '2']);
const chapter = await discoverWeebChapter(catalog.chapters[0], http);
assert.equal(chapter.pages.length, 2);
assert.equal(chapter.added, '2026-01-01');
assert.equal(requested.length, 4);
});
test('configured worker failure does not fall back to local parsing', async () => {
const previous = process.env.SOURCE_PLUGIN_URL;
process.env.SOURCE_PLUGIN_URL = 'http://127.0.0.1:1';
try {
await assert.rejects(parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl }));
} finally {
if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL;
else process.env.SOURCE_PLUGIN_URL = previous;
}
});
test('scraper rejects unsafe responses from a configured worker', async () => {
const server = httpModule.createServer((_request, response) => {
response.writeHead(200, { 'content-type': 'application/json' });
response.end(JSON.stringify({ title: 'X', listUrl: 'http://127.0.0.1:8091/api/state' }));
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const previous = process.env.SOURCE_PLUGIN_URL;
process.env.SOURCE_PLUGIN_URL = `http://127.0.0.1:${server.address().port}`;
try {
await assert.rejects(parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl }), /unexpected/);
} finally {
if (previous === undefined) delete process.env.SOURCE_PLUGIN_URL;
else process.env.SOURCE_PLUGIN_URL = previous;
await new Promise(resolve => server.close(resolve));
test('a new source is an explicit adapter, not another container or remote module', () => {
assert.deepEqual(adapters.map(adapter => adapter.id), ['weebcentral', 'mangadex']);
for (const adapter of adapters) {
assert.equal(typeof adapter.match, 'function');
assert.equal(typeof adapter.discover, 'function');
assert.equal(typeof adapter.run, 'function');
}
assert.equal(adapterFor(seriesUrl)?.adapter.id, 'weebcentral');
assert.equal(adapterFor('https://unknown.example/series/ABC'), null);
});