Maashkinoozhe/MangaCircus
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Checkpoint first-party source parser pilot

Browse Source
This commit is contained in:
Codex committed 2026-10-13 21:45:27 +02:00
1 parent 605fa5b9d8
commit c5a92282a9
9 files changed
+320 -43
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scraper/adapters/weebcentral.js
+13 -43
View File
@@ -2,40 +2,19 @@
const { SlowHttp } = require('../lib/http');
const { fallbackDirection } = require('../../reading-direction');
const parser = require('../plugins/weebcentral');
const { parseSource } = require('../plugins/client');
const decode = value => value.replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/<[^>]+>/g, '').trim();
function absolute(base, value) { return new URL(value, base).href; }
function parseChapters(html, baseUrl) {
const found = [];
const re = /<a\b[^>]*href="([^"]*\/chapters\/([A-Z0-9]+)[^"]*)"[^>]*>([\s\S]*?)<\/a>/gi;
for (const match of html.matchAll(re)) {
const text = decode(match[3]).replace(/\s+/g, ' ');
const season = Number(text.match(/^S(\d+)\s*-/i)?.[1] || 0) || null;
// A few WeebCentral catalogs use a series-specific label (for example
// "Rating 442", "Episode 128", "Night 182", "Log. 65", or "Act 60.6") in the same place where most catalogs
// say "Chapter 442". Treat those labels as chapter numbering instead of
// discarding the entire catalog.
const regular = text.match(/\b(?:Chapter|Rating|Episode|Night|Log|Act)\s*[.:]?\s*([0-9]+(?:\.[0-9]+)?)/i)?.[1];
const prologue = text.match(/Prologue(?:\s+([0-9]+))?/i)?.[1];
const number = regular || (prologue ? `0.${prologue.padStart(2, '0')}` : /Prologue/i.test(text) ? '0' : null);
if (number) found.push({
id: match[2], number, title: regular ? '' : `Prologue ${prologue || ''}`.trim(),
...(season ? { book: { number: season, title: `Season ${season}` } } : {}),
url: absolute(baseUrl, match[1]), raw: text,
});
}
return [...new Map(found.map(chapter => [chapter.id, chapter])).values()];
}
const parseChapters = (html, baseUrl) => parser.catalog({ html, url: baseUrl }).chapters;
const parsePageUrls = (html, baseUrl) => parser.pages({ html, url: baseUrl }).pages;
async function discoverWeebCatalog(seriesUrl, http) {
const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text();
const title = decode(seriesHtml.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || 'Untitled');
const listUrl = seriesHtml.match(/hx-get="([^"]+\/full-chapter-list)"/i)?.[1];
const listHtml = listUrl ? await (await http.get(absolute(seriesUrl, listUrl), { accept: 'text/html' })).text() : seriesHtml;
const chapterLinks = [...listHtml.matchAll(/<a\b[^>]*href="[^"]*\/chapters\/[A-Z0-9]+[^"]*"/gi)].length;
const chapters = parseChapters(listHtml, seriesUrl).sort((a, b) =>
const { title, listUrl } = await parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl });
const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html' })).text() : seriesHtml;
const parsed = await parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl });
const chapterLinks = parsed.chapterLinks;
const chapters = parsed.chapters.sort((a, b) =>
Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number));
return { title, chapters, chapterLinks,
...(chapterLinks && !chapters.length ? { availability: { reason: 'unrecognized-labels',
@@ -45,28 +24,19 @@ async function discoverWeebCatalog(seriesUrl, http) {
async function discoverWeebChapter(chapter, http) {
const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text();
const manifestUrl = pageHtml.match(/['"](https?:\/\/weebcentral\.com\/chapters\/[^'"]+\/images\?is_prev=[^'"]+)['"]/i)?.[1]
|| `${chapter.url}/images?is_prev=False`;
const { manifestUrl, added } = await parseSource('weebcentral', 'chapter', { html: pageHtml, url: chapter.url });
const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip&current_page=1`, { accept: 'text/html' })).text();
const pages = parsePageUrls(fragment, manifestUrl).map(url => ({ url, headers: { referer: chapter.url } }));
const parsed = await parseSource('weebcentral', 'pages', { html: fragment, url: manifestUrl });
const pages = parsed.pages.map(url => ({ url, headers: { referer: chapter.url } }));
if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`);
return {
number: chapter.number, title: chapter.title || '',
...(chapter.book ? { book: chapter.book } : {}),
added: pageHtml.match(/datetime="([^"]+)"/i)?.[1],
added,
source: { name: 'WeebCentral', url: chapter.url }, pages,
};
}
function parsePageUrls(html, baseUrl) {
const urls = [];
for (const match of html.matchAll(/<img\b[^>]*(?:src|data-src)="([^"]+)"/gi)) {
const url = absolute(baseUrl, match[1].replace(/&amp;/g, '&'));
if (!url.startsWith('data:')) urls.push(url);
}
return [...new Set(urls)];
}
async function discoverWeebCentral(seriesUrl, options = {}) {
const http = new SlowHttp(options.http || {});
const catalog = await discoverWeebCatalog(seriesUrl, http);