Keep source plugins in single scraper process
This commit is contained in:
1 parent
0fc1a7b52a
commit
6d713e0a8c
19 files changed
+202
-467
No files matched your search
@@ -3,16 +3,26 @@
|
||||
const { SlowHttp } = require('../lib/http');
|
||||
const { fallbackDirection } = require('../../reading-direction');
|
||||
const parser = require('../plugins/weebcentral');
|
||||
const { parseSource } = require('../plugins/client');
|
||||
const { parse } = require('../plugins/registry');
|
||||
|
||||
function weebCentralInfo(value) {
|
||||
let url;
|
||||
try { url = new URL(String(value)); } catch { return null; }
|
||||
if (url.protocol !== 'https:' || url.hostname !== 'weebcentral.com') return null;
|
||||
const match = /^\/series\/([A-Z0-9]+)(?:\/([^/?#]+))?\/?$/i.exec(url.pathname);
|
||||
if (!match) return null;
|
||||
const title = decodeURIComponent(match[2] || match[1]).replace(/[-_]+/g, ' ').trim();
|
||||
return { source: 'weebcentral', id: match[1].toUpperCase(), title, url: `https://weebcentral.com/series/${match[1]}${match[2] ? `/${match[2]}` : ''}` };
|
||||
}
|
||||
|
||||
const parseChapters = (html, baseUrl) => parser.catalog({ html, url: baseUrl }).chapters;
|
||||
const parsePageUrls = (html, baseUrl) => parser.pages({ html, url: baseUrl }).pages;
|
||||
|
||||
async function discoverWeebCatalog(seriesUrl, http) {
|
||||
const seriesHtml = await (await http.get(seriesUrl, { accept: 'text/html' })).text();
|
||||
const { title, listUrl } = await parseSource('weebcentral', 'series', { html: seriesHtml, url: seriesUrl });
|
||||
const { title, listUrl } = parse('weebcentral', 'series', { html: seriesHtml, url: seriesUrl });
|
||||
const listHtml = listUrl ? await (await http.get(listUrl, { accept: 'text/html' })).text() : seriesHtml;
|
||||
const parsed = await parseSource('weebcentral', 'catalog', { html: listHtml, url: seriesUrl });
|
||||
const parsed = parse('weebcentral', 'catalog', { html: listHtml, url: seriesUrl });
|
||||
const chapterLinks = parsed.chapterLinks;
|
||||
const chapters = parsed.chapters.sort((a, b) =>
|
||||
Number(a.book?.number || 1) - Number(b.book?.number || 1) || Number(a.number) - Number(b.number));
|
||||
@@ -24,9 +34,9 @@ async function discoverWeebCatalog(seriesUrl, http) {
|
||||
|
||||
async function discoverWeebChapter(chapter, http) {
|
||||
const pageHtml = await (await http.get(chapter.url, { accept: 'text/html' })).text();
|
||||
const { manifestUrl, added } = await parseSource('weebcentral', 'chapter', { html: pageHtml, url: chapter.url });
|
||||
const { manifestUrl, added } = parse('weebcentral', 'chapter', { html: pageHtml, url: chapter.url });
|
||||
const fragment = await (await http.get(`${manifestUrl}&reading_style=long_strip¤t_page=1`, { accept: 'text/html' })).text();
|
||||
const parsed = await parseSource('weebcentral', 'pages', { html: fragment, url: manifestUrl });
|
||||
const parsed = parse('weebcentral', 'pages', { html: fragment, url: manifestUrl });
|
||||
const pages = parsed.pages.map(url => ({ url, headers: { referer: chapter.url } }));
|
||||
if (!pages.length) throw new Error(`No pages found for chapter ${chapter.number}`);
|
||||
return {
|
||||
@@ -55,4 +65,4 @@ async function discoverWeebCentral(seriesUrl, options = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { discoverWeebCentral, discoverWeebCatalog, discoverWeebChapter, parseChapters, parsePageUrls };
|
||||
module.exports = { weebCentralInfo, discoverWeebCentral, discoverWeebCatalog, discoverWeebChapter, parseChapters, parsePageUrls };
|
||||
Reference in new issue
Block a user