Keep source plugins in single scraper process
This commit is contained in:
Codex committed
1 parent
0fc1a7b52a
commit
6d713e0a8c
19 files changed
+202
-467
No files matched your search
+15
-112
@@ -1,8 +1,8 @@
|
||||
'use strict';
|
||||
|
||||
const API = 'https://api.mangadex.org';
|
||||
const { specialKey } = require('../../chapter-identity');
|
||||
const { parseSource } = require('../plugins/client');
|
||||
const { parse } = require('../plugins/registry');
|
||||
const { localizedTitle, chapterNumber, selectChapterReleases, availabilitySummary, pageUrls } = require('../plugins/mangadex');
|
||||
|
||||
function mangaDexInfo(value) {
|
||||
let url;
|
||||
@@ -15,78 +15,6 @@ function mangaDexInfo(value) {
|
||||
return { source: 'mangadex', id, title, url: `https://mangadex.org/title/${id}${match[2] ? `/${match[2]}` : ''}` };
|
||||
}
|
||||
|
||||
function localizedTitle(attributes = {}) {
|
||||
const titles = attributes.title || {};
|
||||
return titles.en || titles['ja-ro'] || titles.ja || Object.values(titles).find(Boolean) || 'Untitled';
|
||||
}
|
||||
|
||||
function chapterNumber(value) {
|
||||
const raw = String(value ?? '').trim();
|
||||
if (!/^\d+(?:\.\d+)?$/.test(raw)) return null;
|
||||
const [whole, decimal = ''] = raw.split('.');
|
||||
const normalizedWhole = whole.replace(/^0+(?=\d)/, '') || '0';
|
||||
const normalizedDecimal = decimal.replace(/0+$/, '');
|
||||
return normalizedDecimal ? `${normalizedWhole}.${normalizedDecimal}` : normalizedWhole;
|
||||
}
|
||||
|
||||
function groupFor(resource) {
|
||||
const relation = (resource.relationships || []).find(item => item.type === 'scanlation_group');
|
||||
return relation ? { id: relation.id, name: relation.attributes?.name || 'Unknown group' } : null;
|
||||
}
|
||||
|
||||
function releaseRank(resource) {
|
||||
const group = groupFor(resource)?.name || '';
|
||||
const credited = /official|license/i.test(group) ? 0 : group ? 1 : 2;
|
||||
const date = Date.parse(resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt || '') || Number.MAX_SAFE_INTEGER;
|
||||
return [credited, date, resource.id];
|
||||
}
|
||||
|
||||
function compareRank(left, right) {
|
||||
const a = releaseRank(left), b = releaseRank(right);
|
||||
return a[0] - b[0] || a[1] - b[1] || String(a[2]).localeCompare(String(b[2]));
|
||||
}
|
||||
|
||||
function selectChapterReleases(resources = [], specialLabel = 'Special') {
|
||||
const selected = new Map();
|
||||
for (const resource of resources) {
|
||||
if (resource.attributes?.externalUrl || Number(resource.attributes?.pages || 0) < 1) continue;
|
||||
const number = chapterNumber(resource.attributes?.chapter) || specialKey(resource.id);
|
||||
const prior = selected.get(number);
|
||||
if (!prior || compareRank(resource, prior) < 0) selected.set(number, resource);
|
||||
}
|
||||
return [...selected.entries()].map(([number, resource]) => {
|
||||
const group = groupFor(resource);
|
||||
return {
|
||||
id: resource.id, number, title: resource.attributes?.title || '',
|
||||
...(!chapterNumber(resource.attributes?.chapter) ? { special: true, label: specialLabel, sourceId: resource.id, sourceNumber: resource.attributes?.chapter ?? null } : {}),
|
||||
...(resource.attributes?.volume ? { volume: chapterNumber(resource.attributes.volume) || String(resource.attributes.volume) } : {}),
|
||||
pages: Number(resource.attributes?.pages || 0),
|
||||
added: resource.attributes?.publishAt || resource.attributes?.readableAt || resource.attributes?.createdAt,
|
||||
group,
|
||||
url: `https://mangadex.org/chapter/${resource.id}`,
|
||||
};
|
||||
}).sort((a, b) => Number(!!a.special) - Number(!!b.special) || (a.special ? a.number.localeCompare(b.number) : Number(a.number) - Number(b.number) || a.number.localeCompare(b.number)));
|
||||
}
|
||||
|
||||
function availabilitySummary(attributes, releases, chapters, externalReleases = null) {
|
||||
const numbered = chapters.filter(ref => !ref.special);
|
||||
const expected = chapterNumber(attributes.lastChapter);
|
||||
const maximum = numbered.length ? Math.max(...numbered.map(ref => Number(ref.number))) : null;
|
||||
// Reaching the final chapter number is not full coverage when the catalog
|
||||
// contains fewer distinct numbered releases than the expected chapter span.
|
||||
const expectedCount = expected && /^\d+$/.test(expected) ? Number(expected) : null;
|
||||
const sparse = expectedCount && numbered.length < expectedCount;
|
||||
const reason = chapters.length ? null : externalReleases > 0 ? 'external-only'
|
||||
: Array.isArray(attributes.availableTranslatedLanguages) && !attributes.availableTranslatedLanguages.includes('en') ? 'no-english' : 'no-hosted-pages';
|
||||
const message = reason === 'external-only' ? `MangaDex lists ${externalReleases} English external-link releases, but no hosted English pages. External links are not downloaded.`
|
||||
: reason === 'no-english' ? 'This MangaDex edition has no English releases listed.'
|
||||
: reason ? 'This MangaDex edition has no hosted English releases with pages.' : null;
|
||||
return { language: 'en', hostedReleases: releases.filter(r => !r.attributes?.externalUrl && Number(r.attributes?.pages) > 0).length,
|
||||
selectedChapters: chapters.length, numberedChapters: numbered.length, specialReleases: chapters.length - numbered.length,
|
||||
externalReleases, expectedLastChapter: expected, highestHostedChapter: maximum,
|
||||
coverage: expected && (maximum === null || maximum < Number(expected) || sparse) ? 'partial' : 'unknown', reason, message };
|
||||
}
|
||||
|
||||
async function jsonGet(http, url) {
|
||||
const response = await http.get(url, { accept: 'application/json' });
|
||||
const body = await response.json();
|
||||
@@ -99,10 +27,7 @@ async function discoverMangaDexCatalog(seriesUrl, http) {
|
||||
if (!info) throw new Error('Invalid MangaDex title URL');
|
||||
const manga = (await jsonGet(http, `${API}/manga/${info.id}`)).data;
|
||||
if (!manga || manga.type !== 'manga') throw new Error('MangaDex title was not found');
|
||||
const isolated = !!process.env.MANGADEX_PLUGIN_URL;
|
||||
const pluginMetadata = isolated
|
||||
? await parseSource('mangadex', 'metadata', { html: JSON.stringify(manga), url: `${API}/manga/${info.id}` })
|
||||
: null;
|
||||
const metadata = parse('mangadex', 'metadata', { data: manga, url: `${API}/manga/${info.id}` });
|
||||
|
||||
const releases = [];
|
||||
let offset = 0;
|
||||
@@ -112,67 +37,45 @@ async function discoverMangaDexCatalog(seriesUrl, http) {
|
||||
params.append('includes[]', 'scanlation_group');
|
||||
params.append('order[chapter]', 'asc');
|
||||
const page = await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`);
|
||||
if (isolated) {
|
||||
const parsed = await parseSource('mangadex', 'feed', { html: JSON.stringify(page.data || []), url: `${API}/manga/${info.id}/feed` });
|
||||
releases.push(...parsed.releases);
|
||||
} else releases.push(...(page.data || []));
|
||||
releases.push(...parse('mangadex', 'feed', { data: page.data || [], url: `${API}/manga/${info.id}/feed` }).releases);
|
||||
offset += page.data?.length || 0;
|
||||
if (!page.data?.length || offset >= Number(page.total || 0)) break;
|
||||
}
|
||||
|
||||
const title = isolated ? pluginMetadata.title : localizedTitle(manga.attributes);
|
||||
const specialLabel = isolated ? pluginMetadata.specialLabel : /pre.serialization|pilot/i.test(title) ? 'Pilot'
|
||||
: (manga.attributes?.tags || []).some(tag => tag.attributes?.name?.en === 'Oneshot') ? 'One-shot' : 'Special';
|
||||
const pluginCatalog = async externalReleases => parseSource('mangadex', 'catalog', {
|
||||
html: JSON.stringify({ attributes: pluginMetadata.availabilityAttributes, releases, specialLabel, externalReleases }),
|
||||
const { title, specialLabel } = metadata;
|
||||
const parseCatalog = externalReleases => parse('mangadex', 'catalog', {
|
||||
data: { attributes: metadata.availabilityAttributes, releases, specialLabel, externalReleases },
|
||||
url: `${API}/manga/${info.id}`,
|
||||
});
|
||||
let parsed = isolated ? await pluginCatalog(null) : null;
|
||||
const chapters = isolated ? parsed.chapters : selectChapterReleases(releases, specialLabel);
|
||||
let parsed = parseCatalog(null);
|
||||
const chapters = parsed.chapters;
|
||||
// External-only is a separate feed mode, not a combined hosted+external catalog.
|
||||
let externalReleases = null;
|
||||
if (!chapters.length) {
|
||||
const params = new URLSearchParams({ limit: '1', includeFutureUpdates: '0', includeExternalUrl: '1' });
|
||||
params.append('translatedLanguage[]', 'en');
|
||||
externalReleases = Number((await jsonGet(http, `${API}/manga/${info.id}/feed?${params}`)).total || 0);
|
||||
if (isolated) parsed = await pluginCatalog(externalReleases);
|
||||
parsed = parseCatalog(externalReleases);
|
||||
}
|
||||
return {
|
||||
id: info.id,
|
||||
title,
|
||||
originalLanguage: isolated ? pluginMetadata.originalLanguage : manga.attributes?.originalLanguage || null,
|
||||
links: isolated ? pluginMetadata.links : manga.attributes?.links || {},
|
||||
availability: isolated ? parsed.availability : availabilitySummary(manga.attributes || {}, releases, chapters, externalReleases),
|
||||
originalLanguage: metadata.originalLanguage,
|
||||
links: metadata.links,
|
||||
availability: parsed.availability,
|
||||
chapters,
|
||||
};
|
||||
}
|
||||
|
||||
function pageUrls(atHome, chapterUrl, expectedPages) {
|
||||
const base = new URL(atHome.baseUrl);
|
||||
if (base.protocol !== 'https:' || !(base.hostname === 'uploads.mangadex.org' || base.hostname.endsWith('.mangadex.network'))) {
|
||||
throw new Error(`Unexpected MangaDex@Home host: ${base.hostname}`);
|
||||
}
|
||||
const hash = String(atHome.chapter?.hash || '');
|
||||
const files = atHome.chapter?.data;
|
||||
if (!hash || !Array.isArray(files) || !files.length) throw new Error('MangaDex@Home returned no pages');
|
||||
if (expectedPages && files.length !== expectedPages) throw new Error(`MangaDex page count changed: expected ${expectedPages}, received ${files.length}`);
|
||||
return files.map(file => ({
|
||||
url: new URL(`/data/${encodeURIComponent(hash)}/${encodeURIComponent(file)}`, base).href,
|
||||
headers: { referer: chapterUrl },
|
||||
}));
|
||||
}
|
||||
|
||||
async function discoverMangaDexChapter(ref, http) {
|
||||
const atHome = await jsonGet(http, `${API}/at-home/server/${ref.id}?forcePort443=true`);
|
||||
const groupName = ref.group?.name ? ` · ${ref.group.name}` : '';
|
||||
const parsed = process.env.MANGADEX_PLUGIN_URL
|
||||
? await parseSource('mangadex', 'atHome', { html: JSON.stringify({ response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }), url: `${API}/at-home/server/${ref.id}` })
|
||||
: null;
|
||||
const parsed = parse('mangadex', 'atHome', { data: { response: atHome, chapterUrl: ref.url, expectedPages: ref.pages }, url: `${API}/at-home/server/${ref.id}` });
|
||||
return {
|
||||
number: ref.number, title: ref.title || '', added: ref.added,
|
||||
...(ref.special ? { special: true, label: ref.label, sourceId: ref.id, sourceNumber: ref.sourceNumber ?? null } : {}),
|
||||
source: { name: `MangaDex${groupName}`, url: ref.url },
|
||||
pages: parsed ? parsed.pages : pageUrls(atHome, ref.url, ref.pages),
|
||||
pages: parsed.pages,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user