576 lines
36 KiB
JavaScript
576 lines
36 KiB
JavaScript
'use strict';
|
|
|
|
const test = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const fs = require('node:fs/promises');
|
|
const os = require('node:os');
|
|
const path = require('node:path');
|
|
const { cleanPart, chapterKey, chapterFolder, assertInside } = require('../lib/names');
|
|
const { imageExtension } = require('../lib/images');
|
|
const { publishManifest } = require('../lib/publisher');
|
|
const { mergeImportedMetadata } = require('../lib/metadata-merge');
|
|
const { metadataFromMedia } = require('../metadata/anilist');
|
|
const { normalizeScope, selectByScope, scopeLabel } = require('../lib/scope');
|
|
const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, setAutoUpdate } = require('../lib/library-inventory');
|
|
const { inspectLegacyWork } = require('../lib/legacy-publication');
|
|
const { discoverWeebCatalog, parseChapters, parsePageUrls } = require('../adapters/weebcentral');
|
|
const { chapterNumber, selectChapterReleases, pageUrls } = require('../adapters/mangadex');
|
|
const { adapterFor } = require('../adapters');
|
|
const { ScraperService, validateSettings, settingsForSource } = require('../service');
|
|
|
|
test('MangaCircus chapter keys are normalized', () => {
|
|
assert.equal(chapterKey('12'), 'c0012');
|
|
assert.equal(chapterKey('12.50'), 'c0012.5');
|
|
assert.equal(chapterKey('x2'), 'x02');
|
|
assert.throws(() => chapterKey('special'));
|
|
});
|
|
|
|
test('folder names are portable and versioned', () => {
|
|
assert.equal(cleanPart('Bad:<Title>. '), 'BadTitle');
|
|
assert.equal(cleanPart('CON'), 'CON-manga');
|
|
assert.equal(chapterFolder({ number: '7', title: 'A [Test]', source: { name: 'Host' } }), 'c0007 A Test [Host]');
|
|
assert.throws(() => assertInside('C:/library', 'C:/outside'));
|
|
});
|
|
|
|
test('image signatures are recognized', () => {
|
|
assert.equal(imageExtension(Buffer.from([0xff, 0xd8, 0xff, 0x00])), 'jpg');
|
|
assert.equal(imageExtension(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])), 'png');
|
|
assert.equal(imageExtension(Buffer.from('not an image')), null);
|
|
});
|
|
|
|
test('per-source rate settings default fast and update independently', () => {
|
|
const defaults = validateSettings({});
|
|
const fast = { minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000 };
|
|
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast);
|
|
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), fast);
|
|
|
|
const changed = validateSettings({ sourceRates: { mangadex: {
|
|
minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000,
|
|
} } }, defaults);
|
|
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), {
|
|
minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000,
|
|
});
|
|
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast);
|
|
assert.throws(() => validateSettings({ sourceRates: { mangadex: { minMs: 5000, maxMs: 1000 } } }, defaults));
|
|
});
|
|
|
|
test('AniList metadata keeps manga format and original story credits', () => {
|
|
const metadata = metadataFromMedia({
|
|
id: 150193, idMal: null, format: 'MANGA',
|
|
title: { english: 'Heavy Knight', romaji: 'Juukishi', native: '重騎士' }, synonyms: [],
|
|
staff: { edges: [
|
|
{ role: 'Original Story', node: { name: { full: 'Nekoko' } } },
|
|
{ role: 'Story & Art', node: { name: { full: 'Lee Brocco' } } },
|
|
{ role: 'Character Design', node: { name: { full: 'Jaian' } } },
|
|
] },
|
|
status: 'RELEASING', startDate: { year: 2022 }, genres: ['Fantasy'], description: '',
|
|
siteUrl: 'https://anilist.co/manga/150193', coverImage: { extraLarge: 'https://s4.anilist.co/cover.jpg' },
|
|
});
|
|
assert.deepEqual(metadata.authors, ['Nekoko', 'Lee Brocco']);
|
|
assert.equal(metadata['x-anilist'].format, 'MANGA');
|
|
});
|
|
|
|
test('metadata refresh preserves legacy and manual fields while updating untouched imports', () => {
|
|
const first = mergeImportedMetadata({ title: 'My preferred title', status: 'ongoing' }, {
|
|
title: 'Provider title', status: 'completed', tags: ['Fantasy'], 'x-anilist': { id: 42 },
|
|
}, 'Fallback');
|
|
assert.equal(first.title, 'My preferred title');
|
|
assert.equal(first.status, 'ongoing');
|
|
assert.deepEqual(first.tags, ['Fantasy']);
|
|
assert.equal(first['x-metadata'].imported.title, 'Provider title');
|
|
const second = mergeImportedMetadata({ ...first, tags: ['Fantasy', 'Favorite'] }, {
|
|
title: 'New provider title', status: 'ongoing', tags: ['Action'], 'x-anilist': { id: 42 },
|
|
}, 'Fallback');
|
|
assert.equal(second.title, 'My preferred title');
|
|
assert.equal(second.status, 'ongoing');
|
|
assert.deepEqual(second.tags, ['Fantasy', 'Favorite']);
|
|
const third = mergeImportedMetadata(second, { description: 'First', 'x-anilist': { id: 42 } }, 'Fallback');
|
|
assert.equal(mergeImportedMetadata(third, { description: 'Revised', 'x-anilist': { id: 42 } }, 'Fallback').description, 'Revised');
|
|
assert.throws(() => mergeImportedMetadata(first, { 'x-anilist': { id: 99 } }, 'Fallback'), /identity conflict/);
|
|
assert.throws(() => mergeImportedMetadata({ 'x-work-identity': { scheme: 'mangacircus-work-v1', kind: 'origin', value: 'a'.repeat(64) } },
|
|
{ 'x-work-identity': { scheme: 'mangacircus-work-v1', kind: 'origin', value: 'b'.repeat(64) } }, 'Fallback'), /work identity conflict/);
|
|
});
|
|
|
|
test('publisher rejects a conflicting work ID before fetching any pages', async t => {
|
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-identity-'));
|
|
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
|
const manga = path.join(root, 'Existing');
|
|
await fs.mkdir(manga);
|
|
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Existing',
|
|
sources: [{ name: 'Host', url: 'https://example.test/series' }], 'x-anilist': { id: 42 } }));
|
|
let requested = 0;
|
|
await assert.rejects(publishManifest({ version: 1,
|
|
series: { title: 'Existing', direction: 'rtl', metadata: { 'x-anilist': { id: 99 } },
|
|
sources: [{ name: 'Host', url: 'https://example.test/series' }] },
|
|
chapters: [{ number: '1', source: { name: 'Host', url: 'https://example.test/chapter' },
|
|
pages: [{ url: 'https://example.test/page.png' }] }],
|
|
}, { libraryRoot: root, httpClient: { async get() { requested++; throw new Error('unexpected request'); } } }), /identity conflict/);
|
|
assert.equal(requested, 0);
|
|
assert.equal((await fs.readdir(manga)).includes('v01'), false);
|
|
});
|
|
|
|
test('publisher explains an unresolved same-title folder collision without touching its contents', async t => {
|
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-title-collision-'));
|
|
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
|
const manga = path.join(root, 'Same Title');
|
|
await fs.mkdir(manga);
|
|
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Same Title' }));
|
|
await fs.writeFile(path.join(manga, 'operator-note.txt'), 'keep');
|
|
let requested = 0;
|
|
await assert.rejects(publishManifest({ version: 1,
|
|
series: { title: 'Same Title', direction: 'rtl', sources: [{ name: 'Peer', url: 'peer://p123/m123' }] },
|
|
chapters: [{ number: '1', source: { name: 'Peer', url: 'peer://p123/m123' }, pages: [{ url: 'https://example.test/page.png' }] }],
|
|
}, { libraryRoot: root, httpClient: { async get() { requested++; throw new Error('unexpected request'); } } }),
|
|
/could not be safely matched.*No files were overwritten.*Choose the existing manga/s);
|
|
assert.equal(requested, 0);
|
|
assert.equal(await fs.readFile(path.join(manga, 'operator-note.txt'), 'utf8'), 'keep');
|
|
});
|
|
|
|
test('queue scopes select exact catalog slices', () => {
|
|
const chapters = [
|
|
{ id: 'a', number: '1', volume: '1' }, { id: 'b', number: '2', volume: '1' },
|
|
{ id: 'c', number: '3', volume: '2' }, { id: 'd', number: '4', volume: '2' },
|
|
];
|
|
assert.deepEqual(selectByScope(chapters, { mode: 'latest' }).map(x => x.id), ['d']);
|
|
assert.deepEqual(selectByScope(chapters, { mode: 'last', count: 2 }).map(x => x.id), ['c', 'd']);
|
|
assert.deepEqual(selectByScope(chapters, { mode: 'first', count: 2 }).map(x => x.id), ['a', 'b']);
|
|
assert.deepEqual(selectByScope(chapters, { mode: 'chapter', chapter: '003.0' }).map(x => x.id), ['c']);
|
|
assert.deepEqual(selectByScope(chapters, { mode: 'after', chapter: '2' }).map(x => x.id), ['c', 'd']);
|
|
assert.deepEqual(selectByScope(chapters, { mode: 'volume', volume: '2' }).map(x => x.id), ['c', 'd']);
|
|
assert.deepEqual(selectByScope(chapters, { mode: 'ids', ids: ['b', 'd'] }).map(x => x.id), ['b', 'd']);
|
|
assert.equal(scopeLabel(normalizeScope({ mode: 'last', count: 5 })), 'last 5 chapters');
|
|
});
|
|
|
|
test('library inventory distinguishes missing chapters from alternate source versions', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-inventory-'));
|
|
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
|
const manga = path.join(temp, 'Library Manga'), book = path.join(manga, 'v01');
|
|
await fs.mkdir(path.join(book, 'c0001 [WeebCentral]'), { recursive: true });
|
|
await fs.mkdir(path.join(book, 'c0002 [MangaDex]'), { recursive: true });
|
|
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({
|
|
title: 'Library Manga', status: 'ongoing',
|
|
sources: [{ name: 'WeebCentral', url: 'https://weebcentral.com/series/SERIES/Test' }],
|
|
'x-scraper': { ids: { weebcentral: 'SERIES', anilist: 42 } },
|
|
}));
|
|
await fs.writeFile(path.join(book, 'c0001 [WeebCentral]', 'chapter.json'), JSON.stringify({ source: { name: 'WeebCentral', url: 'https://weebcentral.com/chapters/A' } }));
|
|
await fs.writeFile(path.join(book, 'c0002 [MangaDex]', 'chapter.json'), JSON.stringify({ source: { name: 'MangaDex · Group', url: 'https://mangadex.org/chapter/B' } }));
|
|
const inventory = await scanLibrary(temp);
|
|
const found = findManga(inventory, { source: 'weebcentral', seriesId: 'SERIES' });
|
|
assert.equal(found.id, 'Library Manga');
|
|
assert.equal(findManga(inventory, { source: 'mangadex', seriesId: 'other', anilist: 42 }).id, 'Library Manga');
|
|
assert.equal(found.autoUpdate, true);
|
|
assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'skip-any'), false);
|
|
assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'add-source'), true);
|
|
assert.equal(shouldDownload(found, { number: '1' }, 'weebcentral', 'add-source'), false);
|
|
assert.equal(shouldDownload(found, { number: '3' }, 'weebcentral', 'skip-any'), true);
|
|
assert.equal(isNewerThanLibrary(found, { number: '1' }), false);
|
|
assert.equal(isNewerThanLibrary(found, { number: '3' }), true);
|
|
await setAutoUpdate(temp, 'Library Manga', false);
|
|
assert.equal((await scanLibrary(temp)).mangas[0].autoUpdate, false);
|
|
});
|
|
|
|
test('a shared work ID cannot merge distinct source listings or ambiguous editions', () => {
|
|
const original = { id: 'Main', ids: { weebcentral: 'MAIN', anilist: 42 }, sources: [] };
|
|
assert.equal(findManga({ mangas: [original] }, { source: 'weebcentral', seriesId: 'PILOT', anilist: 42 }), null);
|
|
assert.equal(findManga({ mangas: [original] }, { source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), original);
|
|
assert.equal(findManga({ mangas: [original, { id: 'Pilot', ids: { anilist: 42 }, sources: [] }] },
|
|
{ source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), null);
|
|
});
|
|
|
|
test('recent events read the bounded log tail without losing the newest incident', async t => {
|
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-events-'));
|
|
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
|
const service = new ScraperService({ dataRoot: root });
|
|
await fs.writeFile(service.logFile, `${'x'.repeat(600 * 1024)}\n${JSON.stringify({ level: 'error', message: 'recent' })}\n`);
|
|
assert.deepEqual(await service.recentEvents(1), [{ level: 'error', message: 'recent' }]);
|
|
assert.equal((await service.recentEvents(10)).at(-1).message, 'recent');
|
|
});
|
|
|
|
test('paused legacy work is validated, published, and converted to incremental mode', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-legacy-publish-'));
|
|
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
|
|
const workRoot = path.join(libraryRoot, '.mcscraper-work-SERIES42');
|
|
const stagedManga = path.join(workRoot, 'Legacy Manga'), book = path.join(stagedManga, 'v01');
|
|
await fs.mkdir(dataRoot, { recursive: true });
|
|
await fs.mkdir(book, { recursive: true });
|
|
await fs.writeFile(path.join(stagedManga, 'manga.json'), JSON.stringify({
|
|
title: 'Legacy Manga', status: 'ongoing',
|
|
sources: [{ name: 'Alpha', url: 'https://alpha.test/title/SERIES42' }],
|
|
'x-scraper': { ids: { alpha: 'SERIES42' } },
|
|
}));
|
|
for (const number of [1, 2]) {
|
|
const chapter = path.join(book, `c000${number} [Alpha]`);
|
|
await fs.mkdir(chapter, { recursive: true });
|
|
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: `https://alpha.test/chapter/${number}` } }));
|
|
await fs.writeFile(path.join(chapter, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
|
}
|
|
const incomplete = path.join(book, '.staging-mcscraper-abc123');
|
|
await fs.mkdir(incomplete);
|
|
await fs.writeFile(path.join(incomplete, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
|
await fs.writeFile(path.join(workRoot, 'job.json'), JSON.stringify({
|
|
version: 2, status: 'paused', completed: ['one', 'two'], total: 4,
|
|
}));
|
|
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
|
version: 3, paused: true, status: 'paused', settings: validateSettings({}), history: [], publishedRevision: 0,
|
|
queue: [{ id: 'old-id', seriesId: 'SERIES42', source: 'alpha', title: 'Legacy Manga', url: 'https://alpha.test/title/SERIES42', status: 'queued', order: 0 }],
|
|
}));
|
|
const alpha = {
|
|
id: 'alpha', label: 'Alpha',
|
|
match(value) { return String(value).includes('alpha.test/title/') ? { source: 'alpha', id: 'SERIES42', title: 'Legacy Manga', url: value } : null; },
|
|
async discover() { return { chapters: [] }; }, async run() { return null; },
|
|
};
|
|
const service = await new ScraperService({
|
|
libraryRoot, dataRoot, adapters: [alpha],
|
|
adapterFor: value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; },
|
|
}).init();
|
|
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
|
|
|
const inspection = await inspectLegacyWork({ libraryRoot, seriesId: 'SERIES42' });
|
|
assert.equal(inspection.chapterCount, 2);
|
|
assert.equal(inspection.pageCount, 2);
|
|
assert.equal(inspection.incompleteStages.length, 1);
|
|
const result = await service.publishLegacyReady();
|
|
assert.equal(result.published.length, 1);
|
|
assert.equal(result.published[0].chaptersPublished, 2);
|
|
assert.equal(result.published[0].incompleteStagesQuarantined, 1);
|
|
assert.equal(await fs.stat(path.join(libraryRoot, 'Legacy Manga', 'v01', 'c0002 [Alpha]')).then(stat => stat.isDirectory()), true);
|
|
await assert.rejects(fs.stat(stagedManga));
|
|
const migrated = JSON.parse(await fs.readFile(path.join(workRoot, 'job.json'), 'utf8'));
|
|
assert.equal(migrated.publicationMode, 'incremental');
|
|
assert.equal(migrated.migratedToIncremental.chaptersPublished, 2);
|
|
assert.equal(migrated.migratedToIncremental.incompleteStagesQuarantined, 1);
|
|
assert.equal(await fs.stat(path.join(workRoot, '.legacy-incomplete', 'v01__.staging-mcscraper-abc123')).then(stat => stat.isDirectory()), true);
|
|
assert.equal(service.state.queue[0].libraryMangaId, 'Legacy Manga');
|
|
assert.equal(service.state.publishedRevision, 1);
|
|
});
|
|
|
|
test('WeebCentral fragments preserve chapter and page order', () => {
|
|
const chapters = parseChapters('<a href="/chapters/P1">Prologue 1</a><a href="/chapters/ABC">Chapter 12</a><a href="/chapters/DEF">Chapter 12.5</a><a href="/chapters/S2">S2 - Chapter 1</a><a href="/chapters/R442"><span>Rating 442</span><span>Last Read</span></a><a href="/chapters/E128"><span>Episode 128</span><span>Last Read</span></a><a href="/chapters/E78"><span>Episode: 78</span><span>Last Read</span></a><a href="/chapters/N182"><span>Night 182</span><span>Last Read</span></a><a href="/chapters/L65">Log. 65</a><a href="/chapters/A606">Act 60.6</a><a href="/chapters/C155">Scene 155</a><a href="/chapters/K162">Kapitel 162</a><a href="/chapters/F83">File 83</a><a href="/chapters/B281">Battle 281</a>', 'https://weebcentral.com/s');
|
|
assert.deepEqual(chapters.map(item => item.number), ['0.01', '12', '12.5', '1', '442', '128', '78', '182', '65', '60.6', '155', '162', '83', '281']);
|
|
assert.deepEqual(chapters[3].book, { number: 2, title: 'Season 2' });
|
|
const pages = parsePageUrls('<img src="/001.png"><img data-src="https://img.example/002.webp">', 'https://weebcentral.com/c');
|
|
assert.deepEqual(pages, ['https://weebcentral.com/001.png', 'https://img.example/002.webp']);
|
|
});
|
|
|
|
test('WeebCentral reports unrecognized chapter labels separately from an empty catalog', async () => {
|
|
const series = '<h1>Example</h1><div hx-get="/series/X/full-chapter-list"></div>';
|
|
const httpFor = list => ({ get: async url => ({ text: async () => url.endsWith('full-chapter-list') ? list : series }) });
|
|
const unknown = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<a href="/chapters/ABC">Part One</a>'));
|
|
assert.equal(unknown.chapterLinks, 1);
|
|
assert.equal(unknown.chapters.length, 0);
|
|
assert.equal(unknown.availability.reason, 'unrecognized-labels');
|
|
assert.match(unknown.availability.message, /1 chapter links/);
|
|
const empty = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<div>No chapters</div>'));
|
|
assert.equal(empty.chapterLinks, 0);
|
|
assert.equal(empty.availability, undefined);
|
|
});
|
|
|
|
test('completed download reinspection preserves complete state when all scoped chapters are present', async () => {
|
|
const service = new ScraperService({ libraryRoot: 'unused', dataRoot: 'unused' });
|
|
const item = { id: 'done', title: 'Done', source: 'weebcentral', url: 'https://weebcentral.com/series/EXAMPLE/Done',
|
|
seriesId: 'EXAMPLE', kind: 'inspection', status: 'queued', completed: '2026-10-01T00:00:00Z',
|
|
publishedCount: 2, selectedCount: 2, scope: { mode: 'missing' }, inspectionFromComplete: true };
|
|
service.state = { paused: false, settings: validateSettings({}), queue: [item], incidents: [], sourcePaused: {}, currentIds: {} };
|
|
service.recordCatalog = async (entry, catalog) => { entry.sourceTitle = catalog.title; entry.catalogTotal = catalog.chapters.length; };
|
|
service.libraryInventory = async () => ({ mangas: [{ id: 'Done', ids: { weebcentral: 'EXAMPLE' }, sources: [],
|
|
chapters: [{ book: '1', number: '1' }, { book: '1', number: '2' }] }] });
|
|
service.event = async () => {};
|
|
const adapter = { discover: async () => ({ title: 'Done', chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }] }) };
|
|
await service.runDiscovery(item, { adapter }, 'weebcentral');
|
|
assert.equal(item.status, 'complete');
|
|
assert.equal(item.kind, 'download');
|
|
assert.equal(item.completed, '2026-10-01T00:00:00Z');
|
|
assert.equal(item.publishedCount, 2);
|
|
assert.equal(item.inspectionFromComplete, undefined);
|
|
});
|
|
|
|
test('publisher keeps season chapter numbers in separate books', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-books-'));
|
|
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
|
const page = path.join(temp, 'page.png');
|
|
await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
|
const base = { title: '', source: { name: 'Host', url: 'https://example.test/chapter' }, pages: [{ path: page }] };
|
|
await publishManifest({
|
|
version: 1,
|
|
series: { title: 'Season Test', direction: 'ttb', sources: [{ name: 'Host', url: 'https://example.test/series' }] },
|
|
chapters: [
|
|
{ ...base, number: '1', book: { number: 1, title: 'Season 1' } },
|
|
{ ...base, number: '1', book: { number: 2, title: 'Season 2' } },
|
|
],
|
|
}, { libraryRoot: path.join(temp, 'library') });
|
|
assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v01 Season 1', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 1');
|
|
assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v02 Season 2', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 2');
|
|
});
|
|
|
|
test('publisher stages and atomically exposes a complete new manga', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-test-'));
|
|
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
|
const page = path.join(temp, 'page.png');
|
|
await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
|
const result = await publishManifest({
|
|
version: 1,
|
|
series: { title: 'Test Manga', direction: 'rtl', metadata: { status: 'ongoing' },
|
|
sources: [{ name: 'Test', url: 'https://example.test/manga' }] },
|
|
chapters: [{ number: '1', title: 'Start', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }],
|
|
}, { libraryRoot: path.join(temp, 'library') });
|
|
assert.equal(result.results[0].status, 'published');
|
|
const chapter = path.join(temp, 'library', 'Test Manga', 'v01', 'c0001 Start [Test]');
|
|
assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', 'chapter.json']);
|
|
const meta = JSON.parse(await fs.readFile(path.join(temp, 'library', 'Test Manga', 'manga.json'), 'utf8'));
|
|
assert.equal(meta['x-scraper'].autoUpdate, undefined, 'ongoing is a derived default, not a frozen override');
|
|
assert.equal((await scanLibrary(path.join(temp, 'library'))).mangas[0].autoUpdate, true);
|
|
assert.equal((await fs.readdir(path.join(temp, 'library'))).some(name => name.startsWith('.staging-')), false);
|
|
});
|
|
|
|
test('publisher installs a validated AniList series cover', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-cover-'));
|
|
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
|
const page = path.join(temp, 'page.png');
|
|
const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]);
|
|
await fs.writeFile(page, png);
|
|
let requests = 0;
|
|
const httpClient = {
|
|
async get(url) {
|
|
requests++;
|
|
assert.equal(url, 'https://s4.anilist.co/example.png');
|
|
return { arrayBuffer: async () => png };
|
|
},
|
|
};
|
|
const series = {
|
|
title: 'Covered Manga', direction: 'rtl',
|
|
sources: [{ name: 'Test', url: 'https://example.test/covered' }],
|
|
coverUrl: 'https://s4.anilist.co/example.png',
|
|
};
|
|
const result = await publishManifest({
|
|
version: 1, series,
|
|
chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }],
|
|
}, { libraryRoot: path.join(temp, 'library'), httpClient });
|
|
assert.equal(result.cover.status, 'installed');
|
|
assert.deepEqual(await fs.readFile(path.join(temp, 'library', 'Covered Manga', 'cover.png')), png);
|
|
assert.equal(requests, 1);
|
|
});
|
|
|
|
test('publisher retains validated partial pages and resumes without duplicate requests', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-resume-'));
|
|
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
|
const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]);
|
|
const manifest = {
|
|
version: 1, series: { title: 'Resumable', direction: 'rtl', sources: [{ name: 'Test', url: 'https://example.test/series' }] },
|
|
chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' },
|
|
pages: [1, 2].map(n => ({ url: `https://example.test/${n}.png` })) }],
|
|
};
|
|
const requests = [];
|
|
const httpClient = { async get(url) { requests.push(url); return { arrayBuffer: async () => png }; } };
|
|
const libraryRoot = path.join(temp, 'library');
|
|
await assert.rejects(publishManifest(manifest, { libraryRoot, httpClient,
|
|
onPage() { throw new Error('cooperative pause'); } }), /cooperative pause/);
|
|
await assert.rejects(fs.stat(path.join(libraryRoot, 'Resumable')));
|
|
const hidden = (await fs.readdir(libraryRoot))[0];
|
|
assert.match(hidden, /^\.staging-mcscraper-resume-/);
|
|
const { cleanupOwnTemps } = require('../lib/publisher');
|
|
await cleanupOwnTemps(libraryRoot, -1);
|
|
assert.ok(await fs.stat(path.join(libraryRoot, hidden)));
|
|
await publishManifest(manifest, { libraryRoot, httpClient });
|
|
assert.deepEqual(requests, ['https://example.test/1.png', 'https://example.test/2.png']);
|
|
const chapter = path.join(libraryRoot, 'Resumable', 'v01', 'c0001 [Test]');
|
|
assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', '0002.png', 'chapter.json']);
|
|
});
|
|
|
|
test('adapter registry recognizes supported series URLs', () => {
|
|
const found = adapterFor('https://weebcentral.com/series/01ABCDEF/My-Series');
|
|
assert.equal(found.adapter.id, 'weebcentral');
|
|
assert.equal(found.info.id, '01ABCDEF');
|
|
assert.equal(found.info.title, 'My Series');
|
|
const dex = adapterFor('https://mangadex.org/title/5bdb093f-9707-4a30-b02e-c8b66554ecf0/biomega');
|
|
assert.equal(dex.adapter.id, 'mangadex');
|
|
assert.equal(dex.info.id, '5bdb093f-9707-4a30-b02e-c8b66554ecf0');
|
|
assert.equal(adapterFor('https://example.test/series/1'), null);
|
|
});
|
|
|
|
test('MangaDex releases are normalized, deduplicated, credited, and host-validated', () => {
|
|
assert.equal(chapterNumber('0012.50'), '12.5');
|
|
assert.equal(chapterNumber('Special'), null);
|
|
const release = (id, chapter, group, publishAt) => ({
|
|
id, attributes: { chapter, title: '', pages: 2, publishAt, externalUrl: null },
|
|
relationships: group ? [{ id: `group-${id}`, type: 'scanlation_group', attributes: { name: group } }] : [],
|
|
});
|
|
const selected = selectChapterReleases([
|
|
release('fan', '1', 'Fan Group', '2020-01-01T00:00:00Z'),
|
|
release('official', '1.0', 'Publisher Official', '2021-01-01T00:00:00Z'),
|
|
release('two', '2', 'Fan Group', '2020-01-02T00:00:00Z'),
|
|
]);
|
|
assert.deepEqual(selected.map(item => item.id), ['official', 'two']);
|
|
assert.equal(selected[0].group.name, 'Publisher Official');
|
|
const pages = pageUrls({
|
|
baseUrl: 'https://node.mangadex.network',
|
|
chapter: { hash: 'abc', data: ['1.jpg', '2.png'] },
|
|
}, 'https://mangadex.org/chapter/test', 2);
|
|
assert.deepEqual(pages.map(item => item.url), [
|
|
'https://node.mangadex.network/data/abc/1.jpg',
|
|
'https://node.mangadex.network/data/abc/2.png',
|
|
]);
|
|
assert.throws(() => pageUrls({ baseUrl: 'https://evil.example', chapter: { hash: 'abc', data: ['1.jpg'] } }, 'https://mangadex.org/chapter/test', 1));
|
|
});
|
|
|
|
test('daemon queue state is durable while paused', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-service-'));
|
|
const dataRoot = path.join(temp, 'data');
|
|
await fs.mkdir(dataRoot, { recursive: true });
|
|
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
|
version: 1, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
|
|
}));
|
|
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init();
|
|
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
|
const item = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series' });
|
|
assert.equal(item.status, 'queued');
|
|
const scoped = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series', scope: { mode: 'latest' } });
|
|
assert.notEqual(item.id, scoped.id);
|
|
assert.equal(item.seriesId, scoped.seriesId);
|
|
assert.equal((await service.publicState()).queue.length, 2);
|
|
await service.updateSettings({ minMs: 1500, maxMs: 4500 });
|
|
assert.equal(JSON.parse(await fs.readFile(path.join(dataRoot, 'queue.json'), 'utf8')).settings.minMs, 1500);
|
|
await service.remove(item.id);
|
|
await service.remove(scoped.id);
|
|
assert.equal((await service.publicState()).queue.length, 0);
|
|
});
|
|
|
|
test('retry recovers stale downloading entries before work resumes', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-retry-'));
|
|
const dataRoot = path.join(temp, 'data');
|
|
await fs.mkdir(dataRoot, { recursive: true });
|
|
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
|
version: 1, paused: true, status: 'error', currentId: 'FAILED', settings: validateSettings({}),
|
|
queue: [
|
|
{ id: 'STALE', title: 'Stale', url: 'https://weebcentral.com/series/STALE/Stale', status: 'downloading', order: 0 },
|
|
{ id: 'FAILED', title: 'Failed', url: 'https://weebcentral.com/series/FAILED/Failed', status: 'error', order: 1 },
|
|
], history: [], publishedRevision: 0,
|
|
}));
|
|
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init();
|
|
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
|
await service.control('retry', 'FAILED');
|
|
assert.deepEqual(service.state.queue.map(item => item.status), ['queued', 'queued']);
|
|
assert.equal(service.state.currentId, null);
|
|
assert.equal(service.state.paused, false);
|
|
});
|
|
|
|
test('nightly discovery queues one preferred supported source for ongoing manga', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-'));
|
|
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
|
|
const manga = path.join(libraryRoot, 'Ongoing Manga');
|
|
await fs.mkdir(manga, { recursive: true });
|
|
await fs.mkdir(dataRoot, { recursive: true });
|
|
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({
|
|
title: 'Ongoing Manga', status: 'ongoing',
|
|
sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }],
|
|
'x-scraper': { ids: { alpha: '42' } },
|
|
}));
|
|
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
|
version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
|
|
}));
|
|
const alpha = {
|
|
id: 'alpha', label: 'Alpha',
|
|
match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: url.pathname.split('/').pop(), title: 'Alpha', url: value } : null; },
|
|
async discover() { return { chapters: [] }; }, async run() { return null; },
|
|
};
|
|
const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; };
|
|
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init();
|
|
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
|
const first = await service.enqueueDiscoveryChecks('test');
|
|
const second = await service.enqueueDiscoveryChecks('test');
|
|
assert.deepEqual(first, { queued: 1, skipped: 0 });
|
|
assert.deepEqual(second, { queued: 0, skipped: 1 });
|
|
assert.equal(service.state.queue[0].kind, 'discovery');
|
|
assert.equal(service.state.queue[0].libraryMangaId, 'Ongoing Manga');
|
|
});
|
|
|
|
test('discovery promotes only chapters newer than the library frontier', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-run-'));
|
|
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
|
|
const chapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0001 [Alpha]');
|
|
const laterChapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0003 [Alpha]');
|
|
await fs.mkdir(chapter, { recursive: true });
|
|
await fs.mkdir(laterChapter, { recursive: true });
|
|
await fs.mkdir(dataRoot, { recursive: true });
|
|
await fs.writeFile(path.join(libraryRoot, 'Ongoing Manga', 'manga.json'), JSON.stringify({
|
|
title: 'Ongoing Manga', status: 'ongoing', sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }],
|
|
'x-scraper': { ids: { alpha: '42' } },
|
|
}));
|
|
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/one' } }));
|
|
await fs.writeFile(path.join(laterChapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/three' } }));
|
|
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
|
version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
|
|
}));
|
|
let downloadedScope = null;
|
|
const alpha = {
|
|
id: 'alpha', label: 'Alpha',
|
|
match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: '42', title: 'Alpha', url: value } : null; },
|
|
async discover() { return { chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }, { id: 'three', number: '3' }, { id: 'four', number: '4' }] }; },
|
|
async run(item, options) { downloadedScope = item.scope; await options.onPublished({ chapter: '4' }); return { finalPath: path.join(libraryRoot, 'Ongoing Manga'), publishedCount: 1, selectedCount: 1 }; },
|
|
};
|
|
const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; };
|
|
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init();
|
|
const waitFor = async predicate => {
|
|
const deadline = Date.now() + 3000;
|
|
while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for discovery'); await new Promise(resolveWait => setTimeout(resolveWait, 20)); }
|
|
};
|
|
try {
|
|
await service.enqueueDiscoveryChecks('test');
|
|
await service.control('resume');
|
|
await waitFor(() => service.state.queue[0]?.status === 'complete');
|
|
assert.deepEqual(downloadedScope, { mode: 'ids', ids: ['four'] });
|
|
assert.equal(service.state.queue[0].publishedCount, 1);
|
|
const summary = (await service.publicState()).updateChecks.find(item => item.mangaId === 'Ongoing Manga' && item.source === 'alpha');
|
|
assert.ok(summary.lastCheckedAt && summary.lastNewContentAt && summary.lastDownloadedAt);
|
|
assert.equal(summary.lastResult, 'new-content');
|
|
} finally {
|
|
await service.close();
|
|
await fs.rm(temp, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('source-aware scheduler runs one item per source in parallel', async t => {
|
|
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-sources-'));
|
|
const dataRoot = path.join(temp, 'data');
|
|
await fs.mkdir(dataRoot, { recursive: true });
|
|
const queue = [
|
|
{ id: 'A1', source: 'alpha', title: 'Alpha One', url: 'https://alpha.test/1', status: 'queued', order: 0 },
|
|
{ id: 'A2', source: 'alpha', title: 'Alpha Two', url: 'https://alpha.test/2', status: 'queued', order: 1 },
|
|
{ id: 'B1', source: 'beta', title: 'Beta One', url: 'https://beta.test/1', status: 'queued', order: 2 },
|
|
];
|
|
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
|
version: 2, paused: true, status: 'paused', settings: validateSettings({}), queue, history: [], publishedRevision: 0,
|
|
}));
|
|
const resolvers = new Map(), started = [];
|
|
const fakeAdapter = id => ({
|
|
id, label: id,
|
|
match(value) { const url = new URL(value); return url.hostname === `${id}.test` ? { source: id, id: url.pathname.slice(1), title: id, url: value } : null; },
|
|
run(item) { started.push(item.id); return new Promise(resolve => resolvers.set(item.id, () => resolve(path.join(temp, item.id)))); },
|
|
});
|
|
const fakeAdapters = [fakeAdapter('alpha'), fakeAdapter('beta')];
|
|
const resolveAdapter = value => {
|
|
for (const adapter of fakeAdapters) { const info = adapter.match(value); if (info) return { adapter, info }; }
|
|
return null;
|
|
};
|
|
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot, adapterFor: resolveAdapter, adapters: fakeAdapters }).init();
|
|
const waitFor = async predicate => {
|
|
const deadline = Date.now() + 3000;
|
|
while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for scheduler'); await new Promise(resolve => setTimeout(resolve, 20)); }
|
|
};
|
|
try {
|
|
await service.control('resume');
|
|
await waitFor(() => resolvers.has('A1') && resolvers.has('B1'));
|
|
assert.deepEqual(new Set(started), new Set(['A1', 'B1']));
|
|
assert.equal(resolvers.has('A2'), false);
|
|
resolvers.get('A1')();
|
|
await waitFor(() => resolvers.has('A2'));
|
|
assert.equal(service.state.queue.find(item => item.id === 'A2').status, 'downloading');
|
|
resolvers.get('A2')(); resolvers.get('B1')();
|
|
await waitFor(() => service.state.queue.every(item => item.status === 'complete'));
|
|
assert.equal(service.state.publishedRevision, 3);
|
|
} finally {
|
|
for (const resolve of resolvers.values()) resolve();
|
|
await service.close();
|
|
await fs.rm(temp, { recursive: true, force: true });
|
|
}
|
|
});
|