'use strict';
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const { cleanPart, chapterKey, chapterFolder, assertInside } = require('../lib/names');
const { imageExtension } = require('../lib/images');
const { publishManifest } = require('../lib/publisher');
const { mergeImportedMetadata } = require('../lib/metadata-merge');
const { metadataFromMedia } = require('../metadata/anilist');
const { normalizeScope, selectByScope, scopeLabel } = require('../lib/scope');
const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, setAutoUpdate } = require('../lib/library-inventory');
const { inspectLegacyWork } = require('../lib/legacy-publication');
const { discoverWeebCatalog, parseChapters, parsePageUrls } = require('../adapters/weebcentral');
const { chapterNumber, selectChapterReleases, pageUrls } = require('../adapters/mangadex');
const { adapterFor } = require('../adapters');
const { ScraperService, validateSettings, settingsForSource } = require('../service');
test('MangaCircus chapter keys are normalized', () => {
assert.equal(chapterKey('12'), 'c0012');
assert.equal(chapterKey('12.50'), 'c0012.5');
assert.equal(chapterKey('x2'), 'x02');
assert.throws(() => chapterKey('special'));
});
test('folder names are portable and versioned', () => {
assert.equal(cleanPart('Bad:
. '), 'BadTitle');
assert.equal(cleanPart('CON'), 'CON-manga');
assert.equal(chapterFolder({ number: '7', title: 'A [Test]', source: { name: 'Host' } }), 'c0007 A Test [Host]');
assert.throws(() => assertInside('C:/library', 'C:/outside'));
});
test('image signatures are recognized', () => {
assert.equal(imageExtension(Buffer.from([0xff, 0xd8, 0xff, 0x00])), 'jpg');
assert.equal(imageExtension(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])), 'png');
assert.equal(imageExtension(Buffer.from('not an image')), null);
});
test('per-source rate settings default fast and update independently', () => {
const defaults = validateSettings({});
const fast = { minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000 };
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast);
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), fast);
const changed = validateSettings({ sourceRates: { mangadex: {
minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000,
} } }, defaults);
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), {
minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000,
});
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast);
assert.throws(() => validateSettings({ sourceRates: { mangadex: { minMs: 5000, maxMs: 1000 } } }, defaults));
});
test('AniList metadata keeps manga format and original story credits', () => {
const metadata = metadataFromMedia({
id: 150193, idMal: null, format: 'MANGA',
title: { english: 'Heavy Knight', romaji: 'Juukishi', native: '重騎士' }, synonyms: [],
staff: { edges: [
{ role: 'Original Story', node: { name: { full: 'Nekoko' } } },
{ role: 'Story & Art', node: { name: { full: 'Lee Brocco' } } },
{ role: 'Character Design', node: { name: { full: 'Jaian' } } },
] },
status: 'RELEASING', startDate: { year: 2022 }, genres: ['Fantasy'], description: '',
siteUrl: 'https://anilist.co/manga/150193', coverImage: { extraLarge: 'https://s4.anilist.co/cover.jpg' },
});
assert.deepEqual(metadata.authors, ['Nekoko', 'Lee Brocco']);
assert.equal(metadata['x-anilist'].format, 'MANGA');
});
test('metadata refresh preserves legacy and manual fields while updating untouched imports', () => {
const first = mergeImportedMetadata({ title: 'My preferred title', status: 'ongoing' }, {
title: 'Provider title', status: 'completed', tags: ['Fantasy'], 'x-anilist': { id: 42 },
}, 'Fallback');
assert.equal(first.title, 'My preferred title');
assert.equal(first.status, 'ongoing');
assert.deepEqual(first.tags, ['Fantasy']);
assert.equal(first['x-metadata'].imported.title, 'Provider title');
const second = mergeImportedMetadata({ ...first, tags: ['Fantasy', 'Favorite'] }, {
title: 'New provider title', status: 'ongoing', tags: ['Action'], 'x-anilist': { id: 42 },
}, 'Fallback');
assert.equal(second.title, 'My preferred title');
assert.equal(second.status, 'ongoing');
assert.deepEqual(second.tags, ['Fantasy', 'Favorite']);
const third = mergeImportedMetadata(second, { description: 'First', 'x-anilist': { id: 42 } }, 'Fallback');
assert.equal(mergeImportedMetadata(third, { description: 'Revised', 'x-anilist': { id: 42 } }, 'Fallback').description, 'Revised');
assert.throws(() => mergeImportedMetadata(first, { 'x-anilist': { id: 99 } }, 'Fallback'), /identity conflict/);
assert.throws(() => mergeImportedMetadata({ 'x-work-identity': { scheme: 'mangacircus-work-v1', kind: 'origin', value: 'a'.repeat(64) } },
{ 'x-work-identity': { scheme: 'mangacircus-work-v1', kind: 'origin', value: 'b'.repeat(64) } }, 'Fallback'), /work identity conflict/);
});
test('publisher rejects a conflicting work ID before fetching any pages', async t => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-identity-'));
t.after(() => fs.rm(root, { recursive: true, force: true }));
const manga = path.join(root, 'Existing');
await fs.mkdir(manga);
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Existing',
sources: [{ name: 'Host', url: 'https://example.test/series' }], 'x-anilist': { id: 42 } }));
let requested = 0;
await assert.rejects(publishManifest({ version: 1,
series: { title: 'Existing', direction: 'rtl', metadata: { 'x-anilist': { id: 99 } },
sources: [{ name: 'Host', url: 'https://example.test/series' }] },
chapters: [{ number: '1', source: { name: 'Host', url: 'https://example.test/chapter' },
pages: [{ url: 'https://example.test/page.png' }] }],
}, { libraryRoot: root, httpClient: { async get() { requested++; throw new Error('unexpected request'); } } }), /identity conflict/);
assert.equal(requested, 0);
assert.equal((await fs.readdir(manga)).includes('v01'), false);
});
test('publisher explains an unresolved same-title folder collision without touching its contents', async t => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-title-collision-'));
t.after(() => fs.rm(root, { recursive: true, force: true }));
const manga = path.join(root, 'Same Title');
await fs.mkdir(manga);
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Same Title' }));
await fs.writeFile(path.join(manga, 'operator-note.txt'), 'keep');
let requested = 0;
await assert.rejects(publishManifest({ version: 1,
series: { title: 'Same Title', direction: 'rtl', sources: [{ name: 'Peer', url: 'peer://p123/m123' }] },
chapters: [{ number: '1', source: { name: 'Peer', url: 'peer://p123/m123' }, pages: [{ url: 'https://example.test/page.png' }] }],
}, { libraryRoot: root, httpClient: { async get() { requested++; throw new Error('unexpected request'); } } }),
/could not be safely matched.*No files were overwritten.*Choose the existing manga/s);
assert.equal(requested, 0);
assert.equal(await fs.readFile(path.join(manga, 'operator-note.txt'), 'utf8'), 'keep');
});
test('queue scopes select exact catalog slices', () => {
const chapters = [
{ id: 'a', number: '1', volume: '1' }, { id: 'b', number: '2', volume: '1' },
{ id: 'c', number: '3', volume: '2' }, { id: 'd', number: '4', volume: '2' },
];
assert.deepEqual(selectByScope(chapters, { mode: 'latest' }).map(x => x.id), ['d']);
assert.deepEqual(selectByScope(chapters, { mode: 'last', count: 2 }).map(x => x.id), ['c', 'd']);
assert.deepEqual(selectByScope(chapters, { mode: 'first', count: 2 }).map(x => x.id), ['a', 'b']);
assert.deepEqual(selectByScope(chapters, { mode: 'chapter', chapter: '003.0' }).map(x => x.id), ['c']);
assert.deepEqual(selectByScope(chapters, { mode: 'after', chapter: '2' }).map(x => x.id), ['c', 'd']);
assert.deepEqual(selectByScope(chapters, { mode: 'volume', volume: '2' }).map(x => x.id), ['c', 'd']);
assert.deepEqual(selectByScope(chapters, { mode: 'ids', ids: ['b', 'd'] }).map(x => x.id), ['b', 'd']);
assert.equal(scopeLabel(normalizeScope({ mode: 'last', count: 5 })), 'last 5 chapters');
});
test('library inventory distinguishes missing chapters from alternate source versions', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-inventory-'));
t.after(() => fs.rm(temp, { recursive: true, force: true }));
const manga = path.join(temp, 'Library Manga'), book = path.join(manga, 'v01');
await fs.mkdir(path.join(book, 'c0001 [WeebCentral]'), { recursive: true });
await fs.mkdir(path.join(book, 'c0002 [MangaDex]'), { recursive: true });
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({
title: 'Library Manga', status: 'ongoing',
sources: [{ name: 'WeebCentral', url: 'https://weebcentral.com/series/SERIES/Test' }],
'x-scraper': { ids: { weebcentral: 'SERIES', anilist: 42 } },
}));
await fs.writeFile(path.join(book, 'c0001 [WeebCentral]', 'chapter.json'), JSON.stringify({ source: { name: 'WeebCentral', url: 'https://weebcentral.com/chapters/A' } }));
await fs.writeFile(path.join(book, 'c0002 [MangaDex]', 'chapter.json'), JSON.stringify({ source: { name: 'MangaDex · Group', url: 'https://mangadex.org/chapter/B' } }));
const inventory = await scanLibrary(temp);
const found = findManga(inventory, { source: 'weebcentral', seriesId: 'SERIES' });
assert.equal(found.id, 'Library Manga');
assert.equal(findManga(inventory, { source: 'mangadex', seriesId: 'other', anilist: 42 }).id, 'Library Manga');
assert.equal(found.autoUpdate, true);
assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'skip-any'), false);
assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'add-source'), true);
assert.equal(shouldDownload(found, { number: '1' }, 'weebcentral', 'add-source'), false);
assert.equal(shouldDownload(found, { number: '3' }, 'weebcentral', 'skip-any'), true);
assert.equal(isNewerThanLibrary(found, { number: '1' }), false);
assert.equal(isNewerThanLibrary(found, { number: '3' }), true);
await setAutoUpdate(temp, 'Library Manga', false);
assert.equal((await scanLibrary(temp)).mangas[0].autoUpdate, false);
});
test('a shared work ID cannot merge distinct source listings or ambiguous editions', () => {
const original = { id: 'Main', ids: { weebcentral: 'MAIN', anilist: 42 }, sources: [] };
assert.equal(findManga({ mangas: [original] }, { source: 'weebcentral', seriesId: 'PILOT', anilist: 42 }), null);
assert.equal(findManga({ mangas: [original] }, { source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), original);
assert.equal(findManga({ mangas: [original, { id: 'Pilot', ids: { anilist: 42 }, sources: [] }] },
{ source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), null);
});
test('recent events read the bounded log tail without losing the newest incident', async t => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-events-'));
t.after(() => fs.rm(root, { recursive: true, force: true }));
const service = new ScraperService({ dataRoot: root });
await fs.writeFile(service.logFile, `${'x'.repeat(600 * 1024)}\n${JSON.stringify({ level: 'error', message: 'recent' })}\n`);
assert.deepEqual(await service.recentEvents(1), [{ level: 'error', message: 'recent' }]);
assert.equal((await service.recentEvents(10)).at(-1).message, 'recent');
});
test('paused legacy work is validated, published, and converted to incremental mode', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-legacy-publish-'));
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
const workRoot = path.join(libraryRoot, '.mcscraper-work-SERIES42');
const stagedManga = path.join(workRoot, 'Legacy Manga'), book = path.join(stagedManga, 'v01');
await fs.mkdir(dataRoot, { recursive: true });
await fs.mkdir(book, { recursive: true });
await fs.writeFile(path.join(stagedManga, 'manga.json'), JSON.stringify({
title: 'Legacy Manga', status: 'ongoing',
sources: [{ name: 'Alpha', url: 'https://alpha.test/title/SERIES42' }],
'x-scraper': { ids: { alpha: 'SERIES42' } },
}));
for (const number of [1, 2]) {
const chapter = path.join(book, `c000${number} [Alpha]`);
await fs.mkdir(chapter, { recursive: true });
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: `https://alpha.test/chapter/${number}` } }));
await fs.writeFile(path.join(chapter, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
}
const incomplete = path.join(book, '.staging-mcscraper-abc123');
await fs.mkdir(incomplete);
await fs.writeFile(path.join(incomplete, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
await fs.writeFile(path.join(workRoot, 'job.json'), JSON.stringify({
version: 2, status: 'paused', completed: ['one', 'two'], total: 4,
}));
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
version: 3, paused: true, status: 'paused', settings: validateSettings({}), history: [], publishedRevision: 0,
queue: [{ id: 'old-id', seriesId: 'SERIES42', source: 'alpha', title: 'Legacy Manga', url: 'https://alpha.test/title/SERIES42', status: 'queued', order: 0 }],
}));
const alpha = {
id: 'alpha', label: 'Alpha',
match(value) { return String(value).includes('alpha.test/title/') ? { source: 'alpha', id: 'SERIES42', title: 'Legacy Manga', url: value } : null; },
async discover() { return { chapters: [] }; }, async run() { return null; },
};
const service = await new ScraperService({
libraryRoot, dataRoot, adapters: [alpha],
adapterFor: value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; },
}).init();
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
const inspection = await inspectLegacyWork({ libraryRoot, seriesId: 'SERIES42' });
assert.equal(inspection.chapterCount, 2);
assert.equal(inspection.pageCount, 2);
assert.equal(inspection.incompleteStages.length, 1);
const result = await service.publishLegacyReady();
assert.equal(result.published.length, 1);
assert.equal(result.published[0].chaptersPublished, 2);
assert.equal(result.published[0].incompleteStagesQuarantined, 1);
assert.equal(await fs.stat(path.join(libraryRoot, 'Legacy Manga', 'v01', 'c0002 [Alpha]')).then(stat => stat.isDirectory()), true);
await assert.rejects(fs.stat(stagedManga));
const migrated = JSON.parse(await fs.readFile(path.join(workRoot, 'job.json'), 'utf8'));
assert.equal(migrated.publicationMode, 'incremental');
assert.equal(migrated.migratedToIncremental.chaptersPublished, 2);
assert.equal(migrated.migratedToIncremental.incompleteStagesQuarantined, 1);
assert.equal(await fs.stat(path.join(workRoot, '.legacy-incomplete', 'v01__.staging-mcscraper-abc123')).then(stat => stat.isDirectory()), true);
assert.equal(service.state.queue[0].libraryMangaId, 'Legacy Manga');
assert.equal(service.state.publishedRevision, 1);
});
test('WeebCentral fragments preserve chapter and page order', () => {
const chapters = parseChapters('Prologue 1Chapter 12Chapter 12.5S2 - Chapter 1Rating 442Last ReadEpisode 128Last ReadEpisode: 78Last ReadNight 182Last ReadLog. 65Act 60.6Scene 155Kapitel 162File 83Battle 281', 'https://weebcentral.com/s');
assert.deepEqual(chapters.map(item => item.number), ['0.01', '12', '12.5', '1', '442', '128', '78', '182', '65', '60.6', '155', '162', '83', '281']);
assert.deepEqual(chapters[3].book, { number: 2, title: 'Season 2' });
const pages = parsePageUrls('
', 'https://weebcentral.com/c');
assert.deepEqual(pages, ['https://weebcentral.com/001.png', 'https://img.example/002.webp']);
});
test('WeebCentral reports unrecognized chapter labels separately from an empty catalog', async () => {
const series = 'Example
';
const requests = [];
const httpFor = list => ({ get: async (url, options) => { requests.push({ url, options }); return {
text: async () => url.endsWith('full-chapter-list') ? list : series,
}; } });
const unknown = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('Part One'));
assert.equal(requests[0].options.catalog, undefined);
assert.equal(requests[1].options.catalog, true);
assert.equal(unknown.chapterLinks, 1);
assert.equal(unknown.chapters.length, 0);
assert.equal(unknown.availability.reason, 'unrecognized-labels');
assert.match(unknown.availability.message, /1 chapter links/);
const empty = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('No chapters
'));
assert.equal(empty.chapterLinks, 0);
assert.equal(empty.availability, undefined);
});
test('completed download reinspection preserves complete state when all scoped chapters are present', async () => {
const service = new ScraperService({ libraryRoot: 'unused', dataRoot: 'unused' });
const item = { id: 'done', title: 'Done', source: 'weebcentral', url: 'https://weebcentral.com/series/EXAMPLE/Done',
seriesId: 'EXAMPLE', kind: 'inspection', status: 'queued', completed: '2026-10-01T00:00:00Z',
publishedCount: 2, selectedCount: 2, scope: { mode: 'missing' }, inspectionFromComplete: true };
service.state = { paused: false, settings: validateSettings({}), queue: [item], incidents: [], sourcePaused: {}, currentIds: {} };
service.recordCatalog = async (entry, catalog) => { entry.sourceTitle = catalog.title; entry.catalogTotal = catalog.chapters.length; };
service.libraryInventory = async () => ({ mangas: [{ id: 'Done', ids: { weebcentral: 'EXAMPLE' }, sources: [],
chapters: [{ book: '1', number: '1' }, { book: '1', number: '2' }] }] });
service.event = async () => {};
const adapter = { discover: async () => ({ title: 'Done', chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }] }) };
await service.runDiscovery(item, { adapter }, 'weebcentral');
assert.equal(item.status, 'complete');
assert.equal(item.kind, 'download');
assert.equal(item.completed, '2026-10-01T00:00:00Z');
assert.equal(item.publishedCount, 2);
assert.equal(item.inspectionFromComplete, undefined);
});
test('publisher keeps season chapter numbers in separate books', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-books-'));
t.after(() => fs.rm(temp, { recursive: true, force: true }));
const page = path.join(temp, 'page.png');
await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
const base = { title: '', source: { name: 'Host', url: 'https://example.test/chapter' }, pages: [{ path: page }] };
await publishManifest({
version: 1,
series: { title: 'Season Test', direction: 'ttb', sources: [{ name: 'Host', url: 'https://example.test/series' }] },
chapters: [
{ ...base, number: '1', book: { number: 1, title: 'Season 1' } },
{ ...base, number: '1', book: { number: 2, title: 'Season 2' } },
],
}, { libraryRoot: path.join(temp, 'library') });
assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v01 Season 1', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 1');
assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v02 Season 2', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 2');
});
test('publisher stages and atomically exposes a complete new manga', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-test-'));
t.after(() => fs.rm(temp, { recursive: true, force: true }));
const page = path.join(temp, 'page.png');
await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
const result = await publishManifest({
version: 1,
series: { title: 'Test Manga', direction: 'rtl', metadata: { status: 'ongoing' },
sources: [{ name: 'Test', url: 'https://example.test/manga' }] },
chapters: [{ number: '1', title: 'Start', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }],
}, { libraryRoot: path.join(temp, 'library') });
assert.equal(result.results[0].status, 'published');
const chapter = path.join(temp, 'library', 'Test Manga', 'v01', 'c0001 Start [Test]');
assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', 'chapter.json']);
const meta = JSON.parse(await fs.readFile(path.join(temp, 'library', 'Test Manga', 'manga.json'), 'utf8'));
assert.equal(meta['x-scraper'].autoUpdate, undefined, 'ongoing is a derived default, not a frozen override');
assert.equal((await scanLibrary(path.join(temp, 'library'))).mangas[0].autoUpdate, true);
assert.equal((await fs.readdir(path.join(temp, 'library'))).some(name => name.startsWith('.staging-')), false);
});
test('publisher installs a validated AniList series cover', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-cover-'));
t.after(() => fs.rm(temp, { recursive: true, force: true }));
const page = path.join(temp, 'page.png');
const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]);
await fs.writeFile(page, png);
let requests = 0;
const httpClient = {
async get(url) {
requests++;
assert.equal(url, 'https://s4.anilist.co/example.png');
return { arrayBuffer: async () => png };
},
};
const series = {
title: 'Covered Manga', direction: 'rtl',
sources: [{ name: 'Test', url: 'https://example.test/covered' }],
coverUrl: 'https://s4.anilist.co/example.png',
};
const result = await publishManifest({
version: 1, series,
chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }],
}, { libraryRoot: path.join(temp, 'library'), httpClient });
assert.equal(result.cover.status, 'installed');
assert.deepEqual(await fs.readFile(path.join(temp, 'library', 'Covered Manga', 'cover.png')), png);
assert.equal(requests, 1);
});
test('publisher retains validated partial pages and resumes without duplicate requests', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-resume-'));
t.after(() => fs.rm(temp, { recursive: true, force: true }));
const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]);
const manifest = {
version: 1, series: { title: 'Resumable', direction: 'rtl', sources: [{ name: 'Test', url: 'https://example.test/series' }] },
chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' },
pages: [1, 2].map(n => ({ url: `https://example.test/${n}.png` })) }],
};
const requests = [];
const httpClient = { async get(url) { requests.push(url); return { arrayBuffer: async () => png }; } };
const libraryRoot = path.join(temp, 'library');
await assert.rejects(publishManifest(manifest, { libraryRoot, httpClient,
onPage() { throw new Error('cooperative pause'); } }), /cooperative pause/);
await assert.rejects(fs.stat(path.join(libraryRoot, 'Resumable')));
const hidden = (await fs.readdir(libraryRoot))[0];
assert.match(hidden, /^\.staging-mcscraper-resume-/);
const { cleanupOwnTemps } = require('../lib/publisher');
await cleanupOwnTemps(libraryRoot, -1);
assert.ok(await fs.stat(path.join(libraryRoot, hidden)));
await publishManifest(manifest, { libraryRoot, httpClient });
assert.deepEqual(requests, ['https://example.test/1.png', 'https://example.test/2.png']);
const chapter = path.join(libraryRoot, 'Resumable', 'v01', 'c0001 [Test]');
assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', '0002.png', 'chapter.json']);
});
test('adapter registry recognizes supported series URLs', () => {
const found = adapterFor('https://weebcentral.com/series/01ABCDEF/My-Series');
assert.equal(found.adapter.id, 'weebcentral');
assert.equal(found.info.id, '01ABCDEF');
assert.equal(found.info.title, 'My Series');
const dex = adapterFor('https://mangadex.org/title/5bdb093f-9707-4a30-b02e-c8b66554ecf0/biomega');
assert.equal(dex.adapter.id, 'mangadex');
assert.equal(dex.info.id, '5bdb093f-9707-4a30-b02e-c8b66554ecf0');
assert.equal(adapterFor('https://example.test/series/1'), null);
});
test('MangaDex releases are normalized, deduplicated, credited, and host-validated', () => {
assert.equal(chapterNumber('0012.50'), '12.5');
assert.equal(chapterNumber('Special'), null);
const release = (id, chapter, group, publishAt) => ({
id, attributes: { chapter, title: '', pages: 2, publishAt, externalUrl: null },
relationships: group ? [{ id: `group-${id}`, type: 'scanlation_group', attributes: { name: group } }] : [],
});
const selected = selectChapterReleases([
release('fan', '1', 'Fan Group', '2020-01-01T00:00:00Z'),
release('official', '1.0', 'Publisher Official', '2021-01-01T00:00:00Z'),
release('two', '2', 'Fan Group', '2020-01-02T00:00:00Z'),
]);
assert.deepEqual(selected.map(item => item.id), ['official', 'two']);
assert.equal(selected[0].group.name, 'Publisher Official');
const pages = pageUrls({
baseUrl: 'https://node.mangadex.network',
chapter: { hash: 'abc', data: ['1.jpg', '2.png'] },
}, 'https://mangadex.org/chapter/test', 2);
assert.deepEqual(pages.map(item => item.url), [
'https://node.mangadex.network/data/abc/1.jpg',
'https://node.mangadex.network/data/abc/2.png',
]);
assert.throws(() => pageUrls({ baseUrl: 'https://evil.example', chapter: { hash: 'abc', data: ['1.jpg'] } }, 'https://mangadex.org/chapter/test', 1));
});
test('daemon queue state is durable while paused', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-service-'));
const dataRoot = path.join(temp, 'data');
await fs.mkdir(dataRoot, { recursive: true });
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
version: 1, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
}));
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init();
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
// WeebCentral is off on a fresh installation until an admin switches it on.
await assert.rejects(() => service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series' }), /switched off/);
await service.updateSettings({ sourcesEnabled: { weebcentral: true } });
const item = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series' });
assert.equal(item.status, 'queued');
const scoped = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series', scope: { mode: 'latest' } });
assert.notEqual(item.id, scoped.id);
assert.equal(item.seriesId, scoped.seriesId);
assert.equal((await service.publicState()).queue.length, 2);
await service.updateSettings({ minMs: 1500, maxMs: 4500 });
assert.equal(JSON.parse(await fs.readFile(path.join(dataRoot, 'queue.json'), 'utf8')).settings.minMs, 1500);
await service.remove(item.id);
await service.remove(scoped.id);
assert.equal((await service.publicState()).queue.length, 0);
});
test('sources are off by default but existing installations keep the ones they use', async t => {
assert.deepEqual(validateSettings({}).sourcesEnabled, { weebcentral: false, mangadex: true, mangapill: false });
assert.throws(() => validateSettings({ sourcesEnabled: { weebcentral: 'yes' } }), /true or false/);
assert.throws(() => validateSettings({ sourcesEnabled: { other: true } }), /Unknown source/);
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-sources-'));
const dataRoot = path.join(temp, 'data');
await fs.mkdir(dataRoot, { recursive: true });
const { sourcesEnabled, ...older } = validateSettings({});
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
version: 3, paused: true, status: 'paused', settings: older, publishedRevision: 0,
queue: [{ id: 'A', title: 'A', url: 'https://weebcentral.com/series/AAA/A', source: 'weebcentral', status: 'queued', order: 0 }],
history: [],
}));
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init();
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
assert.deepEqual(service.state.settings.sourcesEnabled, { weebcentral: true, mangadex: true, mangapill: false });
await assert.rejects(() => service.enqueue({ url: 'https://mangapill.com/manga/2/one-piece' }), /MangaPill is switched off/);
// Switching a source off keeps its jobs queued instead of failing them.
await service.updateSettings({ sourcesEnabled: { weebcentral: false } });
assert.equal(service.sourceEnabled('weebcentral'), false);
assert.equal(service.sourceEnabled('peer-abc'), true, 'peers have no switch');
assert.equal(service.state.queue[0].status, 'queued');
});
test('retry recovers stale downloading entries before work resumes', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-retry-'));
const dataRoot = path.join(temp, 'data');
await fs.mkdir(dataRoot, { recursive: true });
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
version: 1, paused: true, status: 'error', currentId: 'FAILED', settings: validateSettings({}),
queue: [
{ id: 'STALE', title: 'Stale', url: 'https://weebcentral.com/series/STALE/Stale', status: 'downloading', order: 0 },
{ id: 'FAILED', title: 'Failed', url: 'https://weebcentral.com/series/FAILED/Failed', status: 'error', order: 1 },
], history: [], publishedRevision: 0,
}));
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init();
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
await service.control('retry', 'FAILED');
assert.deepEqual(service.state.queue.map(item => item.status), ['queued', 'queued']);
assert.equal(service.state.currentId, null);
assert.equal(service.state.paused, false);
});
test('nightly discovery queues one preferred supported source for ongoing manga', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-'));
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
const manga = path.join(libraryRoot, 'Ongoing Manga');
await fs.mkdir(manga, { recursive: true });
await fs.mkdir(dataRoot, { recursive: true });
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({
title: 'Ongoing Manga', status: 'ongoing',
sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }],
'x-scraper': { ids: { alpha: '42' } },
}));
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
}));
const alpha = {
id: 'alpha', label: 'Alpha',
match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: url.pathname.split('/').pop(), title: 'Alpha', url: value } : null; },
async discover() { return { chapters: [] }; }, async run() { return null; },
};
const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; };
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init();
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
const first = await service.enqueueDiscoveryChecks('test');
const second = await service.enqueueDiscoveryChecks('test');
assert.deepEqual(first, { queued: 1, skipped: 0 });
assert.deepEqual(second, { queued: 0, skipped: 1 });
assert.equal(service.state.queue[0].kind, 'discovery');
assert.equal(service.state.queue[0].libraryMangaId, 'Ongoing Manga');
});
test('discovery promotes only chapters newer than the library frontier', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-run-'));
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
const chapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0001 [Alpha]');
const laterChapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0003 [Alpha]');
await fs.mkdir(chapter, { recursive: true });
await fs.mkdir(laterChapter, { recursive: true });
await fs.mkdir(dataRoot, { recursive: true });
await fs.writeFile(path.join(libraryRoot, 'Ongoing Manga', 'manga.json'), JSON.stringify({
title: 'Ongoing Manga', status: 'ongoing', sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }],
'x-scraper': { ids: { alpha: '42' } },
}));
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/one' } }));
await fs.writeFile(path.join(laterChapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/three' } }));
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
}));
let downloadedScope = null;
const alpha = {
id: 'alpha', label: 'Alpha',
match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: '42', title: 'Alpha', url: value } : null; },
async discover() { return { chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }, { id: 'three', number: '3' }, { id: 'four', number: '4' }] }; },
async run(item, options) { downloadedScope = item.scope; await options.onPublished({ chapter: '4' }); return { finalPath: path.join(libraryRoot, 'Ongoing Manga'), publishedCount: 1, selectedCount: 1 }; },
};
const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; };
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init();
const waitFor = async predicate => {
const deadline = Date.now() + 3000;
while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for discovery'); await new Promise(resolveWait => setTimeout(resolveWait, 20)); }
};
try {
await service.enqueueDiscoveryChecks('test');
await service.control('resume');
await waitFor(() => service.state.queue[0]?.status === 'complete');
assert.deepEqual(downloadedScope, { mode: 'ids', ids: ['four'] });
assert.equal(service.state.queue[0].publishedCount, 1);
const summary = (await service.publicState()).updateChecks.find(item => item.mangaId === 'Ongoing Manga' && item.source === 'alpha');
assert.ok(summary.lastCheckedAt && summary.lastNewContentAt && summary.lastDownloadedAt);
assert.equal(summary.lastResult, 'new-content');
} finally {
await service.close();
await fs.rm(temp, { recursive: true, force: true });
}
});
test('source-aware scheduler runs one item per source in parallel', async t => {
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-sources-'));
const dataRoot = path.join(temp, 'data');
await fs.mkdir(dataRoot, { recursive: true });
const queue = [
{ id: 'A1', source: 'alpha', title: 'Alpha One', url: 'https://alpha.test/1', status: 'queued', order: 0 },
{ id: 'A2', source: 'alpha', title: 'Alpha Two', url: 'https://alpha.test/2', status: 'queued', order: 1 },
{ id: 'B1', source: 'beta', title: 'Beta One', url: 'https://beta.test/1', status: 'queued', order: 2 },
];
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
version: 2, paused: true, status: 'paused', settings: validateSettings({}), queue, history: [], publishedRevision: 0,
}));
const resolvers = new Map(), started = [];
const fakeAdapter = id => ({
id, label: id,
match(value) { const url = new URL(value); return url.hostname === `${id}.test` ? { source: id, id: url.pathname.slice(1), title: id, url: value } : null; },
run(item) { started.push(item.id); return new Promise(resolve => resolvers.set(item.id, () => resolve(path.join(temp, item.id)))); },
});
const fakeAdapters = [fakeAdapter('alpha'), fakeAdapter('beta')];
const resolveAdapter = value => {
for (const adapter of fakeAdapters) { const info = adapter.match(value); if (info) return { adapter, info }; }
return null;
};
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot, adapterFor: resolveAdapter, adapters: fakeAdapters }).init();
const waitFor = async predicate => {
const deadline = Date.now() + 3000;
while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for scheduler'); await new Promise(resolve => setTimeout(resolve, 20)); }
};
try {
await service.control('resume');
await waitFor(() => resolvers.has('A1') && resolvers.has('B1'));
assert.deepEqual(new Set(started), new Set(['A1', 'B1']));
assert.equal(resolvers.has('A2'), false);
resolvers.get('A1')();
await waitFor(() => resolvers.has('A2'));
assert.equal(service.state.queue.find(item => item.id === 'A2').status, 'downloading');
resolvers.get('A2')(); resolvers.get('B1')();
await waitFor(() => service.state.queue.every(item => item.status === 'complete'));
assert.equal(service.state.publishedRevision, 3);
} finally {
for (const resolve of resolvers.values()) resolve();
await service.close();
await fs.rm(temp, { recursive: true, force: true });
}
});