'use strict'; const test = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs/promises'); const os = require('node:os'); const path = require('node:path'); const { cleanPart, chapterKey, chapterFolder, assertInside } = require('../lib/names'); const { imageExtension } = require('../lib/images'); const { publishManifest } = require('../lib/publisher'); const { mergeImportedMetadata } = require('../lib/metadata-merge'); const { metadataFromMedia } = require('../metadata/anilist'); const { normalizeScope, selectByScope, scopeLabel } = require('../lib/scope'); const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, setAutoUpdate } = require('../lib/library-inventory'); const { inspectLegacyWork } = require('../lib/legacy-publication'); const { discoverWeebCatalog, parseChapters, parsePageUrls } = require('../adapters/weebcentral'); const { chapterNumber, selectChapterReleases, pageUrls } = require('../adapters/mangadex'); const { adapterFor } = require('../adapters'); const { ScraperService, validateSettings, settingsForSource } = require('../service'); test('MangaCircus chapter keys are normalized', () => { assert.equal(chapterKey('12'), 'c0012'); assert.equal(chapterKey('12.50'), 'c0012.5'); assert.equal(chapterKey('x2'), 'x02'); assert.throws(() => chapterKey('special')); }); test('folder names are portable and versioned', () => { assert.equal(cleanPart('Bad:. '), 'BadTitle'); assert.equal(cleanPart('CON'), 'CON-manga'); assert.equal(chapterFolder({ number: '7', title: 'A [Test]', source: { name: 'Host' } }), 'c0007 A Test [Host]'); assert.throws(() => assertInside('C:/library', 'C:/outside')); }); test('image signatures are recognized', () => { assert.equal(imageExtension(Buffer.from([0xff, 0xd8, 0xff, 0x00])), 'jpg'); assert.equal(imageExtension(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])), 'png'); assert.equal(imageExtension(Buffer.from('not an image')), null); }); test('per-source rate settings default fast and update independently', () => { const defaults = validateSettings({}); const fast = { minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000 }; assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast); assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), fast); const changed = validateSettings({ sourceRates: { mangadex: { minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000, } } }, defaults); assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), { minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000, }); assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast); assert.throws(() => validateSettings({ sourceRates: { mangadex: { minMs: 5000, maxMs: 1000 } } }, defaults)); }); test('AniList metadata keeps manga format and original story credits', () => { const metadata = metadataFromMedia({ id: 150193, idMal: null, format: 'MANGA', title: { english: 'Heavy Knight', romaji: 'Juukishi', native: '重騎士' }, synonyms: [], staff: { edges: [ { role: 'Original Story', node: { name: { full: 'Nekoko' } } }, { role: 'Story & Art', node: { name: { full: 'Lee Brocco' } } }, { role: 'Character Design', node: { name: { full: 'Jaian' } } }, ] }, status: 'RELEASING', startDate: { year: 2022 }, genres: ['Fantasy'], description: '', siteUrl: 'https://anilist.co/manga/150193', coverImage: { extraLarge: 'https://s4.anilist.co/cover.jpg' }, }); assert.deepEqual(metadata.authors, ['Nekoko', 'Lee Brocco']); assert.equal(metadata['x-anilist'].format, 'MANGA'); }); test('metadata refresh preserves legacy and manual fields while updating untouched imports', () => { const first = mergeImportedMetadata({ title: 'My preferred title', status: 'ongoing' }, { title: 'Provider title', status: 'completed', tags: ['Fantasy'], 'x-anilist': { id: 42 }, }, 'Fallback'); assert.equal(first.title, 'My preferred title'); assert.equal(first.status, 'ongoing'); assert.deepEqual(first.tags, ['Fantasy']); assert.equal(first['x-metadata'].imported.title, 'Provider title'); const second = mergeImportedMetadata({ ...first, tags: ['Fantasy', 'Favorite'] }, { title: 'New provider title', status: 'ongoing', tags: ['Action'], 'x-anilist': { id: 42 }, }, 'Fallback'); assert.equal(second.title, 'My preferred title'); assert.equal(second.status, 'ongoing'); assert.deepEqual(second.tags, ['Fantasy', 'Favorite']); const third = mergeImportedMetadata(second, { description: 'First', 'x-anilist': { id: 42 } }, 'Fallback'); assert.equal(mergeImportedMetadata(third, { description: 'Revised', 'x-anilist': { id: 42 } }, 'Fallback').description, 'Revised'); assert.throws(() => mergeImportedMetadata(first, { 'x-anilist': { id: 99 } }, 'Fallback'), /identity conflict/); assert.throws(() => mergeImportedMetadata({ 'x-work-identity': { scheme: 'mangacircus-work-v1', kind: 'origin', value: 'a'.repeat(64) } }, { 'x-work-identity': { scheme: 'mangacircus-work-v1', kind: 'origin', value: 'b'.repeat(64) } }, 'Fallback'), /work identity conflict/); }); test('publisher rejects a conflicting work ID before fetching any pages', async t => { const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-identity-')); t.after(() => fs.rm(root, { recursive: true, force: true })); const manga = path.join(root, 'Existing'); await fs.mkdir(manga); await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Existing', sources: [{ name: 'Host', url: 'https://example.test/series' }], 'x-anilist': { id: 42 } })); let requested = 0; await assert.rejects(publishManifest({ version: 1, series: { title: 'Existing', direction: 'rtl', metadata: { 'x-anilist': { id: 99 } }, sources: [{ name: 'Host', url: 'https://example.test/series' }] }, chapters: [{ number: '1', source: { name: 'Host', url: 'https://example.test/chapter' }, pages: [{ url: 'https://example.test/page.png' }] }], }, { libraryRoot: root, httpClient: { async get() { requested++; throw new Error('unexpected request'); } } }), /identity conflict/); assert.equal(requested, 0); assert.equal((await fs.readdir(manga)).includes('v01'), false); }); test('publisher explains an unresolved same-title folder collision without touching its contents', async t => { const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-title-collision-')); t.after(() => fs.rm(root, { recursive: true, force: true })); const manga = path.join(root, 'Same Title'); await fs.mkdir(manga); await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Same Title' })); await fs.writeFile(path.join(manga, 'operator-note.txt'), 'keep'); let requested = 0; await assert.rejects(publishManifest({ version: 1, series: { title: 'Same Title', direction: 'rtl', sources: [{ name: 'Peer', url: 'peer://p123/m123' }] }, chapters: [{ number: '1', source: { name: 'Peer', url: 'peer://p123/m123' }, pages: [{ url: 'https://example.test/page.png' }] }], }, { libraryRoot: root, httpClient: { async get() { requested++; throw new Error('unexpected request'); } } }), /could not be safely matched.*No files were overwritten.*Choose the existing manga/s); assert.equal(requested, 0); assert.equal(await fs.readFile(path.join(manga, 'operator-note.txt'), 'utf8'), 'keep'); }); test('queue scopes select exact catalog slices', () => { const chapters = [ { id: 'a', number: '1', volume: '1' }, { id: 'b', number: '2', volume: '1' }, { id: 'c', number: '3', volume: '2' }, { id: 'd', number: '4', volume: '2' }, ]; assert.deepEqual(selectByScope(chapters, { mode: 'latest' }).map(x => x.id), ['d']); assert.deepEqual(selectByScope(chapters, { mode: 'last', count: 2 }).map(x => x.id), ['c', 'd']); assert.deepEqual(selectByScope(chapters, { mode: 'first', count: 2 }).map(x => x.id), ['a', 'b']); assert.deepEqual(selectByScope(chapters, { mode: 'chapter', chapter: '003.0' }).map(x => x.id), ['c']); assert.deepEqual(selectByScope(chapters, { mode: 'after', chapter: '2' }).map(x => x.id), ['c', 'd']); assert.deepEqual(selectByScope(chapters, { mode: 'volume', volume: '2' }).map(x => x.id), ['c', 'd']); assert.deepEqual(selectByScope(chapters, { mode: 'ids', ids: ['b', 'd'] }).map(x => x.id), ['b', 'd']); assert.equal(scopeLabel(normalizeScope({ mode: 'last', count: 5 })), 'last 5 chapters'); }); test('library inventory distinguishes missing chapters from alternate source versions', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-inventory-')); t.after(() => fs.rm(temp, { recursive: true, force: true })); const manga = path.join(temp, 'Library Manga'), book = path.join(manga, 'v01'); await fs.mkdir(path.join(book, 'c0001 [WeebCentral]'), { recursive: true }); await fs.mkdir(path.join(book, 'c0002 [MangaDex]'), { recursive: true }); await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Library Manga', status: 'ongoing', sources: [{ name: 'WeebCentral', url: 'https://weebcentral.com/series/SERIES/Test' }], 'x-scraper': { ids: { weebcentral: 'SERIES', anilist: 42 } }, })); await fs.writeFile(path.join(book, 'c0001 [WeebCentral]', 'chapter.json'), JSON.stringify({ source: { name: 'WeebCentral', url: 'https://weebcentral.com/chapters/A' } })); await fs.writeFile(path.join(book, 'c0002 [MangaDex]', 'chapter.json'), JSON.stringify({ source: { name: 'MangaDex · Group', url: 'https://mangadex.org/chapter/B' } })); const inventory = await scanLibrary(temp); const found = findManga(inventory, { source: 'weebcentral', seriesId: 'SERIES' }); assert.equal(found.id, 'Library Manga'); assert.equal(findManga(inventory, { source: 'mangadex', seriesId: 'other', anilist: 42 }).id, 'Library Manga'); assert.equal(found.autoUpdate, true); assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'skip-any'), false); assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'add-source'), true); assert.equal(shouldDownload(found, { number: '1' }, 'weebcentral', 'add-source'), false); assert.equal(shouldDownload(found, { number: '3' }, 'weebcentral', 'skip-any'), true); assert.equal(isNewerThanLibrary(found, { number: '1' }), false); assert.equal(isNewerThanLibrary(found, { number: '3' }), true); await setAutoUpdate(temp, 'Library Manga', false); assert.equal((await scanLibrary(temp)).mangas[0].autoUpdate, false); }); test('a shared work ID cannot merge distinct source listings or ambiguous editions', () => { const original = { id: 'Main', ids: { weebcentral: 'MAIN', anilist: 42 }, sources: [] }; assert.equal(findManga({ mangas: [original] }, { source: 'weebcentral', seriesId: 'PILOT', anilist: 42 }), null); assert.equal(findManga({ mangas: [original] }, { source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), original); assert.equal(findManga({ mangas: [original, { id: 'Pilot', ids: { anilist: 42 }, sources: [] }] }, { source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), null); }); test('recent events read the bounded log tail without losing the newest incident', async t => { const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-events-')); t.after(() => fs.rm(root, { recursive: true, force: true })); const service = new ScraperService({ dataRoot: root }); await fs.writeFile(service.logFile, `${'x'.repeat(600 * 1024)}\n${JSON.stringify({ level: 'error', message: 'recent' })}\n`); assert.deepEqual(await service.recentEvents(1), [{ level: 'error', message: 'recent' }]); assert.equal((await service.recentEvents(10)).at(-1).message, 'recent'); }); test('paused legacy work is validated, published, and converted to incremental mode', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-legacy-publish-')); const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library'); const workRoot = path.join(libraryRoot, '.mcscraper-work-SERIES42'); const stagedManga = path.join(workRoot, 'Legacy Manga'), book = path.join(stagedManga, 'v01'); await fs.mkdir(dataRoot, { recursive: true }); await fs.mkdir(book, { recursive: true }); await fs.writeFile(path.join(stagedManga, 'manga.json'), JSON.stringify({ title: 'Legacy Manga', status: 'ongoing', sources: [{ name: 'Alpha', url: 'https://alpha.test/title/SERIES42' }], 'x-scraper': { ids: { alpha: 'SERIES42' } }, })); for (const number of [1, 2]) { const chapter = path.join(book, `c000${number} [Alpha]`); await fs.mkdir(chapter, { recursive: true }); await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: `https://alpha.test/chapter/${number}` } })); await fs.writeFile(path.join(chapter, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0])); } const incomplete = path.join(book, '.staging-mcscraper-abc123'); await fs.mkdir(incomplete); await fs.writeFile(path.join(incomplete, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0])); await fs.writeFile(path.join(workRoot, 'job.json'), JSON.stringify({ version: 2, status: 'paused', completed: ['one', 'two'], total: 4, })); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ version: 3, paused: true, status: 'paused', settings: validateSettings({}), history: [], publishedRevision: 0, queue: [{ id: 'old-id', seriesId: 'SERIES42', source: 'alpha', title: 'Legacy Manga', url: 'https://alpha.test/title/SERIES42', status: 'queued', order: 0 }], })); const alpha = { id: 'alpha', label: 'Alpha', match(value) { return String(value).includes('alpha.test/title/') ? { source: 'alpha', id: 'SERIES42', title: 'Legacy Manga', url: value } : null; }, async discover() { return { chapters: [] }; }, async run() { return null; }, }; const service = await new ScraperService({ libraryRoot, dataRoot, adapters: [alpha], adapterFor: value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; }, }).init(); t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); }); const inspection = await inspectLegacyWork({ libraryRoot, seriesId: 'SERIES42' }); assert.equal(inspection.chapterCount, 2); assert.equal(inspection.pageCount, 2); assert.equal(inspection.incompleteStages.length, 1); const result = await service.publishLegacyReady(); assert.equal(result.published.length, 1); assert.equal(result.published[0].chaptersPublished, 2); assert.equal(result.published[0].incompleteStagesQuarantined, 1); assert.equal(await fs.stat(path.join(libraryRoot, 'Legacy Manga', 'v01', 'c0002 [Alpha]')).then(stat => stat.isDirectory()), true); await assert.rejects(fs.stat(stagedManga)); const migrated = JSON.parse(await fs.readFile(path.join(workRoot, 'job.json'), 'utf8')); assert.equal(migrated.publicationMode, 'incremental'); assert.equal(migrated.migratedToIncremental.chaptersPublished, 2); assert.equal(migrated.migratedToIncremental.incompleteStagesQuarantined, 1); assert.equal(await fs.stat(path.join(workRoot, '.legacy-incomplete', 'v01__.staging-mcscraper-abc123')).then(stat => stat.isDirectory()), true); assert.equal(service.state.queue[0].libraryMangaId, 'Legacy Manga'); assert.equal(service.state.publishedRevision, 1); }); test('WeebCentral fragments preserve chapter and page order', () => { const chapters = parseChapters('<a href="/chapters/P1">Prologue 1</a><a href="/chapters/ABC">Chapter 12</a><a href="/chapters/DEF">Chapter 12.5</a><a href="/chapters/S2">S2 - Chapter 1</a><a href="/chapters/R442"><span>Rating 442</span><span>Last Read</span></a><a href="/chapters/E128"><span>Episode 128</span><span>Last Read</span></a><a href="/chapters/E78"><span>Episode: 78</span><span>Last Read</span></a><a href="/chapters/N182"><span>Night 182</span><span>Last Read</span></a><a href="/chapters/L65">Log. 65</a><a href="/chapters/A606">Act 60.6</a><a href="/chapters/C155">Scene 155</a><a href="/chapters/K162">Kapitel 162</a><a href="/chapters/F83">File 83</a><a href="/chapters/B281">Battle 281</a>', 'https://weebcentral.com/s'); assert.deepEqual(chapters.map(item => item.number), ['0.01', '12', '12.5', '1', '442', '128', '78', '182', '65', '60.6', '155', '162', '83', '281']); assert.deepEqual(chapters[3].book, { number: 2, title: 'Season 2' }); const pages = parsePageUrls('<img src="/001.png"><img data-src="https://img.example/002.webp">', 'https://weebcentral.com/c'); assert.deepEqual(pages, ['https://weebcentral.com/001.png', 'https://img.example/002.webp']); }); test('WeebCentral reports unrecognized chapter labels separately from an empty catalog', async () => { const series = '<h1>Example</h1><div hx-get="/series/X/full-chapter-list"></div>'; const requests = []; const httpFor = list => ({ get: async (url, options) => { requests.push({ url, options }); return { text: async () => url.endsWith('full-chapter-list') ? list : series, }; } }); const unknown = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<a href="/chapters/ABC">Part One</a>')); assert.equal(requests[0].options.catalog, undefined); assert.equal(requests[1].options.catalog, true); assert.equal(unknown.chapterLinks, 1); assert.equal(unknown.chapters.length, 0); assert.equal(unknown.availability.reason, 'unrecognized-labels'); assert.match(unknown.availability.message, /1 chapter links/); const empty = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<div>No chapters</div>')); assert.equal(empty.chapterLinks, 0); assert.equal(empty.availability, undefined); }); test('completed download reinspection preserves complete state when all scoped chapters are present', async () => { const service = new ScraperService({ libraryRoot: 'unused', dataRoot: 'unused' }); const item = { id: 'done', title: 'Done', source: 'weebcentral', url: 'https://weebcentral.com/series/EXAMPLE/Done', seriesId: 'EXAMPLE', kind: 'inspection', status: 'queued', completed: '2026-10-01T00:00:00Z', publishedCount: 2, selectedCount: 2, scope: { mode: 'missing' }, inspectionFromComplete: true }; service.state = { paused: false, settings: validateSettings({}), queue: [item], incidents: [], sourcePaused: {}, currentIds: {} }; service.recordCatalog = async (entry, catalog) => { entry.sourceTitle = catalog.title; entry.catalogTotal = catalog.chapters.length; }; service.libraryInventory = async () => ({ mangas: [{ id: 'Done', ids: { weebcentral: 'EXAMPLE' }, sources: [], chapters: [{ book: '1', number: '1' }, { book: '1', number: '2' }] }] }); service.event = async () => {}; const adapter = { discover: async () => ({ title: 'Done', chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }] }) }; await service.runDiscovery(item, { adapter }, 'weebcentral'); assert.equal(item.status, 'complete'); assert.equal(item.kind, 'download'); assert.equal(item.completed, '2026-10-01T00:00:00Z'); assert.equal(item.publishedCount, 2); assert.equal(item.inspectionFromComplete, undefined); }); test('publisher keeps season chapter numbers in separate books', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-books-')); t.after(() => fs.rm(temp, { recursive: true, force: true })); const page = path.join(temp, 'page.png'); await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0])); const base = { title: '', source: { name: 'Host', url: 'https://example.test/chapter' }, pages: [{ path: page }] }; await publishManifest({ version: 1, series: { title: 'Season Test', direction: 'ttb', sources: [{ name: 'Host', url: 'https://example.test/series' }] }, chapters: [ { ...base, number: '1', book: { number: 1, title: 'Season 1' } }, { ...base, number: '1', book: { number: 2, title: 'Season 2' } }, ], }, { libraryRoot: path.join(temp, 'library') }); assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v01 Season 1', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 1'); assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v02 Season 2', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 2'); }); test('publisher stages and atomically exposes a complete new manga', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-test-')); t.after(() => fs.rm(temp, { recursive: true, force: true })); const page = path.join(temp, 'page.png'); await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0])); const result = await publishManifest({ version: 1, series: { title: 'Test Manga', direction: 'rtl', metadata: { status: 'ongoing' }, sources: [{ name: 'Test', url: 'https://example.test/manga' }] }, chapters: [{ number: '1', title: 'Start', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }], }, { libraryRoot: path.join(temp, 'library') }); assert.equal(result.results[0].status, 'published'); const chapter = path.join(temp, 'library', 'Test Manga', 'v01', 'c0001 Start [Test]'); assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', 'chapter.json']); const meta = JSON.parse(await fs.readFile(path.join(temp, 'library', 'Test Manga', 'manga.json'), 'utf8')); assert.equal(meta['x-scraper'].autoUpdate, undefined, 'ongoing is a derived default, not a frozen override'); assert.equal((await scanLibrary(path.join(temp, 'library'))).mangas[0].autoUpdate, true); assert.equal((await fs.readdir(path.join(temp, 'library'))).some(name => name.startsWith('.staging-')), false); }); test('publisher installs a validated AniList series cover', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-cover-')); t.after(() => fs.rm(temp, { recursive: true, force: true })); const page = path.join(temp, 'page.png'); const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]); await fs.writeFile(page, png); let requests = 0; const httpClient = { async get(url) { requests++; assert.equal(url, 'https://s4.anilist.co/example.png'); return { arrayBuffer: async () => png }; }, }; const series = { title: 'Covered Manga', direction: 'rtl', sources: [{ name: 'Test', url: 'https://example.test/covered' }], coverUrl: 'https://s4.anilist.co/example.png', }; const result = await publishManifest({ version: 1, series, chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }], }, { libraryRoot: path.join(temp, 'library'), httpClient }); assert.equal(result.cover.status, 'installed'); assert.deepEqual(await fs.readFile(path.join(temp, 'library', 'Covered Manga', 'cover.png')), png); assert.equal(requests, 1); }); test('publisher retains validated partial pages and resumes without duplicate requests', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-resume-')); t.after(() => fs.rm(temp, { recursive: true, force: true })); const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]); const manifest = { version: 1, series: { title: 'Resumable', direction: 'rtl', sources: [{ name: 'Test', url: 'https://example.test/series' }] }, chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [1, 2].map(n => ({ url: `https://example.test/${n}.png` })) }], }; const requests = []; const httpClient = { async get(url) { requests.push(url); return { arrayBuffer: async () => png }; } }; const libraryRoot = path.join(temp, 'library'); await assert.rejects(publishManifest(manifest, { libraryRoot, httpClient, onPage() { throw new Error('cooperative pause'); } }), /cooperative pause/); await assert.rejects(fs.stat(path.join(libraryRoot, 'Resumable'))); const hidden = (await fs.readdir(libraryRoot))[0]; assert.match(hidden, /^\.staging-mcscraper-resume-/); const { cleanupOwnTemps } = require('../lib/publisher'); await cleanupOwnTemps(libraryRoot, -1); assert.ok(await fs.stat(path.join(libraryRoot, hidden))); await publishManifest(manifest, { libraryRoot, httpClient }); assert.deepEqual(requests, ['https://example.test/1.png', 'https://example.test/2.png']); const chapter = path.join(libraryRoot, 'Resumable', 'v01', 'c0001 [Test]'); assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', '0002.png', 'chapter.json']); }); test('adapter registry recognizes supported series URLs', () => { const found = adapterFor('https://weebcentral.com/series/01ABCDEF/My-Series'); assert.equal(found.adapter.id, 'weebcentral'); assert.equal(found.info.id, '01ABCDEF'); assert.equal(found.info.title, 'My Series'); const dex = adapterFor('https://mangadex.org/title/5bdb093f-9707-4a30-b02e-c8b66554ecf0/biomega'); assert.equal(dex.adapter.id, 'mangadex'); assert.equal(dex.info.id, '5bdb093f-9707-4a30-b02e-c8b66554ecf0'); assert.equal(adapterFor('https://example.test/series/1'), null); }); test('MangaDex releases are normalized, deduplicated, credited, and host-validated', () => { assert.equal(chapterNumber('0012.50'), '12.5'); assert.equal(chapterNumber('Special'), null); const release = (id, chapter, group, publishAt) => ({ id, attributes: { chapter, title: '', pages: 2, publishAt, externalUrl: null }, relationships: group ? [{ id: `group-${id}`, type: 'scanlation_group', attributes: { name: group } }] : [], }); const selected = selectChapterReleases([ release('fan', '1', 'Fan Group', '2020-01-01T00:00:00Z'), release('official', '1.0', 'Publisher Official', '2021-01-01T00:00:00Z'), release('two', '2', 'Fan Group', '2020-01-02T00:00:00Z'), ]); assert.deepEqual(selected.map(item => item.id), ['official', 'two']); assert.equal(selected[0].group.name, 'Publisher Official'); const pages = pageUrls({ baseUrl: 'https://node.mangadex.network', chapter: { hash: 'abc', data: ['1.jpg', '2.png'] }, }, 'https://mangadex.org/chapter/test', 2); assert.deepEqual(pages.map(item => item.url), [ 'https://node.mangadex.network/data/abc/1.jpg', 'https://node.mangadex.network/data/abc/2.png', ]); assert.throws(() => pageUrls({ baseUrl: 'https://evil.example', chapter: { hash: 'abc', data: ['1.jpg'] } }, 'https://mangadex.org/chapter/test', 1)); }); test('daemon queue state is durable while paused', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-service-')); const dataRoot = path.join(temp, 'data'); await fs.mkdir(dataRoot, { recursive: true }); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ version: 1, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0, })); const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init(); t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); }); // WeebCentral is off on a fresh installation until an admin switches it on. await assert.rejects(() => service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series' }), /switched off/); await service.updateSettings({ sourcesEnabled: { weebcentral: true } }); const item = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series' }); assert.equal(item.status, 'queued'); const scoped = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series', scope: { mode: 'latest' } }); assert.notEqual(item.id, scoped.id); assert.equal(item.seriesId, scoped.seriesId); assert.equal((await service.publicState()).queue.length, 2); await service.updateSettings({ minMs: 1500, maxMs: 4500 }); assert.equal(JSON.parse(await fs.readFile(path.join(dataRoot, 'queue.json'), 'utf8')).settings.minMs, 1500); await service.remove(item.id); await service.remove(scoped.id); assert.equal((await service.publicState()).queue.length, 0); }); test('sources are off by default but existing installations keep the ones they use', async t => { assert.deepEqual(validateSettings({}).sourcesEnabled, { weebcentral: false, mangadex: true, mangapill: false }); assert.throws(() => validateSettings({ sourcesEnabled: { weebcentral: 'yes' } }), /true or false/); assert.throws(() => validateSettings({ sourcesEnabled: { other: true } }), /Unknown source/); const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-sources-')); const dataRoot = path.join(temp, 'data'); await fs.mkdir(dataRoot, { recursive: true }); const { sourcesEnabled, ...older } = validateSettings({}); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ version: 3, paused: true, status: 'paused', settings: older, publishedRevision: 0, queue: [{ id: 'A', title: 'A', url: 'https://weebcentral.com/series/AAA/A', source: 'weebcentral', status: 'queued', order: 0 }], history: [], })); const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init(); t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); }); assert.deepEqual(service.state.settings.sourcesEnabled, { weebcentral: true, mangadex: true, mangapill: false }); await assert.rejects(() => service.enqueue({ url: 'https://mangapill.com/manga/2/one-piece' }), /MangaPill is switched off/); // Switching a source off keeps its jobs queued instead of failing them. await service.updateSettings({ sourcesEnabled: { weebcentral: false } }); assert.equal(service.sourceEnabled('weebcentral'), false); assert.equal(service.sourceEnabled('peer-abc'), true, 'peers have no switch'); assert.equal(service.state.queue[0].status, 'queued'); }); test('retry recovers stale downloading entries before work resumes', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-retry-')); const dataRoot = path.join(temp, 'data'); await fs.mkdir(dataRoot, { recursive: true }); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ version: 1, paused: true, status: 'error', currentId: 'FAILED', settings: validateSettings({}), queue: [ { id: 'STALE', title: 'Stale', url: 'https://weebcentral.com/series/STALE/Stale', status: 'downloading', order: 0 }, { id: 'FAILED', title: 'Failed', url: 'https://weebcentral.com/series/FAILED/Failed', status: 'error', order: 1 }, ], history: [], publishedRevision: 0, })); const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init(); t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); }); await service.control('retry', 'FAILED'); assert.deepEqual(service.state.queue.map(item => item.status), ['queued', 'queued']); assert.equal(service.state.currentId, null); assert.equal(service.state.paused, false); }); test('nightly discovery queues one preferred supported source for ongoing manga', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-')); const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library'); const manga = path.join(libraryRoot, 'Ongoing Manga'); await fs.mkdir(manga, { recursive: true }); await fs.mkdir(dataRoot, { recursive: true }); await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Ongoing Manga', status: 'ongoing', sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }], 'x-scraper': { ids: { alpha: '42' } }, })); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0, })); const alpha = { id: 'alpha', label: 'Alpha', match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: url.pathname.split('/').pop(), title: 'Alpha', url: value } : null; }, async discover() { return { chapters: [] }; }, async run() { return null; }, }; const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; }; const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init(); t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); }); const first = await service.enqueueDiscoveryChecks('test'); const second = await service.enqueueDiscoveryChecks('test'); assert.deepEqual(first, { queued: 1, skipped: 0 }); assert.deepEqual(second, { queued: 0, skipped: 1 }); assert.equal(service.state.queue[0].kind, 'discovery'); assert.equal(service.state.queue[0].libraryMangaId, 'Ongoing Manga'); }); test('discovery promotes only chapters newer than the library frontier', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-run-')); const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library'); const chapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0001 [Alpha]'); const laterChapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0003 [Alpha]'); await fs.mkdir(chapter, { recursive: true }); await fs.mkdir(laterChapter, { recursive: true }); await fs.mkdir(dataRoot, { recursive: true }); await fs.writeFile(path.join(libraryRoot, 'Ongoing Manga', 'manga.json'), JSON.stringify({ title: 'Ongoing Manga', status: 'ongoing', sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }], 'x-scraper': { ids: { alpha: '42' } }, })); await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/one' } })); await fs.writeFile(path.join(laterChapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/three' } })); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0, })); let downloadedScope = null; const alpha = { id: 'alpha', label: 'Alpha', match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: '42', title: 'Alpha', url: value } : null; }, async discover() { return { chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }, { id: 'three', number: '3' }, { id: 'four', number: '4' }] }; }, async run(item, options) { downloadedScope = item.scope; await options.onPublished({ chapter: '4' }); return { finalPath: path.join(libraryRoot, 'Ongoing Manga'), publishedCount: 1, selectedCount: 1 }; }, }; const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; }; const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init(); const waitFor = async predicate => { const deadline = Date.now() + 3000; while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for discovery'); await new Promise(resolveWait => setTimeout(resolveWait, 20)); } }; try { await service.enqueueDiscoveryChecks('test'); await service.control('resume'); await waitFor(() => service.state.queue[0]?.status === 'complete'); assert.deepEqual(downloadedScope, { mode: 'ids', ids: ['four'] }); assert.equal(service.state.queue[0].publishedCount, 1); const summary = (await service.publicState()).updateChecks.find(item => item.mangaId === 'Ongoing Manga' && item.source === 'alpha'); assert.ok(summary.lastCheckedAt && summary.lastNewContentAt && summary.lastDownloadedAt); assert.equal(summary.lastResult, 'new-content'); } finally { await service.close(); await fs.rm(temp, { recursive: true, force: true }); } }); test('source-aware scheduler runs one item per source in parallel', async t => { const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-sources-')); const dataRoot = path.join(temp, 'data'); await fs.mkdir(dataRoot, { recursive: true }); const queue = [ { id: 'A1', source: 'alpha', title: 'Alpha One', url: 'https://alpha.test/1', status: 'queued', order: 0 }, { id: 'A2', source: 'alpha', title: 'Alpha Two', url: 'https://alpha.test/2', status: 'queued', order: 1 }, { id: 'B1', source: 'beta', title: 'Beta One', url: 'https://beta.test/1', status: 'queued', order: 2 }, ]; await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ version: 2, paused: true, status: 'paused', settings: validateSettings({}), queue, history: [], publishedRevision: 0, })); const resolvers = new Map(), started = []; const fakeAdapter = id => ({ id, label: id, match(value) { const url = new URL(value); return url.hostname === `${id}.test` ? { source: id, id: url.pathname.slice(1), title: id, url: value } : null; }, run(item) { started.push(item.id); return new Promise(resolve => resolvers.set(item.id, () => resolve(path.join(temp, item.id)))); }, }); const fakeAdapters = [fakeAdapter('alpha'), fakeAdapter('beta')]; const resolveAdapter = value => { for (const adapter of fakeAdapters) { const info = adapter.match(value); if (info) return { adapter, info }; } return null; }; const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot, adapterFor: resolveAdapter, adapters: fakeAdapters }).init(); const waitFor = async predicate => { const deadline = Date.now() + 3000; while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for scheduler'); await new Promise(resolve => setTimeout(resolve, 20)); } }; try { await service.control('resume'); await waitFor(() => resolvers.has('A1') && resolvers.has('B1')); assert.deepEqual(new Set(started), new Set(['A1', 'B1'])); assert.equal(resolvers.has('A2'), false); resolvers.get('A1')(); await waitFor(() => resolvers.has('A2')); assert.equal(service.state.queue.find(item => item.id === 'A2').status, 'downloading'); resolvers.get('A2')(); resolvers.get('B1')(); await waitFor(() => service.state.queue.every(item => item.status === 'complete')); assert.equal(service.state.publishedRevision, 3); } finally { for (const resolve of resolvers.values()) resolve(); await service.close(); await fs.rm(temp, { recursive: true, force: true }); } });