Consolidate MangaCircus app, scraper, and Compose baseline
This commit is contained in:
Maashkinoozhe committed
1 parent
cfb2d79103
commit
eef4ffa07c
103 files changed
+9316
-172
No files matched your search
@@ -0,0 +1,71 @@
|
||||
'use strict';
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs').promises;
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { ScraperService } = require('../service');
|
||||
|
||||
const adapter = { id: 'alpha', label: 'Alpha',
|
||||
match(value) { return value.startsWith('https://alpha.test/') ? { id: '42', source: 'alpha', title: 'Test', url: value } : null; },
|
||||
async discover() { return { title: 'Test', chapters: [{ id: 'one', number: '1' }] }; },
|
||||
async run() { throw new Error('A no-change check must not download'); },
|
||||
};
|
||||
const resolve = value => { const info = adapter.match(value); return info ? { adapter, info } : null; };
|
||||
const waitFor = async predicate => {
|
||||
const deadline = Date.now() + 3000;
|
||||
while (!predicate()) {
|
||||
if (Date.now() > deadline) throw new Error('Timed out waiting for local discovery');
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 25));
|
||||
}
|
||||
};
|
||||
|
||||
test('finished-job cleanup preserves unfinished jobs, legacy check summaries, history and library files', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-scraper-dashboard-'));
|
||||
const dataRoot = path.join(root, 'data'), libraryRoot = path.join(root, 'library');
|
||||
await fs.mkdir(dataRoot); await fs.mkdir(path.join(libraryRoot, 'Test'), { recursive: true });
|
||||
const chapterFile = path.join(libraryRoot, 'Test', 'keep.jpg');
|
||||
await fs.writeFile(chapterFile, 'library-bytes');
|
||||
const queue = [
|
||||
{ id: 'done-a', kind: 'download', source: 'alpha', url: 'https://alpha.test/42', status: 'complete' },
|
||||
{ id: 'checked', kind: 'discovery', source: 'alpha', url: 'https://alpha.test/42', status: 'complete', libraryMangaId: 'Test', checked: '2026-10-06T12:00:00Z' },
|
||||
{ id: 'ready', kind: 'inspection', source: 'alpha', url: 'https://alpha.test/42', status: 'ready' },
|
||||
{ id: 'failed', kind: 'download', source: 'alpha', url: 'https://alpha.test/42', status: 'quarantined' },
|
||||
{ id: 'queued', kind: 'download', source: 'alpha', url: 'https://alpha.test/42', status: 'queued' },
|
||||
{ id: 'done-b', kind: 'download', source: 'beta', url: 'https://beta.test/42', status: 'complete' },
|
||||
];
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ paused: true, queue, history: [{ id: 'done-a', status: 'complete' }] }));
|
||||
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [adapter, { id: 'beta', label: 'Beta' }] }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
assert.deepEqual(service.state.queue.map(item => item.id), ['done-a', 'ready', 'failed', 'queued', 'done-b']);
|
||||
assert.equal((await service.publicState()).updateChecks[0].lastCheckedAt, '2026-10-06T12:00:00Z');
|
||||
assert.equal((await service.clearCompleted('alpha')).cleared, 1);
|
||||
assert.deepEqual(service.state.queue.map(item => item.id), ['ready', 'failed', 'queued', 'done-b']);
|
||||
assert.equal((await service.clearCompleted()).cleared, 1);
|
||||
assert.equal((await service.clearCompleted()).cleared, 0);
|
||||
assert.deepEqual(service.state.queue.map(item => item.id), ['ready', 'failed', 'queued']);
|
||||
assert.deepEqual(service.state.queue.map(item => item.order), [0, 1, 2]);
|
||||
assert.equal((await fs.readFile(chapterFile, 'utf8')), 'library-bytes');
|
||||
assert.equal(service.state.history[0].id, 'done-a');
|
||||
assert.equal(JSON.parse(await fs.readFile(path.join(dataRoot, 'queue.json'), 'utf8')).updateChecks['["Test","alpha"]'].lastCheckedAt, '2026-10-06T12:00:00Z');
|
||||
});
|
||||
|
||||
test('a nightly no-change check updates the latest timestamp and retires its queue row', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-scraper-check-summary-'));
|
||||
const dataRoot = path.join(root, 'data'), libraryRoot = path.join(root, 'library');
|
||||
const chapter = path.join(libraryRoot, 'Test', 'v01', 'c0001 [Alpha]');
|
||||
await fs.mkdir(dataRoot); await fs.mkdir(chapter, { recursive: true });
|
||||
await fs.writeFile(path.join(libraryRoot, 'Test', 'manga.json'), JSON.stringify({ title: 'Test', status: 'ongoing', sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }] }));
|
||||
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/one' } }));
|
||||
await fs.writeFile(path.join(chapter, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [adapter] }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
await service.enqueueDiscoveryChecks('schedule');
|
||||
await waitFor(() => service.state.updateChecks['["Test","alpha"]']?.lastCheckedAt);
|
||||
assert.equal(service.state.queue.length, 0);
|
||||
const summary = (await service.publicState()).updateChecks[0];
|
||||
assert.equal(summary.lastResult, 'no-updates');
|
||||
assert.equal(summary.lastNewContentAt, undefined);
|
||||
assert.equal(summary.lastDownloadedAt, undefined);
|
||||
assert.ok(service.state.history.some(item => item.status === 'checked'));
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
'use strict';
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { ScraperService, validateSettings } = require('../service');
|
||||
|
||||
function fixture(scanLibrary) {
|
||||
const service = new ScraperService({ scanLibrary, adapters: [] });
|
||||
service.state = { version: 4, paused: true, status: 'paused', queue: [], settings: validateSettings({}), currentIds: {} };
|
||||
service.recentEvents = async () => [];
|
||||
return service;
|
||||
}
|
||||
const inventory = title => ({ mangas: title ? [{ id: title, title, status: 'ongoing', autoUpdate: true, chapters: [], sources: [] }] : [] });
|
||||
|
||||
test('cold status returns controls immediately and simultaneous probes share one slow scan', { timeout: 1000 }, async () => {
|
||||
let finish, scans = 0;
|
||||
const service = fixture(() => { scans++; return new Promise(resolve => { finish = resolve; }); });
|
||||
const states = await Promise.all(Array.from({ length: 8 }, () => service.publicState()));
|
||||
assert.equal(scans, 1);
|
||||
for (const state of states) {
|
||||
assert.equal(state.paused, true);
|
||||
assert.equal(state.settings.sourceRates.mangadex.minMs, 1000);
|
||||
assert.deepEqual(state.library, []);
|
||||
assert.equal(state.libraryStatus.ready, false);
|
||||
assert.equal(state.libraryStatus.refreshing, true);
|
||||
}
|
||||
finish(inventory('Demo'));
|
||||
await service.inventoryRefresh;
|
||||
const ready = await service.publicState();
|
||||
assert.equal(ready.library[0].title, 'Demo');
|
||||
assert.equal(ready.libraryStatus.ready, true);
|
||||
assert.equal(ready.libraryStatus.stale, false);
|
||||
assert.equal(scans, 1);
|
||||
});
|
||||
|
||||
test('status keeps previous summary but authoritative callers await scans and reject invalidated results', { timeout: 1000 }, async () => {
|
||||
const completions = [];
|
||||
const service = fixture(() => new Promise(resolve => completions.push(resolve)));
|
||||
const first = service.libraryInventory(true);
|
||||
await Promise.resolve();
|
||||
completions[0](inventory('Old'));
|
||||
await first;
|
||||
service.invalidateInventory();
|
||||
const fresh = service.libraryInventory(true);
|
||||
await Promise.resolve();
|
||||
let finished = false;
|
||||
fresh.then(() => { finished = true; });
|
||||
let state = await service.publicState();
|
||||
assert.equal(state.library[0].title, 'Old');
|
||||
assert.equal(state.libraryStatus.stale, true);
|
||||
assert.equal(state.libraryStatus.refreshing, true);
|
||||
assert.equal(finished, false);
|
||||
// Publication during an in-flight scan requires another authoritative scan.
|
||||
service.invalidateInventory();
|
||||
const scan = service.inventoryRefresh;
|
||||
completions[1](inventory('Obsolete'));
|
||||
await scan;
|
||||
await Promise.resolve();
|
||||
assert.equal(completions.length, 3);
|
||||
assert.equal(finished, false);
|
||||
completions[2](inventory('Published'));
|
||||
assert.equal((await fresh).mangas[0].title, 'Published');
|
||||
state = await service.publicState();
|
||||
assert.equal(state.library[0].title, 'Published');
|
||||
assert.equal(state.libraryStatus.stale, false);
|
||||
});
|
||||
|
||||
test('background inventory failures are visible without breaking controls or launching a poll storm', { timeout: 1000 }, async () => {
|
||||
let scans = 0;
|
||||
const service = fixture(async () => { scans++; throw new Error('Scan failed https://test.example/a?token=private'); });
|
||||
await service.publicState();
|
||||
await assert.rejects(service.inventoryRefresh, /Scan failed/);
|
||||
for (let n = 0; n < 3; n++) {
|
||||
const state = await service.publicState();
|
||||
assert.equal(state.paused, true);
|
||||
assert.match(state.libraryStatus.error, /Scan failed/);
|
||||
assert.ok(!state.libraryStatus.error.includes('private'));
|
||||
}
|
||||
assert.equal(scans, 1);
|
||||
await assert.rejects(service.libraryInventory(true), /Scan failed/);
|
||||
assert.equal(scans, 2);
|
||||
});
|
||||
|
||||
test('dashboard polling reuses a recent snapshot while decisions still force a fresh scan', async () => {
|
||||
let scans = 0;
|
||||
const service = fixture(async () => { scans++; return inventory('Demo'); });
|
||||
await service.libraryInventory(true);
|
||||
service.inventoryCachedAt = Date.now() - 30000;
|
||||
for (let n = 0; n < 5; n++) assert.equal((await service.publicState()).library[0].title, 'Demo');
|
||||
assert.equal(scans, 1);
|
||||
await service.libraryInventory(true);
|
||||
assert.equal(scans, 2);
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
'use strict';
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { writeJsonAtomic } = require('../lib/json-file');
|
||||
|
||||
test('Windows rename fallback replaces JSON without deleting the old state first', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-json-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const file = path.join(root, 'queue.json');
|
||||
await fs.writeFile(file, '{"old":true}\n');
|
||||
let failedOnce = false;
|
||||
const io = { ...fs, async rename(from, to) {
|
||||
if (to === file && !failedOnce) {
|
||||
failedOnce = true;
|
||||
const error = new Error('destination busy'); error.code = 'EEXIST'; throw error;
|
||||
}
|
||||
return fs.rename(from, to);
|
||||
} };
|
||||
await writeJsonAtomic(file, { next: true }, io);
|
||||
assert.deepEqual(JSON.parse(await fs.readFile(file, 'utf8')), { next: true });
|
||||
assert.deepEqual(await fs.readdir(root), ['queue.json']);
|
||||
});
|
||||
|
||||
test('failed replacement restores the old JSON', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-json-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const file = path.join(root, 'queue.json');
|
||||
await fs.writeFile(file, '{"old":true}\n');
|
||||
const io = { ...fs, async rename(from, to) {
|
||||
if (to === file && from.endsWith('.tmp')) {
|
||||
const error = new Error('replacement blocked'); error.code = 'EEXIST'; throw error;
|
||||
}
|
||||
return fs.rename(from, to);
|
||||
} };
|
||||
await assert.rejects(writeJsonAtomic(file, { next: true }, io), /replacement blocked/);
|
||||
assert.deepEqual(JSON.parse(await fs.readFile(file, 'utf8')), { old: true });
|
||||
assert.deepEqual(await fs.readdir(root), ['queue.json']);
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises'), os = require('node:os'), path = require('node:path');
|
||||
const { specialKey, parseSpecialFolder } = require('../../chapter-identity');
|
||||
const { selectChapterReleases, availabilitySummary, discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex');
|
||||
const { selectByScope } = require('../lib/scope');
|
||||
const inventory = require('../lib/library-inventory');
|
||||
const { publishManifest } = require('../lib/publisher');
|
||||
const reader = require('../../library');
|
||||
const { ScraperService } = require('../service');
|
||||
const ids = ['37d6949a-c7ce-4855-b838-4fd572e87629', '27dd738c-eca2-488d-a791-53ab4b24313b'];
|
||||
const titleId = '06a37094-b3ba-4f25-bf2b-54c9751e0e91';
|
||||
const release = (id, chapter = null) => ({ id, attributes: { chapter, pages: 46, title: 'Pilot', externalUrl: null }, relationships: [] });
|
||||
const pause = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
async function until(predicate) {
|
||||
const end = Date.now() + 5000;
|
||||
while (!predicate()) { if (Date.now() > end) throw new Error('Fixture timed out'); await pause(20); }
|
||||
}
|
||||
|
||||
test('unnumbered releases retain source IDs, do not invent chapter zero, and remain distinct', async () => {
|
||||
const selected = selectChapterReleases(ids.map(id => release(id)), 'Pilot');
|
||||
assert.equal(selected.length, 2);
|
||||
assert.deepEqual(new Set(selected.map(r => r.number)), new Set(ids.map(specialKey)));
|
||||
assert.ok(selected.every(r => r.special && r.label === 'Pilot' && r.sourceNumber === null));
|
||||
assert.equal(selectByScope(selected, { mode: 'chapter', chapter: '0' }).length, 0);
|
||||
assert.equal(selectByScope(selected, { mode: 'after', chapter: '1' }).length, 0);
|
||||
assert.equal(selectByScope(selected, { mode: 'missing' }).length, 2);
|
||||
assert.equal(selectByScope(selected, { mode: 'ids', ids: [ids[0]] })[0].id, ids[0]);
|
||||
assert.throws(() => specialKey('../escape'));
|
||||
assert.equal(parseSpecialFolder(specialKey(ids[0]) + ' Pilot [MangaDex]').src, 'MangaDex');
|
||||
const chapter = await discoverMangaDexChapter(selected[0], { async get() { return { async json() { return { result: 'ok', baseUrl: 'https://uploads.mangadex.org', chapter: { hash: 'fixture', data: Array.from({ length: 46 }, (_, n) => `${n}.png`) } }; } }; } });
|
||||
assert.equal(chapter.label, 'Pilot'); assert.equal(chapter.sourceId, selected[0].id);
|
||||
});
|
||||
|
||||
test('availability distinguishes no English, external-only, and partial hosted catalogs', () => {
|
||||
assert.equal(availabilitySummary({ availableTranslatedLanguages: ['ar'] }, [], [], 0).reason, 'no-english');
|
||||
const external = availabilitySummary({}, [], [], 8);
|
||||
assert.equal(external.reason, 'external-only'); assert.match(external.message, /8 English external-link/);
|
||||
const chapters = selectChapterReleases([release('regular-fixture', '336')]);
|
||||
const summary = availabilitySummary({ lastChapter: '686' }, [release('regular-fixture', '336')], chapters);
|
||||
assert.equal(summary.coverage, 'partial'); assert.equal(summary.selectedChapters, 1);
|
||||
assert.equal(availabilitySummary({ lastChapter: '336' }, [], chapters).coverage, 'unknown', 'Highest number alone does not prove completeness');
|
||||
});
|
||||
|
||||
test('catalog uses separate external-only feed only when hosted selections are empty', async () => {
|
||||
const modes = [];
|
||||
const http = { async get(url) {
|
||||
const parsed = new URL(url), mode = parsed.searchParams.get('includeExternalUrl');
|
||||
if (!mode) return { async json() { return { data: { type: 'manga', attributes: { title: { en: 'CLAYMORE' }, lastChapter: '155' } } }; } };
|
||||
modes.push(mode);
|
||||
return { async json() { return { result: 'ok', data: [], total: mode === '1' ? 8 : 0 }; } };
|
||||
} };
|
||||
const catalog = await discoverMangaDexCatalog(`https://mangadex.org/title/${titleId}`, http);
|
||||
assert.deepEqual(modes, ['0', '1']); assert.equal(catalog.availability.reason, 'external-only');
|
||||
modes.length = 0;
|
||||
http.get = async url => {
|
||||
const mode = new URL(url).searchParams.get('includeExternalUrl');
|
||||
if (!mode) return { async json() { return { data: { type: 'manga', attributes: { title: { en: 'Bleach (Pre-Serialization)' } } } }; } };
|
||||
modes.push(mode); return { async json() { return { result: 'ok', data: ids.map(id => release(id)), total: 2 }; } };
|
||||
};
|
||||
const pilot = await discoverMangaDexCatalog(`https://mangadex.org/title/${titleId}`, http);
|
||||
assert.deepEqual(modes, ['0']); assert.equal(pilot.availability.specialReleases, 2); assert.equal(pilot.chapters[0].label, 'Pilot');
|
||||
});
|
||||
|
||||
test('special publication is atomic, readable, stable across rescans and duplicate-safe', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-special-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
// Real fixture PNG; no upstream image requests.
|
||||
const { Canvas } = require('../../tools/png');
|
||||
const page = path.join(root, '.fixture.png'); await fs.writeFile(page, new Canvas(10, 20, [1, 2, 3]).png());
|
||||
const manifest = { version: 1, series: { title: 'Pilot edition', direction: 'rtl', sources: [{ name: 'MangaDex', url: `https://mangadex.org/title/${titleId}` }] },
|
||||
chapters: selectChapterReleases(ids.map(id => release(id)), 'Pilot').map(ref => ({ ...ref, source: { name: 'MangaDex', url: ref.url }, pages: [{ path: page }] })) };
|
||||
const first = await publishManifest(manifest, { libraryRoot: root });
|
||||
assert.ok(first.results.every(result => result.status === 'published'));
|
||||
const scanned = (await reader.scanLibrary(root)).index.mangas[0];
|
||||
assert.equal(scanned.chapters.length, 2); assert.ok(scanned.chapters.every(c => c.label === 'Pilot' && c.num === null && !c.extra));
|
||||
const stableIds = scanned.chapters.map(c => c.id);
|
||||
assert.deepEqual((await publishManifest(manifest, { libraryRoot: root })).results.map(r => r.status), ['skipped', 'skipped']);
|
||||
assert.deepEqual((await reader.scanLibrary(root)).index.mangas[0].chapters.map(c => c.id), stableIds);
|
||||
const manga = (await inventory.scanLibrary(root)).mangas[0];
|
||||
assert.equal(inventory.shouldDownload(manga, manifest.chapters[0], 'mangadex'), false);
|
||||
assert.equal(inventory.isNewerThanLibrary(manga, manifest.chapters[0]), false);
|
||||
assert.equal(inventory.isNewerThanLibrary(manga, { number: '1' }), true);
|
||||
assert.equal(reader.parseKey('c0012.5 Existing [Host]').key, '12.5');
|
||||
assert.equal(reader.parseKey('v01').key, '1');
|
||||
const regular = { ...manifest, chapters: [{ number: '12.5', title: 'Existing chapter', source: { name: 'Host', url: 'https://host.test/12' }, pages: [{ path: page }] }] };
|
||||
await publishManifest(regular, { libraryRoot: root });
|
||||
assert.ok((await reader.scanLibrary(root)).index.mangas[0].chapterById.has('1/12.5'));
|
||||
});
|
||||
|
||||
test('legacy unavailable messages become unverified without retrying or downloading', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-unavailable-')), dataRoot = path.join(root, 'data');
|
||||
await fs.mkdir(dataRoot); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ paused: true,
|
||||
queue: [{ id: 'old-job', source: 'mangadex', url: `https://mangadex.org/title/${titleId}`, status: 'unavailable', error: 'No downloadable English MangaDex chapters found', failure: { kind: 'unavailable', scope: 'job' } }],
|
||||
settings: { autoUpdateEnabled: false } }));
|
||||
const service = await new ScraperService({ libraryRoot: path.join(root, 'library'), dataRoot }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
assert.equal(service.state.queue[0].status, 'unavailable'); assert.equal(service.active.size, 0);
|
||||
assert.match(service.state.queue[0].error, /Inspect this source again/);
|
||||
assert.equal(service.state.queue[0].failure.kind, 'unavailable');
|
||||
});
|
||||
|
||||
test('inspection waits for confirmation, survives restart, and uses one normal source lane', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-inspect-')), dataRoot = path.join(root, 'data');
|
||||
await fs.mkdir(dataRoot); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ paused: true, queue: [], settings: { autoUpdateEnabled: false } }));
|
||||
let running = 0, maxRunning = 0, downloads = 0;
|
||||
const catalog = { title: 'Canonical Pilot', chapters: ids.map(id => ({ id, number: specialKey(id), special: true })), availability: { selectedChapters: 2, specialReleases: 2, coverage: 'unknown' } };
|
||||
const adapter = { id: 'fixture', label: 'Fixture', match: url => ({ source: 'fixture', id: url.split('/').pop(), url, title: 'Misleading slug' }),
|
||||
async discover() { maxRunning = Math.max(maxRunning, ++running); await pause(50); running--; return catalog; },
|
||||
async run(item, options) { downloads++; await options.onCatalog(catalog); return { publishedCount: 1, selectedCount: 1 }; } };
|
||||
const options = { libraryRoot: path.join(root, 'library'), dataRoot, adapters: [adapter], adapterFor: url => ({ adapter, info: adapter.match(url) }) };
|
||||
let service = await new ScraperService(options).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
const item = await service.enqueue({ url: 'https://fixture.test/one', inspect: true });
|
||||
const other = await service.enqueue({ url: 'https://fixture.test/two', inspect: true });
|
||||
await assert.rejects(service.control('confirm', item.id), /Inspect the source first/);
|
||||
await service.control('resume'); await until(() => item.status === 'ready' && other.status === 'ready');
|
||||
assert.equal(downloads, 0); assert.equal(maxRunning, 1); assert.equal(item.sourceTitle, 'Canonical Pilot');
|
||||
await service.close(); service = await new ScraperService(options).init();
|
||||
const restored = service.state.queue.find(q => q.id === item.id);
|
||||
assert.equal(restored.kind, 'inspection'); assert.equal(restored.status, 'ready');
|
||||
await service.control('confirm', restored.id); await until(() => restored.status === 'complete');
|
||||
await until(() => service.active.size === 0);
|
||||
assert.equal(downloads, 1); assert.equal(restored.selectedCount, 1);
|
||||
assert.equal(service.state.history.at(-1).availability.specialReleases, 2);
|
||||
service.state.sourcePaused.fixture = true;
|
||||
await assert.rejects(service.control('inspect', restored.id), /access block/);
|
||||
});
|
||||
|
||||
test('edition change after confirmation cannot silently download another edition', async () => {
|
||||
const service = Object.create(ScraperService.prototype);
|
||||
service.persist = async () => {};
|
||||
const item = { title: 'slug', confirmedSourceTitle: 'Pilot edition' };
|
||||
await assert.rejects(service.recordCatalog(item, { title: 'Main edition', chapters: [] }), /edition title changed/);
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs').promises;
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { detectVerticalStrip, resolveReadingDirection, fallbackDirection } = require('../../reading-direction');
|
||||
const { publishManifest } = require('../lib/publisher');
|
||||
const { scanLibrary } = require('../../library');
|
||||
|
||||
const strip = (w = 800) => ({ w, h: 12000 });
|
||||
|
||||
test('strip detection tolerates a landscape title card and rejects weak evidence', () => {
|
||||
const pages = [{ w: 1200, h: 800 }, strip(), strip(), strip()];
|
||||
assert.equal(detectVerticalStrip(pages).value, 'ttb');
|
||||
assert.equal(detectVerticalStrip(pages).evidence.tallPages, 3);
|
||||
assert.equal(detectVerticalStrip([strip()]), null);
|
||||
assert.equal(detectVerticalStrip([strip(), strip(), { w: 800, h: 1200 }, { w: 800, h: 1200 }]), null);
|
||||
assert.equal(detectVerticalStrip([strip(), strip(), strip(1400), { w: 1200, h: 800 }]), null);
|
||||
assert.equal(detectVerticalStrip([{ w: 800, h: 12000, r: 90 }, strip(), strip()]), null);
|
||||
assert.equal(detectVerticalStrip([{ w: 12000, h: 800, r: 90 }, strip(), strip()]).value, 'ttb');
|
||||
});
|
||||
|
||||
test('manual choices at chapter, book or manga level beat detection and fallback', () => {
|
||||
const pages = [strip(), strip(), strip()];
|
||||
const automatic = fallbackDirection();
|
||||
assert.equal(resolveReadingDirection([automatic], pages).value, 'ttb');
|
||||
assert.equal(resolveReadingDirection([automatic]).source, 'fallback');
|
||||
assert.equal(resolveReadingDirection([{ direction: 'ltr' }, automatic], pages).value, 'ltr');
|
||||
assert.equal(resolveReadingDirection([{}, { direction: 'rtl' }, automatic], pages).value, 'rtl');
|
||||
assert.equal(resolveReadingDirection([{ directionOverride: 'rtl', ...automatic }], pages).value, 'rtl');
|
||||
assert.equal(resolveReadingDirection([{ ...automatic, direction: 'ltr' }], pages).value, 'ltr');
|
||||
assert.equal(resolveReadingDirection([{ direction: 'rtl' }], pages).source, 'manual');
|
||||
assert.equal(resolveReadingDirection([], pages).value, 'ttb');
|
||||
});
|
||||
|
||||
function png(w, h) {
|
||||
const b = Buffer.alloc(24);
|
||||
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]).copy(b);
|
||||
b.writeUInt32BE(w, 16); b.writeUInt32BE(h, 20);
|
||||
return b;
|
||||
}
|
||||
|
||||
async function json(file) { return JSON.parse(await fs.readFile(file, 'utf8')); }
|
||||
|
||||
test('publisher detects new strips, preserves updates and manual metadata, and scanner agrees', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-direction-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const libraryRoot = path.join(temp, 'library');
|
||||
const sizes = [[1200, 800], [800, 12000], [800, 12500], [800, 14210]];
|
||||
const files = [];
|
||||
for (const [i, size] of sizes.entries()) {
|
||||
const file = path.join(temp, `page${i}.png`);
|
||||
await fs.writeFile(file, png(...size)); files.push(file);
|
||||
}
|
||||
const series = { title: 'Strip Work', ...fallbackDirection(),
|
||||
sources: [{ name: 'Test', url: 'https://example.test/series' }] };
|
||||
const chapter = number => ({ number, source: { name: 'Test', url: `https://example.test/${number}` },
|
||||
book: { number: 1, title: 'Season 1' }, pages: files.map(file => ({ path: file })) });
|
||||
const first = await publishManifest({ version: 1, series, chapters: [chapter('1')] }, { libraryRoot });
|
||||
const mangaDir = first.mangaDir, bookDir = path.join(mangaDir, 'v01 Season 1');
|
||||
const mangaFile = path.join(mangaDir, 'manga.json');
|
||||
assert.equal((await json(mangaFile)).direction, 'ttb');
|
||||
assert.equal((await json(mangaFile))['x-reading-direction'].source, 'detected');
|
||||
const scan = (await scanLibrary(libraryRoot)).index.mangas[0];
|
||||
assert.equal(scan.chapters[0].versions[0].direction, 'ttb');
|
||||
assert.equal(scan.chapters[0].versions[0].pages[0].w, 1200);
|
||||
|
||||
const same = await publishManifest({ version: 1, series, chapters: [chapter('2')] }, { libraryRoot });
|
||||
assert.equal(same.mangaDir, mangaDir);
|
||||
assert.equal((await json(mangaFile)).direction, 'ttb');
|
||||
|
||||
const bookFile = path.join(bookDir, 'book.json');
|
||||
await fs.writeFile(bookFile, JSON.stringify({ title: 'Season 1', direction: 'ltr', note: 'human' }));
|
||||
const chapterDir = path.join(bookDir, first.results[0].folder);
|
||||
const chapterFile = path.join(chapterDir, 'chapter.json');
|
||||
await fs.writeFile(chapterFile, JSON.stringify({ ...await json(chapterFile), directionOverride: 'rtl', note: 'human' }));
|
||||
await publishManifest({ version: 1, series, chapters: [chapter('1')] }, { libraryRoot, replace: true });
|
||||
assert.equal((await json(bookFile)).direction, 'ltr');
|
||||
assert.equal((await json(bookFile)).note, 'human');
|
||||
assert.equal((await json(chapterFile)).direction, 'rtl');
|
||||
assert.equal((await json(chapterFile)).directionOverride, 'rtl');
|
||||
assert.equal((await json(chapterFile)).note, 'human');
|
||||
const rescanned = (await scanLibrary(libraryRoot)).index.mangas[0];
|
||||
assert.equal(rescanned.chapters[0].versions[0].direction, 'rtl');
|
||||
assert.equal(rescanned.chapters[1].versions[0].direction, 'ltr');
|
||||
|
||||
await fs.writeFile(mangaFile, JSON.stringify({ ...await json(mangaFile), direction: 'rtl' }));
|
||||
await publishManifest({ version: 1, series, chapters: [chapter('3')] }, { libraryRoot });
|
||||
assert.equal((await json(mangaFile)).direction, 'rtl');
|
||||
assert.equal((await json(mangaFile))['x-reading-direction'].source, 'manual');
|
||||
assert.equal((await scanLibrary(libraryRoot)).index.mangas[0].chapters[2].versions[0].direction, 'ltr');
|
||||
});
|
||||
|
||||
test('legacy plain RTL stays protected across automatic imports', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-legacy-direction-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const libraryRoot = path.join(temp, 'library');
|
||||
const page = path.join(temp, 'tall.png'); await fs.writeFile(page, png(800, 12000));
|
||||
const series = { title: 'Legacy', ...fallbackDirection(), sources: [{ name: 'Test', url: 'https://example.test/legacy' }] };
|
||||
const chapter = { number: '1', source: { name: 'Test', url: 'https://example.test/legacy/1' },
|
||||
pages: [1, 2, 3].map(() => ({ path: page })) };
|
||||
const first = await publishManifest({ version: 1, series, chapters: [chapter] }, { libraryRoot });
|
||||
const file = path.join(first.mangaDir, 'manga.json');
|
||||
const meta = await json(file); delete meta['x-reading-direction']; meta.direction = 'rtl';
|
||||
await fs.writeFile(file, JSON.stringify(meta));
|
||||
await publishManifest({ version: 1, series, chapters: [{ ...chapter, number: '2', source: { name: 'Test', url: 'https://example.test/legacy/2' } }] }, { libraryRoot });
|
||||
assert.equal((await json(file)).direction, 'rtl');
|
||||
assert.equal((await json(file))['x-reading-direction'].source, 'manual');
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict'), fs = require('fs').promises, os = require('os'), path = require('path');
|
||||
const { ScraperService, validateSettings } = require('../service');
|
||||
const { classifyFailure, safeMessage } = require('../lib/failures');
|
||||
test('failure classification protects access challenges and distinguishes job failures', () => {
|
||||
for (const status of [401, 403, 429]) assert.equal(classifyFailure(Object.assign(new Error('Host response'), { status })).kind, 'access');
|
||||
assert.equal(classifyFailure(new Error('No downloadable English MangaDex chapters found')).scope, 'job');
|
||||
assert.equal(classifyFailure(Object.assign(new Error('Disk full'), { code: 'ENOSPC' })).scope, 'global');
|
||||
assert.equal(classifyFailure(new Error('HTTP 503')).retryable, true);
|
||||
assert.equal(safeMessage('HTTP 403 https://test.example/a?token=secret'), 'HTTP 403 https://test.example/a');
|
||||
});
|
||||
async function fixture(t, errors, options = {}) {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-recovery-'));
|
||||
const dataRoot = path.join(dir, 'data'); await fs.mkdir(dataRoot);
|
||||
const counts = {};
|
||||
const adapter = { id: 'test', label: 'Test', run: async item => {
|
||||
counts[item.id] = (counts[item.id] || 0) + 1;
|
||||
if (errors[item.id]) throw errors[item.id]; return { publishedCount: 0 };
|
||||
} };
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ version: 3, paused: true, settings: validateSettings({ autoUpdateEnabled: false }),
|
||||
queue: Object.keys(errors).concat('healthy').map(id => ({ id, seriesId: id, source: 'test', title: id, url: 'https://test.example/' + id, status: 'queued' })), history: [] }));
|
||||
const service = await new ScraperService({ libraryRoot: path.join(dir, 'library'), dataRoot, adapterFor: url => ({ adapter, info: { id: url.split('/').pop() } }), adapters: [adapter], ...options }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(dir, { recursive: true, force: true }); });
|
||||
await service.control('resume');
|
||||
return { service, counts };
|
||||
}
|
||||
async function wait(predicate) {
|
||||
const end = Date.now() + 6000;
|
||||
while (!predicate()) { if (Date.now() > end) throw new Error('Timeout'); await new Promise(r => setTimeout(r, 25)); }
|
||||
}
|
||||
test('unavailable titles and one parser failure do not stop healthy jobs', async t => {
|
||||
const { service } = await fixture(t, { absent: new Error('No downloadable English MangaDex chapters found'), parser: new Error('Catalog markup changed') });
|
||||
await wait(() => service.state.queue.find(i => i.id === 'healthy').status === 'complete');
|
||||
assert.equal(service.state.queue.find(i => i.id === 'absent').status, 'unavailable');
|
||||
assert.equal(service.state.queue.find(i => i.id === 'parser').status, 'quarantined');
|
||||
assert.equal(service.state.sourcePaused.test, undefined);
|
||||
});
|
||||
test('transient recovery is bounded and access errors are never retried automatically', async t => {
|
||||
const { service, counts } = await fixture(t, { broken: new Error('HTTP 503') }, { retryDelayMs: 10 });
|
||||
await wait(() => service.state.sourcePaused.test);
|
||||
assert.equal(counts.broken, 2); assert.equal(service.state.queue[0].status, 'error');
|
||||
});
|
||||
test('protection failures pause only the affected lane and keep checkpoints', async t => {
|
||||
const { service, counts } = await fixture(t, { blocked: new Error('HTTP 403') });
|
||||
await wait(() => service.state.sourcePaused.test);
|
||||
assert.equal(counts.blocked, 1); assert.equal(counts.healthy, undefined);
|
||||
assert.equal(service.state.incidents[0].failure.kind, 'access');
|
||||
});
|
||||
@@ -0,0 +1,556 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { cleanPart, chapterKey, chapterFolder, assertInside } = require('../lib/names');
|
||||
const { imageExtension } = require('../lib/images');
|
||||
const { publishManifest } = require('../lib/publisher');
|
||||
const { mergeImportedMetadata } = require('../lib/metadata-merge');
|
||||
const { metadataFromMedia } = require('../metadata/anilist');
|
||||
const { normalizeScope, selectByScope, scopeLabel } = require('../lib/scope');
|
||||
const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, setAutoUpdate } = require('../lib/library-inventory');
|
||||
const { inspectLegacyWork } = require('../lib/legacy-publication');
|
||||
const { discoverWeebCatalog, parseChapters, parsePageUrls } = require('../adapters/weebcentral');
|
||||
const { chapterNumber, selectChapterReleases, pageUrls } = require('../adapters/mangadex');
|
||||
const { adapterFor } = require('../adapters');
|
||||
const { ScraperService, validateSettings, settingsForSource } = require('../service');
|
||||
|
||||
test('MangaCircus chapter keys are normalized', () => {
|
||||
assert.equal(chapterKey('12'), 'c0012');
|
||||
assert.equal(chapterKey('12.50'), 'c0012.5');
|
||||
assert.equal(chapterKey('x2'), 'x02');
|
||||
assert.throws(() => chapterKey('special'));
|
||||
});
|
||||
|
||||
test('folder names are portable and versioned', () => {
|
||||
assert.equal(cleanPart('Bad:<Title>. '), 'BadTitle');
|
||||
assert.equal(cleanPart('CON'), 'CON-manga');
|
||||
assert.equal(chapterFolder({ number: '7', title: 'A [Test]', source: { name: 'Host' } }), 'c0007 A Test [Host]');
|
||||
assert.throws(() => assertInside('C:/library', 'C:/outside'));
|
||||
});
|
||||
|
||||
test('image signatures are recognized', () => {
|
||||
assert.equal(imageExtension(Buffer.from([0xff, 0xd8, 0xff, 0x00])), 'jpg');
|
||||
assert.equal(imageExtension(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])), 'png');
|
||||
assert.equal(imageExtension(Buffer.from('not an image')), null);
|
||||
});
|
||||
|
||||
test('per-source rate settings default fast and update independently', () => {
|
||||
const defaults = validateSettings({});
|
||||
const fast = { minMs: 1000, maxMs: 4000, chapterMinMs: 5000, chapterMaxMs: 15000 };
|
||||
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast);
|
||||
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(defaults, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), fast);
|
||||
|
||||
const changed = validateSettings({ sourceRates: { mangadex: {
|
||||
minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000,
|
||||
} } }, defaults);
|
||||
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'mangadex')).filter(([key]) => key.endsWith('Ms'))), {
|
||||
minMs: 1500, maxMs: 3500, chapterMinMs: 6000, chapterMaxMs: 12000,
|
||||
});
|
||||
assert.deepEqual(Object.fromEntries(Object.entries(settingsForSource(changed, 'weebcentral')).filter(([key]) => key.endsWith('Ms'))), fast);
|
||||
assert.throws(() => validateSettings({ sourceRates: { mangadex: { minMs: 5000, maxMs: 1000 } } }, defaults));
|
||||
});
|
||||
|
||||
test('AniList metadata keeps manga format and original story credits', () => {
|
||||
const metadata = metadataFromMedia({
|
||||
id: 150193, idMal: null, format: 'MANGA',
|
||||
title: { english: 'Heavy Knight', romaji: 'Juukishi', native: '重騎士' }, synonyms: [],
|
||||
staff: { edges: [
|
||||
{ role: 'Original Story', node: { name: { full: 'Nekoko' } } },
|
||||
{ role: 'Story & Art', node: { name: { full: 'Lee Brocco' } } },
|
||||
{ role: 'Character Design', node: { name: { full: 'Jaian' } } },
|
||||
] },
|
||||
status: 'RELEASING', startDate: { year: 2022 }, genres: ['Fantasy'], description: '',
|
||||
siteUrl: 'https://anilist.co/manga/150193', coverImage: { extraLarge: 'https://s4.anilist.co/cover.jpg' },
|
||||
});
|
||||
assert.deepEqual(metadata.authors, ['Nekoko', 'Lee Brocco']);
|
||||
assert.equal(metadata['x-anilist'].format, 'MANGA');
|
||||
});
|
||||
|
||||
test('metadata refresh preserves legacy and manual fields while updating untouched imports', () => {
|
||||
const first = mergeImportedMetadata({ title: 'My preferred title', status: 'ongoing' }, {
|
||||
title: 'Provider title', status: 'completed', tags: ['Fantasy'], 'x-anilist': { id: 42 },
|
||||
}, 'Fallback');
|
||||
assert.equal(first.title, 'My preferred title');
|
||||
assert.equal(first.status, 'ongoing');
|
||||
assert.deepEqual(first.tags, ['Fantasy']);
|
||||
assert.equal(first['x-metadata'].imported.title, 'Provider title');
|
||||
const second = mergeImportedMetadata({ ...first, tags: ['Fantasy', 'Favorite'] }, {
|
||||
title: 'New provider title', status: 'ongoing', tags: ['Action'], 'x-anilist': { id: 42 },
|
||||
}, 'Fallback');
|
||||
assert.equal(second.title, 'My preferred title');
|
||||
assert.equal(second.status, 'ongoing');
|
||||
assert.deepEqual(second.tags, ['Fantasy', 'Favorite']);
|
||||
const third = mergeImportedMetadata(second, { description: 'First', 'x-anilist': { id: 42 } }, 'Fallback');
|
||||
assert.equal(mergeImportedMetadata(third, { description: 'Revised', 'x-anilist': { id: 42 } }, 'Fallback').description, 'Revised');
|
||||
assert.throws(() => mergeImportedMetadata(first, { 'x-anilist': { id: 99 } }, 'Fallback'), /identity conflict/);
|
||||
});
|
||||
|
||||
test('publisher rejects a conflicting work ID before fetching any pages', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-identity-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const manga = path.join(root, 'Existing');
|
||||
await fs.mkdir(manga);
|
||||
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Existing',
|
||||
sources: [{ name: 'Host', url: 'https://example.test/series' }], 'x-anilist': { id: 42 } }));
|
||||
let requested = 0;
|
||||
await assert.rejects(publishManifest({ version: 1,
|
||||
series: { title: 'Existing', direction: 'rtl', metadata: { 'x-anilist': { id: 99 } },
|
||||
sources: [{ name: 'Host', url: 'https://example.test/series' }] },
|
||||
chapters: [{ number: '1', source: { name: 'Host', url: 'https://example.test/chapter' },
|
||||
pages: [{ url: 'https://example.test/page.png' }] }],
|
||||
}, { libraryRoot: root, httpClient: { async get() { requested++; throw new Error('unexpected request'); } } }), /identity conflict/);
|
||||
assert.equal(requested, 0);
|
||||
assert.equal((await fs.readdir(manga)).includes('v01'), false);
|
||||
});
|
||||
|
||||
test('queue scopes select exact catalog slices', () => {
|
||||
const chapters = [
|
||||
{ id: 'a', number: '1', volume: '1' }, { id: 'b', number: '2', volume: '1' },
|
||||
{ id: 'c', number: '3', volume: '2' }, { id: 'd', number: '4', volume: '2' },
|
||||
];
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'latest' }).map(x => x.id), ['d']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'last', count: 2 }).map(x => x.id), ['c', 'd']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'first', count: 2 }).map(x => x.id), ['a', 'b']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'chapter', chapter: '003.0' }).map(x => x.id), ['c']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'after', chapter: '2' }).map(x => x.id), ['c', 'd']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'volume', volume: '2' }).map(x => x.id), ['c', 'd']);
|
||||
assert.deepEqual(selectByScope(chapters, { mode: 'ids', ids: ['b', 'd'] }).map(x => x.id), ['b', 'd']);
|
||||
assert.equal(scopeLabel(normalizeScope({ mode: 'last', count: 5 })), 'last 5 chapters');
|
||||
});
|
||||
|
||||
test('library inventory distinguishes missing chapters from alternate source versions', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-inventory-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const manga = path.join(temp, 'Library Manga'), book = path.join(manga, 'v01');
|
||||
await fs.mkdir(path.join(book, 'c0001 [WeebCentral]'), { recursive: true });
|
||||
await fs.mkdir(path.join(book, 'c0002 [MangaDex]'), { recursive: true });
|
||||
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({
|
||||
title: 'Library Manga', status: 'ongoing',
|
||||
sources: [{ name: 'WeebCentral', url: 'https://weebcentral.com/series/SERIES/Test' }],
|
||||
'x-scraper': { ids: { weebcentral: 'SERIES', anilist: 42 } },
|
||||
}));
|
||||
await fs.writeFile(path.join(book, 'c0001 [WeebCentral]', 'chapter.json'), JSON.stringify({ source: { name: 'WeebCentral', url: 'https://weebcentral.com/chapters/A' } }));
|
||||
await fs.writeFile(path.join(book, 'c0002 [MangaDex]', 'chapter.json'), JSON.stringify({ source: { name: 'MangaDex · Group', url: 'https://mangadex.org/chapter/B' } }));
|
||||
const inventory = await scanLibrary(temp);
|
||||
const found = findManga(inventory, { source: 'weebcentral', seriesId: 'SERIES' });
|
||||
assert.equal(found.id, 'Library Manga');
|
||||
assert.equal(findManga(inventory, { source: 'mangadex', seriesId: 'other', anilist: 42 }).id, 'Library Manga');
|
||||
assert.equal(found.autoUpdate, true);
|
||||
assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'skip-any'), false);
|
||||
assert.equal(shouldDownload(found, { number: '1' }, 'mangadex', 'add-source'), true);
|
||||
assert.equal(shouldDownload(found, { number: '1' }, 'weebcentral', 'add-source'), false);
|
||||
assert.equal(shouldDownload(found, { number: '3' }, 'weebcentral', 'skip-any'), true);
|
||||
assert.equal(isNewerThanLibrary(found, { number: '1' }), false);
|
||||
assert.equal(isNewerThanLibrary(found, { number: '3' }), true);
|
||||
await setAutoUpdate(temp, 'Library Manga', false);
|
||||
assert.equal((await scanLibrary(temp)).mangas[0].autoUpdate, false);
|
||||
});
|
||||
|
||||
test('a shared work ID cannot merge distinct source listings or ambiguous editions', () => {
|
||||
const original = { id: 'Main', ids: { weebcentral: 'MAIN', anilist: 42 }, sources: [] };
|
||||
assert.equal(findManga({ mangas: [original] }, { source: 'weebcentral', seriesId: 'PILOT', anilist: 42 }), null);
|
||||
assert.equal(findManga({ mangas: [original] }, { source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), original);
|
||||
assert.equal(findManga({ mangas: [original, { id: 'Pilot', ids: { anilist: 42 }, sources: [] }] },
|
||||
{ source: 'mangadex', seriesId: 'OTHER', anilist: 42 }), null);
|
||||
});
|
||||
|
||||
test('recent events read the bounded log tail without losing the newest incident', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-events-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const service = new ScraperService({ dataRoot: root });
|
||||
await fs.writeFile(service.logFile, `${'x'.repeat(600 * 1024)}\n${JSON.stringify({ level: 'error', message: 'recent' })}\n`);
|
||||
assert.deepEqual(await service.recentEvents(1), [{ level: 'error', message: 'recent' }]);
|
||||
assert.equal((await service.recentEvents(10)).at(-1).message, 'recent');
|
||||
});
|
||||
|
||||
test('paused legacy work is validated, published, and converted to incremental mode', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-legacy-publish-'));
|
||||
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
|
||||
const workRoot = path.join(libraryRoot, '.mcscraper-work-SERIES42');
|
||||
const stagedManga = path.join(workRoot, 'Legacy Manga'), book = path.join(stagedManga, 'v01');
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.mkdir(book, { recursive: true });
|
||||
await fs.writeFile(path.join(stagedManga, 'manga.json'), JSON.stringify({
|
||||
title: 'Legacy Manga', status: 'ongoing',
|
||||
sources: [{ name: 'Alpha', url: 'https://alpha.test/title/SERIES42' }],
|
||||
'x-scraper': { ids: { alpha: 'SERIES42' } },
|
||||
}));
|
||||
for (const number of [1, 2]) {
|
||||
const chapter = path.join(book, `c000${number} [Alpha]`);
|
||||
await fs.mkdir(chapter, { recursive: true });
|
||||
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: `https://alpha.test/chapter/${number}` } }));
|
||||
await fs.writeFile(path.join(chapter, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
}
|
||||
const incomplete = path.join(book, '.staging-mcscraper-abc123');
|
||||
await fs.mkdir(incomplete);
|
||||
await fs.writeFile(path.join(incomplete, '0001.png'), Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
await fs.writeFile(path.join(workRoot, 'job.json'), JSON.stringify({
|
||||
version: 2, status: 'paused', completed: ['one', 'two'], total: 4,
|
||||
}));
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 3, paused: true, status: 'paused', settings: validateSettings({}), history: [], publishedRevision: 0,
|
||||
queue: [{ id: 'old-id', seriesId: 'SERIES42', source: 'alpha', title: 'Legacy Manga', url: 'https://alpha.test/title/SERIES42', status: 'queued', order: 0 }],
|
||||
}));
|
||||
const alpha = {
|
||||
id: 'alpha', label: 'Alpha',
|
||||
match(value) { return String(value).includes('alpha.test/title/') ? { source: 'alpha', id: 'SERIES42', title: 'Legacy Manga', url: value } : null; },
|
||||
async discover() { return { chapters: [] }; }, async run() { return null; },
|
||||
};
|
||||
const service = await new ScraperService({
|
||||
libraryRoot, dataRoot, adapters: [alpha],
|
||||
adapterFor: value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; },
|
||||
}).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
||||
|
||||
const inspection = await inspectLegacyWork({ libraryRoot, seriesId: 'SERIES42' });
|
||||
assert.equal(inspection.chapterCount, 2);
|
||||
assert.equal(inspection.pageCount, 2);
|
||||
assert.equal(inspection.incompleteStages.length, 1);
|
||||
const result = await service.publishLegacyReady();
|
||||
assert.equal(result.published.length, 1);
|
||||
assert.equal(result.published[0].chaptersPublished, 2);
|
||||
assert.equal(result.published[0].incompleteStagesQuarantined, 1);
|
||||
assert.equal(await fs.stat(path.join(libraryRoot, 'Legacy Manga', 'v01', 'c0002 [Alpha]')).then(stat => stat.isDirectory()), true);
|
||||
await assert.rejects(fs.stat(stagedManga));
|
||||
const migrated = JSON.parse(await fs.readFile(path.join(workRoot, 'job.json'), 'utf8'));
|
||||
assert.equal(migrated.publicationMode, 'incremental');
|
||||
assert.equal(migrated.migratedToIncremental.chaptersPublished, 2);
|
||||
assert.equal(migrated.migratedToIncremental.incompleteStagesQuarantined, 1);
|
||||
assert.equal(await fs.stat(path.join(workRoot, '.legacy-incomplete', 'v01__.staging-mcscraper-abc123')).then(stat => stat.isDirectory()), true);
|
||||
assert.equal(service.state.queue[0].libraryMangaId, 'Legacy Manga');
|
||||
assert.equal(service.state.publishedRevision, 1);
|
||||
});
|
||||
|
||||
test('WeebCentral fragments preserve chapter and page order', () => {
|
||||
const chapters = parseChapters('<a href="/chapters/P1">Prologue 1</a><a href="/chapters/ABC">Chapter 12</a><a href="/chapters/DEF">Chapter 12.5</a><a href="/chapters/S2">S2 - Chapter 1</a><a href="/chapters/R442"><span>Rating 442</span><span>Last Read</span></a><a href="/chapters/E128"><span>Episode 128</span><span>Last Read</span></a><a href="/chapters/E78"><span>Episode: 78</span><span>Last Read</span></a><a href="/chapters/N182"><span>Night 182</span><span>Last Read</span></a><a href="/chapters/L65">Log. 65</a><a href="/chapters/A606">Act 60.6</a>', 'https://weebcentral.com/s');
|
||||
assert.deepEqual(chapters.map(item => item.number), ['0.01', '12', '12.5', '1', '442', '128', '78', '182', '65', '60.6']);
|
||||
assert.deepEqual(chapters[3].book, { number: 2, title: 'Season 2' });
|
||||
const pages = parsePageUrls('<img src="/001.png"><img data-src="https://img.example/002.webp">', 'https://weebcentral.com/c');
|
||||
assert.deepEqual(pages, ['https://weebcentral.com/001.png', 'https://img.example/002.webp']);
|
||||
});
|
||||
|
||||
test('WeebCentral reports unrecognized chapter labels separately from an empty catalog', async () => {
|
||||
const series = '<h1>Example</h1><div hx-get="/series/X/full-chapter-list"></div>';
|
||||
const httpFor = list => ({ get: async url => ({ text: async () => url.endsWith('full-chapter-list') ? list : series }) });
|
||||
const unknown = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<a href="/chapters/ABC">Part One</a>'));
|
||||
assert.equal(unknown.chapterLinks, 1);
|
||||
assert.equal(unknown.chapters.length, 0);
|
||||
assert.equal(unknown.availability.reason, 'unrecognized-labels');
|
||||
assert.match(unknown.availability.message, /1 chapter links/);
|
||||
const empty = await discoverWeebCatalog('https://weebcentral.com/series/X/Example', httpFor('<div>No chapters</div>'));
|
||||
assert.equal(empty.chapterLinks, 0);
|
||||
assert.equal(empty.availability, undefined);
|
||||
});
|
||||
|
||||
test('completed download reinspection preserves complete state when all scoped chapters are present', async () => {
|
||||
const service = new ScraperService({ libraryRoot: 'unused', dataRoot: 'unused' });
|
||||
const item = { id: 'done', title: 'Done', source: 'weebcentral', url: 'https://weebcentral.com/series/EXAMPLE/Done',
|
||||
seriesId: 'EXAMPLE', kind: 'inspection', status: 'queued', completed: '2026-10-01T00:00:00Z',
|
||||
publishedCount: 2, selectedCount: 2, scope: { mode: 'missing' }, inspectionFromComplete: true };
|
||||
service.state = { paused: false, settings: validateSettings({}), queue: [item], incidents: [], sourcePaused: {}, currentIds: {} };
|
||||
service.recordCatalog = async (entry, catalog) => { entry.sourceTitle = catalog.title; entry.catalogTotal = catalog.chapters.length; };
|
||||
service.libraryInventory = async () => ({ mangas: [{ id: 'Done', ids: { weebcentral: 'EXAMPLE' }, sources: [],
|
||||
chapters: [{ book: '1', number: '1' }, { book: '1', number: '2' }] }] });
|
||||
service.event = async () => {};
|
||||
const adapter = { discover: async () => ({ title: 'Done', chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }] }) };
|
||||
await service.runDiscovery(item, { adapter }, 'weebcentral');
|
||||
assert.equal(item.status, 'complete');
|
||||
assert.equal(item.kind, 'download');
|
||||
assert.equal(item.completed, '2026-10-01T00:00:00Z');
|
||||
assert.equal(item.publishedCount, 2);
|
||||
assert.equal(item.inspectionFromComplete, undefined);
|
||||
});
|
||||
|
||||
test('publisher keeps season chapter numbers in separate books', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-books-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const page = path.join(temp, 'page.png');
|
||||
await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
const base = { title: '', source: { name: 'Host', url: 'https://example.test/chapter' }, pages: [{ path: page }] };
|
||||
await publishManifest({
|
||||
version: 1,
|
||||
series: { title: 'Season Test', direction: 'ttb', sources: [{ name: 'Host', url: 'https://example.test/series' }] },
|
||||
chapters: [
|
||||
{ ...base, number: '1', book: { number: 1, title: 'Season 1' } },
|
||||
{ ...base, number: '1', book: { number: 2, title: 'Season 2' } },
|
||||
],
|
||||
}, { libraryRoot: path.join(temp, 'library') });
|
||||
assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v01 Season 1', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 1');
|
||||
assert.equal(await fs.readFile(path.join(temp, 'library', 'Season Test', 'v02 Season 2', 'book.json'), 'utf8').then(JSON.parse).then(x => x.title), 'Season 2');
|
||||
});
|
||||
|
||||
test('publisher stages and atomically exposes a complete new manga', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-test-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const page = path.join(temp, 'page.png');
|
||||
await fs.writeFile(page, Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]));
|
||||
const result = await publishManifest({
|
||||
version: 1,
|
||||
series: { title: 'Test Manga', direction: 'rtl', metadata: { status: 'ongoing' },
|
||||
sources: [{ name: 'Test', url: 'https://example.test/manga' }] },
|
||||
chapters: [{ number: '1', title: 'Start', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }],
|
||||
}, { libraryRoot: path.join(temp, 'library') });
|
||||
assert.equal(result.results[0].status, 'published');
|
||||
const chapter = path.join(temp, 'library', 'Test Manga', 'v01', 'c0001 Start [Test]');
|
||||
assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', 'chapter.json']);
|
||||
const meta = JSON.parse(await fs.readFile(path.join(temp, 'library', 'Test Manga', 'manga.json'), 'utf8'));
|
||||
assert.equal(meta['x-scraper'].autoUpdate, undefined, 'ongoing is a derived default, not a frozen override');
|
||||
assert.equal((await scanLibrary(path.join(temp, 'library'))).mangas[0].autoUpdate, true);
|
||||
assert.equal((await fs.readdir(path.join(temp, 'library'))).some(name => name.startsWith('.staging-')), false);
|
||||
});
|
||||
|
||||
test('publisher installs a validated AniList series cover', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-cover-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const page = path.join(temp, 'page.png');
|
||||
const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]);
|
||||
await fs.writeFile(page, png);
|
||||
let requests = 0;
|
||||
const httpClient = {
|
||||
async get(url) {
|
||||
requests++;
|
||||
assert.equal(url, 'https://s4.anilist.co/example.png');
|
||||
return { arrayBuffer: async () => png };
|
||||
},
|
||||
};
|
||||
const series = {
|
||||
title: 'Covered Manga', direction: 'rtl',
|
||||
sources: [{ name: 'Test', url: 'https://example.test/covered' }],
|
||||
coverUrl: 'https://s4.anilist.co/example.png',
|
||||
};
|
||||
const result = await publishManifest({
|
||||
version: 1, series,
|
||||
chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' }, pages: [{ path: page }] }],
|
||||
}, { libraryRoot: path.join(temp, 'library'), httpClient });
|
||||
assert.equal(result.cover.status, 'installed');
|
||||
assert.deepEqual(await fs.readFile(path.join(temp, 'library', 'Covered Manga', 'cover.png')), png);
|
||||
assert.equal(requests, 1);
|
||||
});
|
||||
|
||||
test('publisher retains validated partial pages and resumes without duplicate requests', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-resume-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0]);
|
||||
const manifest = {
|
||||
version: 1, series: { title: 'Resumable', direction: 'rtl', sources: [{ name: 'Test', url: 'https://example.test/series' }] },
|
||||
chapters: [{ number: '1', source: { name: 'Test', url: 'https://example.test/chapter/1' },
|
||||
pages: [1, 2].map(n => ({ url: `https://example.test/${n}.png` })) }],
|
||||
};
|
||||
const requests = [];
|
||||
const httpClient = { async get(url) { requests.push(url); return { arrayBuffer: async () => png }; } };
|
||||
const libraryRoot = path.join(temp, 'library');
|
||||
await assert.rejects(publishManifest(manifest, { libraryRoot, httpClient,
|
||||
onPage() { throw new Error('cooperative pause'); } }), /cooperative pause/);
|
||||
await assert.rejects(fs.stat(path.join(libraryRoot, 'Resumable')));
|
||||
const hidden = (await fs.readdir(libraryRoot))[0];
|
||||
assert.match(hidden, /^\.staging-mcscraper-resume-/);
|
||||
const { cleanupOwnTemps } = require('../lib/publisher');
|
||||
await cleanupOwnTemps(libraryRoot, -1);
|
||||
assert.ok(await fs.stat(path.join(libraryRoot, hidden)));
|
||||
await publishManifest(manifest, { libraryRoot, httpClient });
|
||||
assert.deepEqual(requests, ['https://example.test/1.png', 'https://example.test/2.png']);
|
||||
const chapter = path.join(libraryRoot, 'Resumable', 'v01', 'c0001 [Test]');
|
||||
assert.deepEqual((await fs.readdir(chapter)).sort(), ['0001.png', '0002.png', 'chapter.json']);
|
||||
});
|
||||
|
||||
test('adapter registry recognizes supported series URLs', () => {
|
||||
const found = adapterFor('https://weebcentral.com/series/01ABCDEF/My-Series');
|
||||
assert.equal(found.adapter.id, 'weebcentral');
|
||||
assert.equal(found.info.id, '01ABCDEF');
|
||||
assert.equal(found.info.title, 'My Series');
|
||||
const dex = adapterFor('https://mangadex.org/title/5bdb093f-9707-4a30-b02e-c8b66554ecf0/biomega');
|
||||
assert.equal(dex.adapter.id, 'mangadex');
|
||||
assert.equal(dex.info.id, '5bdb093f-9707-4a30-b02e-c8b66554ecf0');
|
||||
assert.equal(adapterFor('https://example.test/series/1'), null);
|
||||
});
|
||||
|
||||
test('MangaDex releases are normalized, deduplicated, credited, and host-validated', () => {
|
||||
assert.equal(chapterNumber('0012.50'), '12.5');
|
||||
assert.equal(chapterNumber('Special'), null);
|
||||
const release = (id, chapter, group, publishAt) => ({
|
||||
id, attributes: { chapter, title: '', pages: 2, publishAt, externalUrl: null },
|
||||
relationships: group ? [{ id: `group-${id}`, type: 'scanlation_group', attributes: { name: group } }] : [],
|
||||
});
|
||||
const selected = selectChapterReleases([
|
||||
release('fan', '1', 'Fan Group', '2020-01-01T00:00:00Z'),
|
||||
release('official', '1.0', 'Publisher Official', '2021-01-01T00:00:00Z'),
|
||||
release('two', '2', 'Fan Group', '2020-01-02T00:00:00Z'),
|
||||
]);
|
||||
assert.deepEqual(selected.map(item => item.id), ['official', 'two']);
|
||||
assert.equal(selected[0].group.name, 'Publisher Official');
|
||||
const pages = pageUrls({
|
||||
baseUrl: 'https://node.mangadex.network',
|
||||
chapter: { hash: 'abc', data: ['1.jpg', '2.png'] },
|
||||
}, 'https://mangadex.org/chapter/test', 2);
|
||||
assert.deepEqual(pages.map(item => item.url), [
|
||||
'https://node.mangadex.network/data/abc/1.jpg',
|
||||
'https://node.mangadex.network/data/abc/2.png',
|
||||
]);
|
||||
assert.throws(() => pageUrls({ baseUrl: 'https://evil.example', chapter: { hash: 'abc', data: ['1.jpg'] } }, 'https://mangadex.org/chapter/test', 1));
|
||||
});
|
||||
|
||||
test('daemon queue state is durable while paused', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-service-'));
|
||||
const dataRoot = path.join(temp, 'data');
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 1, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
|
||||
}));
|
||||
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
||||
const item = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series' });
|
||||
assert.equal(item.status, 'queued');
|
||||
const scoped = await service.enqueue({ url: 'https://weebcentral.com/series/01ABCDEF/My-Series', scope: { mode: 'latest' } });
|
||||
assert.notEqual(item.id, scoped.id);
|
||||
assert.equal(item.seriesId, scoped.seriesId);
|
||||
assert.equal((await service.publicState()).queue.length, 2);
|
||||
await service.updateSettings({ minMs: 1500, maxMs: 4500 });
|
||||
assert.equal(JSON.parse(await fs.readFile(path.join(dataRoot, 'queue.json'), 'utf8')).settings.minMs, 1500);
|
||||
await service.remove(item.id);
|
||||
await service.remove(scoped.id);
|
||||
assert.equal((await service.publicState()).queue.length, 0);
|
||||
});
|
||||
|
||||
test('retry recovers stale downloading entries before work resumes', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-retry-'));
|
||||
const dataRoot = path.join(temp, 'data');
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 1, paused: true, status: 'error', currentId: 'FAILED', settings: validateSettings({}),
|
||||
queue: [
|
||||
{ id: 'STALE', title: 'Stale', url: 'https://weebcentral.com/series/STALE/Stale', status: 'downloading', order: 0 },
|
||||
{ id: 'FAILED', title: 'Failed', url: 'https://weebcentral.com/series/FAILED/Failed', status: 'error', order: 1 },
|
||||
], history: [], publishedRevision: 0,
|
||||
}));
|
||||
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
||||
await service.control('retry', 'FAILED');
|
||||
assert.deepEqual(service.state.queue.map(item => item.status), ['queued', 'queued']);
|
||||
assert.equal(service.state.currentId, null);
|
||||
assert.equal(service.state.paused, false);
|
||||
});
|
||||
|
||||
test('nightly discovery queues one preferred supported source for ongoing manga', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-'));
|
||||
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
|
||||
const manga = path.join(libraryRoot, 'Ongoing Manga');
|
||||
await fs.mkdir(manga, { recursive: true });
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({
|
||||
title: 'Ongoing Manga', status: 'ongoing',
|
||||
sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }],
|
||||
'x-scraper': { ids: { alpha: '42' } },
|
||||
}));
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
|
||||
}));
|
||||
const alpha = {
|
||||
id: 'alpha', label: 'Alpha',
|
||||
match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: url.pathname.split('/').pop(), title: 'Alpha', url: value } : null; },
|
||||
async discover() { return { chapters: [] }; }, async run() { return null; },
|
||||
};
|
||||
const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; };
|
||||
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(temp, { recursive: true, force: true }); });
|
||||
const first = await service.enqueueDiscoveryChecks('test');
|
||||
const second = await service.enqueueDiscoveryChecks('test');
|
||||
assert.deepEqual(first, { queued: 1, skipped: 0 });
|
||||
assert.deepEqual(second, { queued: 0, skipped: 1 });
|
||||
assert.equal(service.state.queue[0].kind, 'discovery');
|
||||
assert.equal(service.state.queue[0].libraryMangaId, 'Ongoing Manga');
|
||||
});
|
||||
|
||||
test('discovery promotes only chapters newer than the library frontier', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-discovery-run-'));
|
||||
const dataRoot = path.join(temp, 'data'), libraryRoot = path.join(temp, 'library');
|
||||
const chapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0001 [Alpha]');
|
||||
const laterChapter = path.join(libraryRoot, 'Ongoing Manga', 'v01', 'c0003 [Alpha]');
|
||||
await fs.mkdir(chapter, { recursive: true });
|
||||
await fs.mkdir(laterChapter, { recursive: true });
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
await fs.writeFile(path.join(libraryRoot, 'Ongoing Manga', 'manga.json'), JSON.stringify({
|
||||
title: 'Ongoing Manga', status: 'ongoing', sources: [{ name: 'Alpha', url: 'https://alpha.test/title/42' }],
|
||||
'x-scraper': { ids: { alpha: '42' } },
|
||||
}));
|
||||
await fs.writeFile(path.join(chapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/one' } }));
|
||||
await fs.writeFile(path.join(laterChapter, 'chapter.json'), JSON.stringify({ source: { name: 'Alpha', url: 'https://alpha.test/chapter/three' } }));
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 3, paused: true, status: 'paused', settings: validateSettings({}), queue: [], history: [], publishedRevision: 0,
|
||||
}));
|
||||
let downloadedScope = null;
|
||||
const alpha = {
|
||||
id: 'alpha', label: 'Alpha',
|
||||
match(value) { const url = new URL(value); return url.hostname === 'alpha.test' ? { source: 'alpha', id: '42', title: 'Alpha', url: value } : null; },
|
||||
async discover() { return { chapters: [{ id: 'one', number: '1' }, { id: 'two', number: '2' }, { id: 'three', number: '3' }, { id: 'four', number: '4' }] }; },
|
||||
async run(item, options) { downloadedScope = item.scope; await options.onPublished({ chapter: '4' }); return { finalPath: path.join(libraryRoot, 'Ongoing Manga'), publishedCount: 1, selectedCount: 1 }; },
|
||||
};
|
||||
const resolve = value => { const info = alpha.match(value); return info ? { adapter: alpha, info } : null; };
|
||||
const service = await new ScraperService({ libraryRoot, dataRoot, adapterFor: resolve, adapters: [alpha] }).init();
|
||||
const waitFor = async predicate => {
|
||||
const deadline = Date.now() + 3000;
|
||||
while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for discovery'); await new Promise(resolveWait => setTimeout(resolveWait, 20)); }
|
||||
};
|
||||
try {
|
||||
await service.enqueueDiscoveryChecks('test');
|
||||
await service.control('resume');
|
||||
await waitFor(() => service.state.queue[0]?.status === 'complete');
|
||||
assert.deepEqual(downloadedScope, { mode: 'ids', ids: ['four'] });
|
||||
assert.equal(service.state.queue[0].publishedCount, 1);
|
||||
const summary = (await service.publicState()).updateChecks.find(item => item.mangaId === 'Ongoing Manga' && item.source === 'alpha');
|
||||
assert.ok(summary.lastCheckedAt && summary.lastNewContentAt && summary.lastDownloadedAt);
|
||||
assert.equal(summary.lastResult, 'new-content');
|
||||
} finally {
|
||||
await service.close();
|
||||
await fs.rm(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('source-aware scheduler runs one item per source in parallel', async t => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-sources-'));
|
||||
const dataRoot = path.join(temp, 'data');
|
||||
await fs.mkdir(dataRoot, { recursive: true });
|
||||
const queue = [
|
||||
{ id: 'A1', source: 'alpha', title: 'Alpha One', url: 'https://alpha.test/1', status: 'queued', order: 0 },
|
||||
{ id: 'A2', source: 'alpha', title: 'Alpha Two', url: 'https://alpha.test/2', status: 'queued', order: 1 },
|
||||
{ id: 'B1', source: 'beta', title: 'Beta One', url: 'https://beta.test/1', status: 'queued', order: 2 },
|
||||
];
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({
|
||||
version: 2, paused: true, status: 'paused', settings: validateSettings({}), queue, history: [], publishedRevision: 0,
|
||||
}));
|
||||
const resolvers = new Map(), started = [];
|
||||
const fakeAdapter = id => ({
|
||||
id, label: id,
|
||||
match(value) { const url = new URL(value); return url.hostname === `${id}.test` ? { source: id, id: url.pathname.slice(1), title: id, url: value } : null; },
|
||||
run(item) { started.push(item.id); return new Promise(resolve => resolvers.set(item.id, () => resolve(path.join(temp, item.id)))); },
|
||||
});
|
||||
const fakeAdapters = [fakeAdapter('alpha'), fakeAdapter('beta')];
|
||||
const resolveAdapter = value => {
|
||||
for (const adapter of fakeAdapters) { const info = adapter.match(value); if (info) return { adapter, info }; }
|
||||
return null;
|
||||
};
|
||||
const service = await new ScraperService({ libraryRoot: path.join(temp, 'library'), dataRoot, adapterFor: resolveAdapter, adapters: fakeAdapters }).init();
|
||||
const waitFor = async predicate => {
|
||||
const deadline = Date.now() + 3000;
|
||||
while (!predicate()) { if (Date.now() > deadline) throw new Error('Timed out waiting for scheduler'); await new Promise(resolve => setTimeout(resolve, 20)); }
|
||||
};
|
||||
try {
|
||||
await service.control('resume');
|
||||
await waitFor(() => resolvers.has('A1') && resolvers.has('B1'));
|
||||
assert.deepEqual(new Set(started), new Set(['A1', 'B1']));
|
||||
assert.equal(resolvers.has('A2'), false);
|
||||
resolvers.get('A1')();
|
||||
await waitFor(() => resolvers.has('A2'));
|
||||
assert.equal(service.state.queue.find(item => item.id === 'A2').status, 'downloading');
|
||||
resolvers.get('A2')(); resolvers.get('B1')();
|
||||
await waitFor(() => service.state.queue.every(item => item.status === 'complete'));
|
||||
assert.equal(service.state.publishedRevision, 3);
|
||||
} finally {
|
||||
for (const resolve of resolvers.values()) resolve();
|
||||
await service.close();
|
||||
await fs.rm(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user