Consolidate MangaCircus app, scraper, and Compose baseline

scraper/test/mangadex-specials.test.js

@@ -0,0 +1,135 @@
+'use strict';
+const test = require('node:test'), assert = require('node:assert/strict');
+const fs = require('node:fs/promises'), os = require('node:os'), path = require('node:path');
+const { specialKey, parseSpecialFolder } = require('../../chapter-identity');
+const { selectChapterReleases, availabilitySummary, discoverMangaDexCatalog, discoverMangaDexChapter } = require('../adapters/mangadex');
+const { selectByScope } = require('../lib/scope');
+const inventory = require('../lib/library-inventory');
+const { publishManifest } = require('../lib/publisher');
+const reader = require('../../library');
+const { ScraperService } = require('../service');
+const ids = ['37d6949a-c7ce-4855-b838-4fd572e87629', '27dd738c-eca2-488d-a791-53ab4b24313b'];
+const titleId = '06a37094-b3ba-4f25-bf2b-54c9751e0e91';
+const release = (id, chapter = null) => ({ id, attributes: { chapter, pages: 46, title: 'Pilot', externalUrl: null }, relationships: [] });
+const pause = ms => new Promise(resolve => setTimeout(resolve, ms));
+async function until(predicate) {
+  const end = Date.now() + 5000;
+  while (!predicate()) { if (Date.now() > end) throw new Error('Fixture timed out'); await pause(20); }
+}
+
+test('unnumbered releases retain source IDs, do not invent chapter zero, and remain distinct', async () => {
+  const selected = selectChapterReleases(ids.map(id => release(id)), 'Pilot');
+  assert.equal(selected.length, 2);
+  assert.deepEqual(new Set(selected.map(r => r.number)), new Set(ids.map(specialKey)));
+  assert.ok(selected.every(r => r.special && r.label === 'Pilot' && r.sourceNumber === null));
+  assert.equal(selectByScope(selected, { mode: 'chapter', chapter: '0' }).length, 0);
+  assert.equal(selectByScope(selected, { mode: 'after', chapter: '1' }).length, 0);
+  assert.equal(selectByScope(selected, { mode: 'missing' }).length, 2);
+  assert.equal(selectByScope(selected, { mode: 'ids', ids: [ids[0]] })[0].id, ids[0]);
+  assert.throws(() => specialKey('../escape'));
+  assert.equal(parseSpecialFolder(specialKey(ids[0]) + ' Pilot [MangaDex]').src, 'MangaDex');
+  const chapter = await discoverMangaDexChapter(selected[0], { async get() { return { async json() { return { result: 'ok', baseUrl: 'https://uploads.mangadex.org', chapter: { hash: 'fixture', data: Array.from({ length: 46 }, (_, n) => `${n}.png`) } }; } }; } });
+  assert.equal(chapter.label, 'Pilot'); assert.equal(chapter.sourceId, selected[0].id);
+});
+
+test('availability distinguishes no English, external-only, and partial hosted catalogs', () => {
+  assert.equal(availabilitySummary({ availableTranslatedLanguages: ['ar'] }, [], [], 0).reason, 'no-english');
+  const external = availabilitySummary({}, [], [], 8);
+  assert.equal(external.reason, 'external-only'); assert.match(external.message, /8 English external-link/);
+  const chapters = selectChapterReleases([release('regular-fixture', '336')]);
+  const summary = availabilitySummary({ lastChapter: '686' }, [release('regular-fixture', '336')], chapters);
+  assert.equal(summary.coverage, 'partial'); assert.equal(summary.selectedChapters, 1);
+  assert.equal(availabilitySummary({ lastChapter: '336' }, [], chapters).coverage, 'unknown', 'Highest number alone does not prove completeness');
+});
+
+test('catalog uses separate external-only feed only when hosted selections are empty', async () => {
+  const modes = [];
+  const http = { async get(url) {
+    const parsed = new URL(url), mode = parsed.searchParams.get('includeExternalUrl');
+    if (!mode) return { async json() { return { data: { type: 'manga', attributes: { title: { en: 'CLAYMORE' }, lastChapter: '155' } } }; } };
+    modes.push(mode);
+    return { async json() { return { result: 'ok', data: [], total: mode === '1' ? 8 : 0 }; } };
+  } };
+  const catalog = await discoverMangaDexCatalog(`https://mangadex.org/title/${titleId}`, http);
+  assert.deepEqual(modes, ['0', '1']); assert.equal(catalog.availability.reason, 'external-only');
+  modes.length = 0;
+  http.get = async url => {
+    const mode = new URL(url).searchParams.get('includeExternalUrl');
+    if (!mode) return { async json() { return { data: { type: 'manga', attributes: { title: { en: 'Bleach (Pre-Serialization)' } } } }; } };
+    modes.push(mode); return { async json() { return { result: 'ok', data: ids.map(id => release(id)), total: 2 }; } };
+  };
+  const pilot = await discoverMangaDexCatalog(`https://mangadex.org/title/${titleId}`, http);
+  assert.deepEqual(modes, ['0']); assert.equal(pilot.availability.specialReleases, 2); assert.equal(pilot.chapters[0].label, 'Pilot');
+});
+
+test('special publication is atomic, readable, stable across rescans and duplicate-safe', async t => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-special-'));
+  t.after(() => fs.rm(root, { recursive: true, force: true }));
+  // Real fixture PNG; no upstream image requests.
+  const { Canvas } = require('../../tools/png');
+  const page = path.join(root, '.fixture.png'); await fs.writeFile(page, new Canvas(10, 20, [1, 2, 3]).png());
+  const manifest = { version: 1, series: { title: 'Pilot edition', direction: 'rtl', sources: [{ name: 'MangaDex', url: `https://mangadex.org/title/${titleId}` }] },
+    chapters: selectChapterReleases(ids.map(id => release(id)), 'Pilot').map(ref => ({ ...ref, source: { name: 'MangaDex', url: ref.url }, pages: [{ path: page }] })) };
+  const first = await publishManifest(manifest, { libraryRoot: root });
+  assert.ok(first.results.every(result => result.status === 'published'));
+  const scanned = (await reader.scanLibrary(root)).index.mangas[0];
+  assert.equal(scanned.chapters.length, 2); assert.ok(scanned.chapters.every(c => c.label === 'Pilot' && c.num === null && !c.extra));
+  const stableIds = scanned.chapters.map(c => c.id);
+  assert.deepEqual((await publishManifest(manifest, { libraryRoot: root })).results.map(r => r.status), ['skipped', 'skipped']);
+  assert.deepEqual((await reader.scanLibrary(root)).index.mangas[0].chapters.map(c => c.id), stableIds);
+  const manga = (await inventory.scanLibrary(root)).mangas[0];
+  assert.equal(inventory.shouldDownload(manga, manifest.chapters[0], 'mangadex'), false);
+  assert.equal(inventory.isNewerThanLibrary(manga, manifest.chapters[0]), false);
+  assert.equal(inventory.isNewerThanLibrary(manga, { number: '1' }), true);
+  assert.equal(reader.parseKey('c0012.5 Existing [Host]').key, '12.5');
+  assert.equal(reader.parseKey('v01').key, '1');
+  const regular = { ...manifest, chapters: [{ number: '12.5', title: 'Existing chapter', source: { name: 'Host', url: 'https://host.test/12' }, pages: [{ path: page }] }] };
+  await publishManifest(regular, { libraryRoot: root });
+  assert.ok((await reader.scanLibrary(root)).index.mangas[0].chapterById.has('1/12.5'));
+});
+
+test('legacy unavailable messages become unverified without retrying or downloading', async t => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-unavailable-')), dataRoot = path.join(root, 'data');
+  await fs.mkdir(dataRoot); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ paused: true,
+    queue: [{ id: 'old-job', source: 'mangadex', url: `https://mangadex.org/title/${titleId}`, status: 'unavailable', error: 'No downloadable English MangaDex chapters found', failure: { kind: 'unavailable', scope: 'job' } }],
+    settings: { autoUpdateEnabled: false } }));
+  const service = await new ScraperService({ libraryRoot: path.join(root, 'library'), dataRoot }).init();
+  t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
+  assert.equal(service.state.queue[0].status, 'unavailable'); assert.equal(service.active.size, 0);
+  assert.match(service.state.queue[0].error, /Inspect this source again/);
+  assert.equal(service.state.queue[0].failure.kind, 'unavailable');
+});
+
+test('inspection waits for confirmation, survives restart, and uses one normal source lane', async t => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-inspect-')), dataRoot = path.join(root, 'data');
+  await fs.mkdir(dataRoot); await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ paused: true, queue: [], settings: { autoUpdateEnabled: false } }));
+  let running = 0, maxRunning = 0, downloads = 0;
+  const catalog = { title: 'Canonical Pilot', chapters: ids.map(id => ({ id, number: specialKey(id), special: true })), availability: { selectedChapters: 2, specialReleases: 2, coverage: 'unknown' } };
+  const adapter = { id: 'fixture', label: 'Fixture', match: url => ({ source: 'fixture', id: url.split('/').pop(), url, title: 'Misleading slug' }),
+    async discover() { maxRunning = Math.max(maxRunning, ++running); await pause(50); running--; return catalog; },
+    async run(item, options) { downloads++; await options.onCatalog(catalog); return { publishedCount: 1, selectedCount: 1 }; } };
+  const options = { libraryRoot: path.join(root, 'library'), dataRoot, adapters: [adapter], adapterFor: url => ({ adapter, info: adapter.match(url) }) };
+  let service = await new ScraperService(options).init();
+  t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
+  const item = await service.enqueue({ url: 'https://fixture.test/one', inspect: true });
+  const other = await service.enqueue({ url: 'https://fixture.test/two', inspect: true });
+  await assert.rejects(service.control('confirm', item.id), /Inspect the source first/);
+  await service.control('resume'); await until(() => item.status === 'ready' && other.status === 'ready');
+  assert.equal(downloads, 0); assert.equal(maxRunning, 1); assert.equal(item.sourceTitle, 'Canonical Pilot');
+  await service.close(); service = await new ScraperService(options).init();
+  const restored = service.state.queue.find(q => q.id === item.id);
+  assert.equal(restored.kind, 'inspection'); assert.equal(restored.status, 'ready');
+  await service.control('confirm', restored.id); await until(() => restored.status === 'complete');
+  await until(() => service.active.size === 0);
+  assert.equal(downloads, 1); assert.equal(restored.selectedCount, 1);
+  assert.equal(service.state.history.at(-1).availability.specialReleases, 2);
+  service.state.sourcePaused.fixture = true;
+  await assert.rejects(service.control('inspect', restored.id), /access block/);
+});
+
+test('edition change after confirmation cannot silently download another edition', async () => {
+  const service = Object.create(ScraperService.prototype);
+  service.persist = async () => {};
+  const item = { title: 'slug', confirmedSourceTitle: 'Pilot edition' };
+  await assert.rejects(service.recordCatalog(item, { title: 'Main edition', chapters: [] }), /edition title changed/);
+});