Checkpoint peering identity and opt-in discovery
This commit is contained in:
1 parent
eba9b50e30
commit
47de081721
44 files changed
+1557
-55
No files matched your search
@@ -7,6 +7,7 @@ const { createWeebHttp } = require('../plugins/broker');
|
||||
const { discoverWeebCatalog } = require('./weebcentral');
|
||||
const { mangaDexInfo, discoverMangaDexCatalog } = require('./mangadex');
|
||||
const { ArchiveControlError } = require('../lib/control');
|
||||
const { adapterForPeer } = require('../../peering/adapter');
|
||||
|
||||
function controlledHttp(http, shouldStop) {
|
||||
return { async get(...args) {
|
||||
@@ -87,6 +88,8 @@ const adapters = [{
|
||||
}];
|
||||
|
||||
function adapterFor(url) {
|
||||
const peer = adapterForPeer(url);
|
||||
if (peer) return peer;
|
||||
for (const adapter of adapters) {
|
||||
const info = adapter.match(url);
|
||||
if (info) return { adapter, info };
|
||||
|
||||
@@ -6,6 +6,7 @@ const { assertInside } = require('./names');
|
||||
const { chapterNumber } = require('./scope');
|
||||
const { isSpecialKey, parseSpecialFolder, parseKey } = require('../../chapter-identity');
|
||||
const { writeJsonAtomic } = require('./json-file');
|
||||
const { resolveWorkIdentity, sameWorkIdentity } = require('./work-identity');
|
||||
|
||||
async function readJson(file) {
|
||||
try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return {}; }
|
||||
@@ -20,7 +21,9 @@ function keyFromName(name) { return parseKey(name)?.key || null; }
|
||||
|
||||
function sourceKey(url, name = '') {
|
||||
try {
|
||||
const host = new URL(String(url)).hostname.replace(/^www\./, '').toLowerCase();
|
||||
const parsed = new URL(String(url));
|
||||
const host = parsed.hostname.replace(/^www\./, '').toLowerCase();
|
||||
if (parsed.protocol === 'peer:' && /^p[0-9a-f]{24}$/.test(host)) return `peer-${host}`;
|
||||
if (host === 'weebcentral.com') return 'weebcentral';
|
||||
if (host === 'mangadex.org') return 'mangadex';
|
||||
return host;
|
||||
@@ -53,7 +56,7 @@ async function scanLibrary(libraryRoot) {
|
||||
const chapterMeta = await readJson(path.join(bookPath, versionEntry.name, 'chapter.json'));
|
||||
const source = typeof chapterMeta.source === 'string' ? { name: chapterMeta.source, url: '' } : (chapterMeta.source || {});
|
||||
chapters.push({
|
||||
book, number, version: versionEntry.name,
|
||||
book, bookFolder: bookEntry.name, number, version: versionEntry.name,
|
||||
sourceName: String(source.name || ''), sourceUrl: String(source.url || ''),
|
||||
sourceKey: sourceKey(source.url, source.name),
|
||||
});
|
||||
@@ -75,6 +78,13 @@ function findManga(inventory, identity = {}) {
|
||||
const anilist = identity.anilist == null ? '' : String(identity.anilist);
|
||||
const explicit = identity.libraryMangaId == null ? '' : String(identity.libraryMangaId);
|
||||
if (explicit) return inventory.mangas.find(manga => manga.id === explicit) || null;
|
||||
if (identity.workIdentity) {
|
||||
const matches = inventory.mangas.filter(manga => sameWorkIdentity(
|
||||
resolveWorkIdentity(manga.meta, manga.id, identity.localServerId), identity.workIdentity));
|
||||
if (matches.length !== 1) return null;
|
||||
if (source && seriesId && matches[0].ids[source] != null && String(matches[0].ids[source]) !== seriesId) return null;
|
||||
return matches[0];
|
||||
}
|
||||
if (source && seriesId) {
|
||||
const bySourceId = inventory.mangas.find(manga => String(manga.ids[source] ?? '') === seriesId);
|
||||
if (bySourceId) return bySourceId;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
const IMPORTED_FIELDS = ['title', 'altTitles', 'authors', 'status', 'year', 'tags', 'description'];
|
||||
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
||||
const { normalizeWorkIdentity, sameWorkIdentity } = require('./work-identity');
|
||||
|
||||
// The effective fields remain at the top of manga.json. A snapshot of the last
|
||||
// imported values lets later imports refresh unchanged fields without replacing
|
||||
@@ -25,6 +26,12 @@ function mergeImportedMetadata(prior, imported, fallbackTitle) {
|
||||
throw new Error(`AniList identity conflict: existing ${oldId}, imported ${newId}; review the manga match`);
|
||||
}
|
||||
if (imported['x-anilist']) result['x-anilist'] = { ...(prior['x-anilist'] || {}), ...imported['x-anilist'] };
|
||||
const oldWorkIdentity = normalizeWorkIdentity(prior['x-work-identity']);
|
||||
const newWorkIdentity = normalizeWorkIdentity(imported['x-work-identity']);
|
||||
if (oldWorkIdentity && newWorkIdentity && !sameWorkIdentity(oldWorkIdentity, newWorkIdentity)) {
|
||||
throw new Error(`MangaCircus work identity conflict: existing ${oldWorkIdentity.kind} identity differs from imported ${newWorkIdentity.kind} identity`);
|
||||
}
|
||||
if (newWorkIdentity) result['x-work-identity'] = newWorkIdentity;
|
||||
const snapshot = { ...previous };
|
||||
for (const field of IMPORTED_FIELDS) if (Object.hasOwn(imported, field)) snapshot[field] = imported[field];
|
||||
if (Object.keys(snapshot).length) result['x-metadata'] = { ...(prior['x-metadata'] || {}), imported: snapshot };
|
||||
|
||||
@@ -10,6 +10,7 @@ const { cleanPart, chapterFolder, assertInside } = require('./names');
|
||||
const { sizeFromBuffer, imageSize } = require('../../library');
|
||||
const { directionFields } = require('../../reading-direction');
|
||||
const { mergeImportedMetadata } = require('./metadata-merge');
|
||||
const { normalizeWorkIdentity } = require('./work-identity');
|
||||
|
||||
const token = () => crypto.randomBytes(5).toString('hex');
|
||||
// Staging is a recovery asset, not disposable garbage. Legacy stages without an
|
||||
@@ -204,7 +205,16 @@ async function publishManifest(manifest, options = {}) {
|
||||
const isNew = !mangaDir;
|
||||
const finalName = cleanPart(manifest.series.folder || manifest.series.title);
|
||||
const finalDir = assertInside(libraryRoot, path.join(libraryRoot, finalName));
|
||||
if (!mangaDir && await exists(finalDir)) throw new Error(`Folder already belongs to another work: ${finalDir}`);
|
||||
if (!mangaDir && await exists(finalDir)) {
|
||||
const existing = await readMetadata(path.join(finalDir, 'manga.json'));
|
||||
const incomingIdentity = normalizeWorkIdentity(manifest.series.metadata?.['x-work-identity']);
|
||||
const existingIdentity = normalizeWorkIdentity(existing['x-work-identity']);
|
||||
if (incomingIdentity && existingIdentity && incomingIdentity.kind === existingIdentity.kind
|
||||
&& incomingIdentity.value === existingIdentity.value) {
|
||||
throw new Error(`Work identity matches the existing “${existing.title || finalName}” folder, but its source association was not resolved. Review the existing manga target; no files were overwritten.`);
|
||||
}
|
||||
throw new Error(`A library folder named “${finalName}” already exists, but this download could not be safely matched to it. No files were overwritten. Choose the existing manga as the target or review its work identity before syncing.`);
|
||||
}
|
||||
mangaDir = mangaDir || path.join(libraryRoot, `.staging-mcscraper-resume-${digest([finalName, [...sourceUrls].sort()])}`);
|
||||
await fsp.mkdir(mangaDir, { recursive: true, mode: 0o755 });
|
||||
const http = options.httpClient || new SlowHttp(options.http || manifest.http || {});
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
'use strict';
|
||||
|
||||
const { normalizeWorkIdentity } = require('./work-identity');
|
||||
|
||||
function workKey(item) {
|
||||
const identity = normalizeWorkIdentity(item.workIdentity);
|
||||
if (identity) return `${identity.scheme}:${identity.kind}:${identity.value}`;
|
||||
if (item.libraryMangaId) return `library:${item.libraryMangaId}`;
|
||||
return `source:${item.source}:${item.seriesId || item.url}`;
|
||||
}
|
||||
|
||||
function sourceKey(item) {
|
||||
return `${item.source}:${item.seriesId || item.url}`;
|
||||
}
|
||||
|
||||
function recordSource(ledger, item, observation = {}) {
|
||||
const key = sourceKey(item);
|
||||
const prior = ledger[key] || {};
|
||||
const availability = observation.availability || item.availability;
|
||||
const count = observation.catalogTotal ?? item.catalogTotal;
|
||||
const observedAt = observation.observedAt || (count != null ? item.checked || item.completed : null) || null;
|
||||
const catalog = count != null ? 'found' : prior.catalog || 'unverified';
|
||||
const adapter = count != null
|
||||
? Number(count) > 0 ? 'compatible' : availability?.reason === 'unrecognized-labels' ? 'unsupported-labels' : 'no-selected-pages'
|
||||
: prior.adapter || 'unverified';
|
||||
ledger[key] = {
|
||||
...prior, workKey: workKey(item), source: item.source, seriesId: item.seriesId || null,
|
||||
url: item.url || prior.url || null, title: item.title || prior.title || null,
|
||||
sourceTitle: observation.sourceTitle || item.sourceTitle || prior.sourceTitle || null,
|
||||
language: availability?.language || prior.language || null,
|
||||
catalog, adapter, rights: prior.rights || 'unverified',
|
||||
hostedCount: count == null ? prior.hostedCount ?? null : Number(count),
|
||||
externalCount: availability?.externalReleases ?? prior.externalCount ?? null,
|
||||
coverage: availability?.coverage || prior.coverage || 'unknown',
|
||||
reason: observation.availability ? availability.reason || null : availability?.reason || prior.reason || null,
|
||||
lastObservedAt: observedAt || prior.lastObservedAt || null,
|
||||
evidence: count != null && observedAt ? { kind: 'scraper-catalog', jobId: item.id || null } : prior.evidence || { kind: 'queue-url', jobId: item.id || null },
|
||||
};
|
||||
if (observation.catalogTotal != null && Number(count) > 0) delete ledger[key].lastFailure;
|
||||
return ledger[key];
|
||||
}
|
||||
|
||||
function recordFailure(ledger, item, failure, at = new Date().toISOString()) {
|
||||
const row = recordSource(ledger, item);
|
||||
row.lastFailure = { kind: failure.kind, message: String(failure.message || '').slice(0, 300), at };
|
||||
return row;
|
||||
}
|
||||
|
||||
function ledgerRows(ledger) {
|
||||
return Object.values(ledger || {}).sort((a, b) => String(a.title || '').localeCompare(String(b.title || ''))
|
||||
|| String(a.source || '').localeCompare(String(b.source || '')));
|
||||
}
|
||||
|
||||
module.exports = { workKey, sourceKey, recordSource, recordFailure, ledgerRows };
|
||||
@@ -0,0 +1,49 @@
|
||||
'use strict';
|
||||
|
||||
const crypto = require('node:crypto');
|
||||
|
||||
const SCHEME = 'mangacircus-work-v1';
|
||||
const KINDS = new Set(['anilist', 'mangadex', 'weebcentral', 'hoster', 'origin']);
|
||||
|
||||
function normalizeWorkIdentity(value) {
|
||||
if (!value || typeof value !== 'object' || value.scheme !== SCHEME
|
||||
|| !KINDS.has(value.kind) || typeof value.value !== 'string'
|
||||
|| !value.value.trim() || value.value.length > 300) return null;
|
||||
return { scheme: SCHEME, kind: value.kind, value: value.value.trim() };
|
||||
}
|
||||
|
||||
function resolveWorkIdentity(meta = {}, mangaId, serverId) {
|
||||
const carried = normalizeWorkIdentity(meta['x-work-identity']);
|
||||
if (carried) return carried;
|
||||
|
||||
const anilist = meta['x-anilist']?.id ?? meta['x-scraper']?.ids?.anilist;
|
||||
if (anilist != null && String(anilist).trim()) return { scheme: SCHEME, kind: 'anilist', value: String(anilist).trim() };
|
||||
|
||||
const ids = meta['x-scraper']?.ids || {};
|
||||
for (const source of ['mangadex', 'weebcentral']) {
|
||||
if (ids[source] != null && String(ids[source]).trim()) {
|
||||
return { scheme: SCHEME, kind: source, value: String(ids[source]).trim() };
|
||||
}
|
||||
}
|
||||
for (const source of Object.keys(ids).sort()) {
|
||||
if (!['anilist', 'mangadex', 'weebcentral'].includes(source) && !source.startsWith('peer-')
|
||||
&& ids[source] != null && String(ids[source]).trim()) {
|
||||
return { scheme: SCHEME, kind: 'hoster', value: `${source}:${String(ids[source]).trim()}` };
|
||||
}
|
||||
}
|
||||
|
||||
// A private, stable lineage ID lets a title safely make a round trip even if
|
||||
// neither server has external metadata. It reveals neither the folder nor title.
|
||||
if (typeof mangaId === 'string' && mangaId && typeof serverId === 'string' && serverId) {
|
||||
const value = crypto.createHash('sha256').update(`${serverId}\0${mangaId}`).digest('hex');
|
||||
return { scheme: SCHEME, kind: 'origin', value };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function sameWorkIdentity(left, right) {
|
||||
const a = normalizeWorkIdentity(left), b = normalizeWorkIdentity(right);
|
||||
return !!a && !!b && a.scheme === b.scheme && a.kind === b.kind && a.value === b.value;
|
||||
}
|
||||
|
||||
module.exports = { SCHEME, normalizeWorkIdentity, resolveWorkIdentity, sameWorkIdentity };
|
||||
+36
-1
@@ -7,6 +7,8 @@ const { adapterFor, adapters } = require('./adapters');
|
||||
const { ArchiveControlError } = require('./lib/control');
|
||||
const { sleep } = require('./lib/delay');
|
||||
const { normalizeScope, selectByScope, scopeLabel } = require('./lib/scope');
|
||||
const { normalizeWorkIdentity, resolveWorkIdentity } = require('./lib/work-identity');
|
||||
const { recordSource, recordFailure, sourceKey, ledgerRows } = require('./lib/source-ledger');
|
||||
const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, summarizeInventory, setAutoUpdate } = require('./lib/library-inventory');
|
||||
const { inspectLegacyWork, promoteLegacyWork } = require('./lib/legacy-publication');
|
||||
const { classifyFailure, safeMessage, progressKey } = require('./lib/failures');
|
||||
@@ -132,6 +134,7 @@ class ScraperService {
|
||||
this.state.incidents = this.state.incidents || [];
|
||||
this.state.circuits = this.state.circuits || {};
|
||||
this.state.updateChecks = this.state.updateChecks || {};
|
||||
this.state.sourceLedger = this.state.sourceLedger || {};
|
||||
// Older releases retained successful no-change checks in the work queue.
|
||||
// Keep their latest result before retiring those finished rows.
|
||||
for (const item of this.state.queue) {
|
||||
@@ -164,6 +167,9 @@ class ScraperService {
|
||||
item.error = 'The previous adapter selected no numbered hosted English chapters. Inspect this source again to verify its edition and availability, including pilots and specials.';
|
||||
if (item.failure) item.failure = { ...item.failure, message: item.error };
|
||||
}
|
||||
// Existing queue rows are leads. Only a recorded catalog observation may
|
||||
// promote one to an adapter-compatible source.
|
||||
if (!this.state.sourceLedger[sourceKey(item)]) recordSource(this.state.sourceLedger, item);
|
||||
}
|
||||
this.state.version = 4;
|
||||
this.state.currentId = null;
|
||||
@@ -287,13 +293,32 @@ class ScraperService {
|
||||
const generation = this.inventoryGeneration;
|
||||
this.inventoryAttemptedAt = Date.now();
|
||||
this.inventoryRefresh = Promise.resolve().then(() => this.scanInventory(this.libraryRoot))
|
||||
.then(inventory => {
|
||||
.then(async inventory => {
|
||||
// A chapter or metadata edit during the scan invalidates its result.
|
||||
if (generation === this.inventoryGeneration) {
|
||||
this.inventoryCache = inventory;
|
||||
this.inventoryCachedAt = Date.now();
|
||||
this.inventoryCacheGeneration = generation;
|
||||
}
|
||||
// Backfill source leads from existing library metadata without probing
|
||||
// hosts or asserting that their current catalog is complete.
|
||||
let added = 0;
|
||||
if (generation === this.inventoryGeneration) {
|
||||
for (const manga of inventory.mangas || []) {
|
||||
for (const source of manga.sources || []) {
|
||||
const found = this.resolveAdapter(source.url);
|
||||
if (!found) continue;
|
||||
const lead = { source: found.adapter.id, seriesId: found.info.id, url: source.url,
|
||||
title: manga.title, libraryMangaId: manga.id,
|
||||
workIdentity: resolveWorkIdentity(manga.meta, manga.id) };
|
||||
const existing = this.state.sourceLedger[sourceKey(lead)];
|
||||
if (existing && !existing.workKey?.startsWith('source:')) continue;
|
||||
recordSource(this.state.sourceLedger, lead);
|
||||
added++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (added) await this.persist();
|
||||
this.inventoryError = null;
|
||||
return inventory;
|
||||
}).catch(error => {
|
||||
@@ -478,6 +503,10 @@ class ScraperService {
|
||||
item.sourceTitle = title;
|
||||
item.catalogTotal = catalog.chapters?.length || 0;
|
||||
item.availability = catalog.availability || { selectedChapters: item.catalogTotal, coverage: 'unknown' };
|
||||
recordSource(this.state.sourceLedger ||= {}, item, {
|
||||
catalogTotal: item.catalogTotal, availability: item.availability,
|
||||
sourceTitle: title, observedAt: new Date().toISOString(),
|
||||
});
|
||||
await this.persist();
|
||||
if (changedEdition) throw Object.assign(new Error('The source edition title changed since inspection. Inspect and confirm it again before downloading.'), { failureKind: 'unavailable' });
|
||||
}
|
||||
@@ -529,6 +558,7 @@ class ScraperService {
|
||||
error: this.inventoryError,
|
||||
},
|
||||
adapters: this.adapterList.map(item => ({ id: item.id, label: item.label })),
|
||||
sourceLedger: ledgerRows(this.state.sourceLedger),
|
||||
events: await this.recentEvents(25),
|
||||
incidents: (this.state.incidents || []).slice(-50).reverse(), circuits: this.state.circuits || {},
|
||||
runtimeHeartbeat: new Date(this.runtimeHeartbeat).toISOString(),
|
||||
@@ -538,6 +568,8 @@ class ScraperService {
|
||||
async enqueue(input) {
|
||||
const found = this.resolveAdapter(input.url);
|
||||
if (!found) throw new Error('No installed scraper adapter accepts this URL');
|
||||
const workIdentity = normalizeWorkIdentity(input.workIdentity);
|
||||
if (input.workIdentity != null && !workIdentity) throw new Error('Invalid manga work identity');
|
||||
const scope = normalizeScope(input.scope || {});
|
||||
const policy = existingPolicy(input.existingPolicy);
|
||||
const signature = JSON.stringify([found.adapter.id, found.info.id, scope, policy, input.libraryMangaId || null]);
|
||||
@@ -551,10 +583,12 @@ class ScraperService {
|
||||
metadataSearch: String(input.metadataSearch || input.title || found.info.title).trim().slice(0, 200),
|
||||
metadataSearchExplicit: !!String(input.metadataSearch || '').trim(),
|
||||
scope, scopeLabel: scopeLabel(scope), existingPolicy: policy,
|
||||
...(workIdentity ? { workIdentity } : {}),
|
||||
...(input.libraryMangaId ? { libraryMangaId: String(input.libraryMangaId) } : {}),
|
||||
status: 'queued', order: this.state.queue.length, added: new Date().toISOString(),
|
||||
};
|
||||
this.state.queue.push(item);
|
||||
recordSource(this.state.sourceLedger ||= {}, item);
|
||||
await this.event('info', `Queued ${item.title}: ${item.scopeLabel}`, { id: item.id, seriesId: item.seriesId, title: item.title });
|
||||
return item;
|
||||
}
|
||||
@@ -768,6 +802,7 @@ class ScraperService {
|
||||
}
|
||||
const failure = classifyFailure(error);
|
||||
item.failure = failure; item.error = failure.message;
|
||||
recordFailure(this.state.sourceLedger ||= {}, item, failure);
|
||||
const incident = { id: crypto.randomUUID(), at: new Date().toISOString(), jobId: item.id, source, title: item.title, failure, state: 'open' };
|
||||
this.state.incidents.push(incident); this.state.incidents = this.state.incidents.slice(-200);
|
||||
if (failure.kind === 'unavailable') { item.status = 'unavailable'; incident.state = 'resolved'; incident.resolution = 'skipped-unavailable'; }
|
||||
|
||||
@@ -87,6 +87,8 @@ test('metadata refresh preserves legacy and manual fields while updating untouch
|
||||
const third = mergeImportedMetadata(second, { description: 'First', 'x-anilist': { id: 42 } }, 'Fallback');
|
||||
assert.equal(mergeImportedMetadata(third, { description: 'Revised', 'x-anilist': { id: 42 } }, 'Fallback').description, 'Revised');
|
||||
assert.throws(() => mergeImportedMetadata(first, { 'x-anilist': { id: 99 } }, 'Fallback'), /identity conflict/);
|
||||
assert.throws(() => mergeImportedMetadata({ 'x-work-identity': { scheme: 'mangacircus-work-v1', kind: 'origin', value: 'a'.repeat(64) } },
|
||||
{ 'x-work-identity': { scheme: 'mangacircus-work-v1', kind: 'origin', value: 'b'.repeat(64) } }, 'Fallback'), /work identity conflict/);
|
||||
});
|
||||
|
||||
test('publisher rejects a conflicting work ID before fetching any pages', async t => {
|
||||
@@ -107,6 +109,23 @@ test('publisher rejects a conflicting work ID before fetching any pages', async
|
||||
assert.equal((await fs.readdir(manga)).includes('v01'), false);
|
||||
});
|
||||
|
||||
test('publisher explains an unresolved same-title folder collision without touching its contents', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mcscraper-title-collision-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
const manga = path.join(root, 'Same Title');
|
||||
await fs.mkdir(manga);
|
||||
await fs.writeFile(path.join(manga, 'manga.json'), JSON.stringify({ title: 'Same Title' }));
|
||||
await fs.writeFile(path.join(manga, 'operator-note.txt'), 'keep');
|
||||
let requested = 0;
|
||||
await assert.rejects(publishManifest({ version: 1,
|
||||
series: { title: 'Same Title', direction: 'rtl', sources: [{ name: 'Peer', url: 'peer://p123/m123' }] },
|
||||
chapters: [{ number: '1', source: { name: 'Peer', url: 'peer://p123/m123' }, pages: [{ url: 'https://example.test/page.png' }] }],
|
||||
}, { libraryRoot: root, httpClient: { async get() { requested++; throw new Error('unexpected request'); } } }),
|
||||
/could not be safely matched.*No files were overwritten.*Choose the existing manga/s);
|
||||
assert.equal(requested, 0);
|
||||
assert.equal(await fs.readFile(path.join(manga, 'operator-note.txt'), 'utf8'), 'keep');
|
||||
});
|
||||
|
||||
test('queue scopes select exact catalog slices', () => {
|
||||
const chapters = [
|
||||
{ id: 'a', number: '1', volume: '1' }, { id: 'b', number: '2', volume: '1' },
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { recordSource, recordFailure, ledgerRows } = require('../lib/source-ledger');
|
||||
const { ScraperService } = require('../service');
|
||||
|
||||
test('a URL lead, a parsed catalog, and rights remain separate states', () => {
|
||||
const ledger = {};
|
||||
const item = { id: 'job', source: 'weebcentral', seriesId: 'series', url: 'https://weebcentral.com/series/series', title: 'Example' };
|
||||
assert.equal(recordSource(ledger, item).catalog, 'unverified');
|
||||
const row = recordSource(ledger, item, { catalogTotal: 0, availability: { reason: 'unrecognized-labels', coverage: 'unknown' },
|
||||
observedAt: '2026-10-13T12:00:00Z' });
|
||||
assert.equal(row.catalog, 'found');
|
||||
assert.equal(row.adapter, 'unsupported-labels');
|
||||
assert.equal(row.rights, 'unverified');
|
||||
assert.equal(row.hostedCount, 0);
|
||||
recordFailure(ledger, item, { kind: 'adapter', message: 'Parser could not read labels' });
|
||||
assert.equal(ledger['weebcentral:series'].lastFailure.kind, 'adapter');
|
||||
assert.equal(ledgerRows(ledger).length, 1);
|
||||
});
|
||||
|
||||
test('a catalog observation survives completed queue cleanup and service restart', async t => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'mc-source-ledger-'));
|
||||
const dataRoot = path.join(root, 'data'), libraryRoot = path.join(root, 'library');
|
||||
await fs.mkdir(dataRoot);
|
||||
await fs.writeFile(path.join(dataRoot, 'queue.json'), JSON.stringify({ paused: true, queue: [],
|
||||
settings: { autoUpdateEnabled: false } }));
|
||||
const adapter = { id: 'fixture', label: 'Fixture', match: url => ({ source: 'fixture', id: url.split('/').at(-1), url, title: 'Example' }) };
|
||||
const options = { dataRoot, libraryRoot, adapters: [adapter], adapterFor: url => ({ adapter, info: adapter.match(url) }) };
|
||||
let service = await new ScraperService(options).init();
|
||||
t.after(async () => { await service.close(); await fs.rm(root, { recursive: true, force: true }); });
|
||||
const item = await service.enqueue({ url: 'https://example.test/series', inspect: true });
|
||||
assert.equal(service.state.sourceLedger['fixture:series'].catalog, 'unverified');
|
||||
await service.recordCatalog(item, { title: 'Canonical Example', chapters: [{ id: 'one' }],
|
||||
availability: { language: 'en', coverage: 'unknown' } });
|
||||
item.kind = 'download'; item.status = 'complete';
|
||||
await service.clearCompleted();
|
||||
await service.close();
|
||||
service = await new ScraperService(options).init();
|
||||
const [row] = (await service.publicState()).sourceLedger;
|
||||
assert.equal(service.state.queue.length, 0);
|
||||
assert.equal(row.sourceTitle, 'Canonical Example');
|
||||
assert.equal(row.adapter, 'compatible');
|
||||
assert.equal(row.hostedCount, 1);
|
||||
assert.equal(row.rights, 'unverified');
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
'use strict';
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { normalizeWorkIdentity, resolveWorkIdentity, sameWorkIdentity } = require('../lib/work-identity');
|
||||
const { findManga } = require('../lib/library-inventory');
|
||||
|
||||
test('work identity follows canonical source priority and preserves an imported lineage', () => {
|
||||
const anilist = { scheme: 'mangacircus-work-v1', kind: 'anilist', value: '123' };
|
||||
const meta = { 'x-anilist': { id: 123 }, 'x-scraper': { ids: { mangadex: 'md-1', weebcentral: 'wc-1' } } };
|
||||
assert.deepEqual(resolveWorkIdentity(meta, 'folder', 'server'), anilist);
|
||||
const imported = { scheme: 'mangacircus-work-v1', kind: 'origin', value: 'a'.repeat(64) };
|
||||
assert.deepEqual(resolveWorkIdentity({ ...meta, 'x-work-identity': imported }, 'other', 'other-server'), imported);
|
||||
assert.equal(normalizeWorkIdentity({ ...imported, value: '' }), null);
|
||||
assert.equal(sameWorkIdentity(imported, { ...imported }), true);
|
||||
assert.equal(sameWorkIdentity(imported, { ...imported, value: 'b'.repeat(64) }), false);
|
||||
assert.equal(resolveWorkIdentity({ 'x-scraper': { ids: { 'peer-p123': 'edition-only' } } }, 'folder', 'server').kind,
|
||||
'origin', 'A peer edition ID is not a canonical work identity');
|
||||
});
|
||||
|
||||
test('library matching uses a peer-carried stable identity before title or peer-edition IDs', () => {
|
||||
const localServerId = 'p-local-server';
|
||||
const inventory = { mangas: [
|
||||
{ id: 'Peer A Test Manga', meta: { title: 'Peer A Test Manga' }, ids: {}, sources: [], chapters: [] },
|
||||
{ id: 'Unrelated edition', meta: { title: 'Peer A Test Manga' }, ids: { 'peer-p-other': 'p-other-manga' }, sources: [], chapters: [] },
|
||||
] };
|
||||
const identity = resolveWorkIdentity(inventory.mangas[0].meta, inventory.mangas[0].id, localServerId);
|
||||
assert.equal(findManga(inventory, { source: 'peer-p-returning', seriesId: 'p-returning-manga',
|
||||
workIdentity: identity, localServerId }).id, 'Peer A Test Manga');
|
||||
assert.equal(findManga(inventory, { source: 'peer-p-returning', seriesId: 'p-returning-manga',
|
||||
workIdentity: { ...identity, value: 'f'.repeat(64) }, localServerId }), null,
|
||||
'A matching title alone must not merge unrelated work');
|
||||
});
|
||||
Reference in new issue
Block a user