Checkpoint peering identity and opt-in discovery

scraper/service.js

@@ -7,6 +7,8 @@ const { adapterFor, adapters } = require('./adapters');
 const { ArchiveControlError } = require('./lib/control');
 const { sleep } = require('./lib/delay');
 const { normalizeScope, selectByScope, scopeLabel } = require('./lib/scope');
+const { normalizeWorkIdentity, resolveWorkIdentity } = require('./lib/work-identity');
+const { recordSource, recordFailure, sourceKey, ledgerRows } = require('./lib/source-ledger');
 const { scanLibrary, findManga, shouldDownload, isNewerThanLibrary, summarizeInventory, setAutoUpdate } = require('./lib/library-inventory');
 const { inspectLegacyWork, promoteLegacyWork } = require('./lib/legacy-publication');
 const { classifyFailure, safeMessage, progressKey } = require('./lib/failures');
@@ -132,6 +134,7 @@ class ScraperService {
     this.state.incidents = this.state.incidents || [];
     this.state.circuits = this.state.circuits || {};
     this.state.updateChecks = this.state.updateChecks || {};
+    this.state.sourceLedger = this.state.sourceLedger || {};
     // Older releases retained successful no-change checks in the work queue.
     // Keep their latest result before retiring those finished rows.
     for (const item of this.state.queue) {
@@ -164,6 +167,9 @@ class ScraperService {
         item.error = 'The previous adapter selected no numbered hosted English chapters. Inspect this source again to verify its edition and availability, including pilots and specials.';
         if (item.failure) item.failure = { ...item.failure, message: item.error };
       }
+      // Existing queue rows are leads. Only a recorded catalog observation may
+      // promote one to an adapter-compatible source.
+      if (!this.state.sourceLedger[sourceKey(item)]) recordSource(this.state.sourceLedger, item);
     }
     this.state.version = 4;
     this.state.currentId = null;
@@ -287,13 +293,32 @@ class ScraperService {
     const generation = this.inventoryGeneration;
     this.inventoryAttemptedAt = Date.now();
     this.inventoryRefresh = Promise.resolve().then(() => this.scanInventory(this.libraryRoot))
-      .then(inventory => {
+      .then(async inventory => {
         // A chapter or metadata edit during the scan invalidates its result.
         if (generation === this.inventoryGeneration) {
           this.inventoryCache = inventory;
           this.inventoryCachedAt = Date.now();
           this.inventoryCacheGeneration = generation;
         }
+        // Backfill source leads from existing library metadata without probing
+        // hosts or asserting that their current catalog is complete.
+        let added = 0;
+        if (generation === this.inventoryGeneration) {
+          for (const manga of inventory.mangas || []) {
+            for (const source of manga.sources || []) {
+              const found = this.resolveAdapter(source.url);
+              if (!found) continue;
+              const lead = { source: found.adapter.id, seriesId: found.info.id, url: source.url,
+                title: manga.title, libraryMangaId: manga.id,
+                workIdentity: resolveWorkIdentity(manga.meta, manga.id) };
+              const existing = this.state.sourceLedger[sourceKey(lead)];
+              if (existing && !existing.workKey?.startsWith('source:')) continue;
+              recordSource(this.state.sourceLedger, lead);
+              added++;
+            }
+          }
+        }
+        if (added) await this.persist();
         this.inventoryError = null;
         return inventory;
       }).catch(error => {
@@ -478,6 +503,10 @@ class ScraperService {
     item.sourceTitle = title;
     item.catalogTotal = catalog.chapters?.length || 0;
     item.availability = catalog.availability || { selectedChapters: item.catalogTotal, coverage: 'unknown' };
+    recordSource(this.state.sourceLedger ||= {}, item, {
+      catalogTotal: item.catalogTotal, availability: item.availability,
+      sourceTitle: title, observedAt: new Date().toISOString(),
+    });
     await this.persist();
     if (changedEdition) throw Object.assign(new Error('The source edition title changed since inspection. Inspect and confirm it again before downloading.'), { failureKind: 'unavailable' });
   }
@@ -529,6 +558,7 @@ class ScraperService {
         error: this.inventoryError,
       },
       adapters: this.adapterList.map(item => ({ id: item.id, label: item.label })),
+      sourceLedger: ledgerRows(this.state.sourceLedger),
       events: await this.recentEvents(25),
       incidents: (this.state.incidents || []).slice(-50).reverse(), circuits: this.state.circuits || {},
       runtimeHeartbeat: new Date(this.runtimeHeartbeat).toISOString(),
@@ -538,6 +568,8 @@ class ScraperService {
   async enqueue(input) {
     const found = this.resolveAdapter(input.url);
     if (!found) throw new Error('No installed scraper adapter accepts this URL');
+    const workIdentity = normalizeWorkIdentity(input.workIdentity);
+    if (input.workIdentity != null && !workIdentity) throw new Error('Invalid manga work identity');
     const scope = normalizeScope(input.scope || {});
     const policy = existingPolicy(input.existingPolicy);
     const signature = JSON.stringify([found.adapter.id, found.info.id, scope, policy, input.libraryMangaId || null]);
@@ -551,10 +583,12 @@ class ScraperService {
       metadataSearch: String(input.metadataSearch || input.title || found.info.title).trim().slice(0, 200),
       metadataSearchExplicit: !!String(input.metadataSearch || '').trim(),
       scope, scopeLabel: scopeLabel(scope), existingPolicy: policy,
+      ...(workIdentity ? { workIdentity } : {}),
       ...(input.libraryMangaId ? { libraryMangaId: String(input.libraryMangaId) } : {}),
       status: 'queued', order: this.state.queue.length, added: new Date().toISOString(),
     };
     this.state.queue.push(item);
+    recordSource(this.state.sourceLedger ||= {}, item);
     await this.event('info', `Queued ${item.title}: ${item.scopeLabel}`, { id: item.id, seriesId: item.seriesId, title: item.title });
     return item;
   }
@@ -768,6 +802,7 @@ class ScraperService {
       }
       const failure = classifyFailure(error);
       item.failure = failure; item.error = failure.message;
+      recordFailure(this.state.sourceLedger ||= {}, item, failure);
       const incident = { id: crypto.randomUUID(), at: new Date().toISOString(), jobId: item.id, source, title: item.title, failure, state: 'open' };
       this.state.incidents.push(incident); this.state.incidents = this.state.incidents.slice(-200);
       if (failure.kind === 'unavailable') { item.status = 'unavailable'; incident.state = 'resolved'; incident.resolution = 'skipped-unavailable'; }