Checkpoint peering identity and opt-in discovery

scraper/lib/source-ledger.js

@@ -0,0 +1,54 @@
+'use strict';
+
+const { normalizeWorkIdentity } = require('./work-identity');
+
+function workKey(item) {
+  const identity = normalizeWorkIdentity(item.workIdentity);
+  if (identity) return `${identity.scheme}:${identity.kind}:${identity.value}`;
+  if (item.libraryMangaId) return `library:${item.libraryMangaId}`;
+  return `source:${item.source}:${item.seriesId || item.url}`;
+}
+
+function sourceKey(item) {
+  return `${item.source}:${item.seriesId || item.url}`;
+}
+
+function recordSource(ledger, item, observation = {}) {
+  const key = sourceKey(item);
+  const prior = ledger[key] || {};
+  const availability = observation.availability || item.availability;
+  const count = observation.catalogTotal ?? item.catalogTotal;
+  const observedAt = observation.observedAt || (count != null ? item.checked || item.completed : null) || null;
+  const catalog = count != null ? 'found' : prior.catalog || 'unverified';
+  const adapter = count != null
+    ? Number(count) > 0 ? 'compatible' : availability?.reason === 'unrecognized-labels' ? 'unsupported-labels' : 'no-selected-pages'
+    : prior.adapter || 'unverified';
+  ledger[key] = {
+    ...prior, workKey: workKey(item), source: item.source, seriesId: item.seriesId || null,
+    url: item.url || prior.url || null, title: item.title || prior.title || null,
+    sourceTitle: observation.sourceTitle || item.sourceTitle || prior.sourceTitle || null,
+    language: availability?.language || prior.language || null,
+    catalog, adapter, rights: prior.rights || 'unverified',
+    hostedCount: count == null ? prior.hostedCount ?? null : Number(count),
+    externalCount: availability?.externalReleases ?? prior.externalCount ?? null,
+    coverage: availability?.coverage || prior.coverage || 'unknown',
+    reason: observation.availability ? availability.reason || null : availability?.reason || prior.reason || null,
+    lastObservedAt: observedAt || prior.lastObservedAt || null,
+    evidence: count != null && observedAt ? { kind: 'scraper-catalog', jobId: item.id || null } : prior.evidence || { kind: 'queue-url', jobId: item.id || null },
+  };
+  if (observation.catalogTotal != null && Number(count) > 0) delete ledger[key].lastFailure;
+  return ledger[key];
+}
+
+function recordFailure(ledger, item, failure, at = new Date().toISOString()) {
+  const row = recordSource(ledger, item);
+  row.lastFailure = { kind: failure.kind, message: String(failure.message || '').slice(0, 300), at };
+  return row;
+}
+
+function ledgerRows(ledger) {
+  return Object.values(ledger || {}).sort((a, b) => String(a.title || '').localeCompare(String(b.title || ''))
+    || String(a.source || '').localeCompare(String(b.source || '')));
+}
+
+module.exports = { workKey, sourceKey, recordSource, recordFailure, ledgerRows };