Consolidate MangaCircus app, scraper, and Compose baseline

server.js

@@ -7,9 +7,16 @@
  *   SIGNUP=closed|open   (the very first account can always be created and becomes admin)
  *   SCAN_INTERVAL=10     minutes between automatic library rescans, 0 = off
  *   COOKIE_SECURE=1      mark the session cookie Secure (when served over HTTPS)
+ *   SCRAPER_URL=http://127.0.0.1:8091   scraper daemon API
+ *   SCRAPER_TOKEN=...                    shared daemon token (optional on loopback)
  */
 const http = require('http'), fs = require('fs'), path = require('path'), crypto = require('crypto');
 const { scanLibrary } = require('./library');
+const { Catalog, CatalogError, collectionViews, TYPES, ROLES } = require('./catalog');
+const { secret } = require('./runtime/config');
+const {
+  applyPulse, applyVisit, chapterInsight, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary,
+} = require('./reading-analytics');
 
 const PORT = +process.env.PORT || 8080;
 const LIB = path.resolve(process.env.LIBRARY || 'library');
@@ -17,6 +24,9 @@ const DATA = path.resolve(process.env.DATA || 'data');
 const OPEN_SIGNUP = process.env.SIGNUP === 'open';
 const SCAN_MINUTES = +(process.env.SCAN_INTERVAL || 10);
 const SECURE_COOKIE = process.env.COOKIE_SECURE === '1';
+const SCRAPER_URL = String(process.env.SCRAPER_URL || 'http://127.0.0.1:8091').replace(/\/$/, '');
+const SCRAPER_TOKEN = secret('SCRAPER_TOKEN');
+const SENTINEL_URL = String(process.env.SENTINEL_URL || 'http://127.0.0.1:8092').replace(/\/$/, '');
 const PUB = path.join(__dirname, 'public');
 const IMG = /\.(jpe?g|png|webp|gif|avif|bmp)$/i;
 const MIME = {
@@ -27,8 +37,9 @@ const MIME = {
 
 // ---------- storage: one JSON file, written atomically ----------
 fs.mkdirSync(DATA, { recursive: true });
+const catalog = new Catalog(DATA);
 const DB_FILE = path.join(DATA, 'db.json'), CACHE_FILE = path.join(DATA, 'scan-cache.json');
-const db = { users: {}, sessions: {}, progress: {}, follows: {}, notes: {}, votes: {}, known: {} };
+const db = { users: {}, sessions: {}, progress: {}, reading: {}, follows: {}, notes: {}, votes: {}, known: {} };
 // A corrupt db.json stops the server instead of silently starting empty and overwriting it.
 if (fs.existsSync(DB_FILE)) Object.assign(db, JSON.parse(fs.readFileSync(DB_FILE, 'utf8')));
 function writeAtomic(file, text) { fs.writeFileSync(file + '.tmp', text); fs.renameSync(file + '.tmp', file); }
@@ -48,7 +59,7 @@ function checkPw(pw, stored) {
   return want.length === got.length && crypto.timingSafeEqual(want, got);
 }
 const DUMMY_HASH = hashPw('x');
-const DEFAULTS = { startMode: 'last', lastMode: 'page', adapt: true, skipExtras: false, skipCovers: false, zoom: {}, webhook: '' };
+const DEFAULTS = { startMode: 'last', lastMode: 'page', adapt: true, skipExtras: false, skipCovers: false, flowPageGap: true, readingClock: false, readingStory: true, zoom: {}, webhook: '' };
 const settingsOf = u => ({ ...DEFAULTS, ...u.settings, zoom: { ...(u.settings && u.settings.zoom) } });
 
 function createUser(name, pw, admin, replace) {
@@ -126,8 +137,27 @@ const url = rel => (rel ? 'lib/' + rel.split('/').map(encodeURIComponent).join('
 const mangaOf = id => lib.byId.get(String(id)) || bad(404, 'Manga not found');
 const chapterOf = (m, id) => m.chapterById.get(String(id)) || bad(404, 'Chapter not found');
 const progressOf = (u, m) => (db.progress[u.name] && db.progress[u.name][m.id]) || {};
+const mangaReadingOf = (u, m) => (db.reading[u.name] && db.reading[u.name][m.id]) || {};
+const readingOf = (u, m, ch) => (db.reading[u.name] && db.reading[u.name][m.id] && db.reading[u.name][m.id][ch.id]) || null;
 const unreadOf = u => (db.notes[u.name] || []).filter(n => !n.read).length;
 const admin = u => u.admin || bad(403, 'Admins only');
+const READING_LEASE_MS = 15000;
+const readingLeases = new Map();
+
+async function scraperRequest(endpoint, body) {
+  const headers = { accept: 'application/json' };
+  if (SCRAPER_TOKEN) headers.authorization = `Bearer ${SCRAPER_TOKEN}`;
+  const options = { headers, signal: AbortSignal.timeout(7000) };
+  if (body !== undefined) {
+    options.method = 'POST'; headers['content-type'] = 'application/json'; options.body = JSON.stringify(body);
+  }
+  let response;
+  try { response = await fetch(SCRAPER_URL + endpoint, options); }
+  catch (error) { throw new HttpError(503, `Scraper service unavailable: ${error.message}`); }
+  const data = await response.json().catch(() => ({}));
+  if (!response.ok) throw new HttpError(response.status >= 500 ? 502 : response.status, data.error || `Scraper HTTP ${response.status}`);
+  return data;
+}
 
 // Versions sorted by the sum of all user votes; ties go to the higher "priority" in chapter.json, then folder order.
 function ranked(m, ch) {
@@ -142,6 +172,28 @@ function versionsOf(m, ch, u) {
     id: v.id, source: v.source, sourceUrl: v.sourceUrl, pages: v.pages.length, added: v.added, score, my: (votes[v.id] && votes[v.id][u.name]) || 0,
   }));
 }
+
+// Normalize the release-date shapes accepted by manga.json for library sorting.
+function firstReleaseOf(meta) {
+  meta = meta || {};
+  const raw = meta.firstReleaseDate || meta.releaseDate || meta.startDate;
+  let year, month, day, label, time;
+  if (raw && typeof raw === 'object') {
+    year = +raw.year; month = +raw.month || 1; day = +raw.day || 1;
+    if (year > 0) {
+      label = [String(year).padStart(4, '0'), raw.month ? String(month).padStart(2, '0') : null, raw.day ? String(day).padStart(2, '0') : null].filter(Boolean).join('-');
+      time = Date.UTC(year, month - 1, day);
+    }
+  } else if (typeof raw === 'string' && raw.trim()) {
+    const parsed = Date.parse(raw);
+    if (Number.isFinite(parsed)) { label = raw.trim(); time = parsed; }
+  }
+  if (!Number.isFinite(time)) {
+    year = +meta.year;
+    if (year > 0) { label = String(year); time = Date.UTC(year, 0, 1); }
+  }
+  return { label: label || '', time: Number.isFinite(time) ? time : 0 };
+}
 function continueOf(m, prog, s) {
   const ok = c => !(s.skipExtras && c.extra);
   let last = null;
@@ -162,6 +214,48 @@ function neighbour(m, ch, step, s) {
 // ---------- API ----------
 const PUBLIC = new Set(['/api/auth', '/api/login', '/api/signup']);
 const routes = {
+  'GET /api/health': () => ({ ok: !scanState.error, service: 'reader', uptime: Math.round(process.uptime()), scan: scanState }),
+  'GET /api/collections': ({ u }) => ({ collections: routes['GET /api/library']({ u }).collections }),
+  'GET /api/collection': ({ u, q }) => {
+    const view = routes['GET /api/library']({ u }).collections.find(c => c.id === q.get('id')) || bad(404, 'Collection not found');
+    const next = view.members.find(m => m.read < m.chapters) || view.members[0];
+    const manga = mangaOf(next.id);
+    return { ...view, continue: { m: next.id, ...continueOf(manga, progressOf(u, manga), settingsOf(u)) },
+      relations: catalog.state.relations.filter(r => r.state === 'confirmed' && view.members.some(m => m.id === r.from || m.id === r.to)) };
+  },
+  'GET /api/admin/catalog': ({ u }) => { admin(u); return { ...catalog.state, types: TYPES, roles: ROLES, mangas: lib.mangas.map(m => ({ id: m.id, title: m.title })) }; },
+  'POST /api/admin/catalog': ({ u, b }) => { admin(u); return catalog.update(b, new Set(lib.byId.keys()), u.name); },
+  'POST /api/admin/catalog/suggest': ({ u, b }) => {
+    admin(u);
+    return catalog.update({ ...catalog.state, revision: b.revision, relations: catalog.suggest(lib.mangas) }, new Set(lib.byId.keys()), u.name);
+  },
+  'POST /api/admin/catalog/research': async ({ u, b }) => {
+    admin(u);
+    const m = mangaOf(b.m), id = m.meta?.['x-anilist']?.id;
+    if (!id) bad(400, 'Assign a verified AniList identity first');
+    const { lookupAniListById } = require('./scraper/metadata/anilist');
+    const meta = await lookupAniListById(id, { cacheDir: path.join(DATA, 'metadata', 'anilist') });
+    const enriched = lib.mangas.map(item => item.id === m.id ? { ...item, meta } : item);
+    return catalog.update({ ...catalog.state, revision: b.revision, relations: catalog.suggest(enriched) }, new Set(lib.byId.keys()), u.name);
+  },
+  'POST /api/admin/collection-cover': ({ u, b }) => {
+    admin(u);
+    const c = catalog.state.collections.find(c => c.id === b.id) || bad(404, 'Collection not found');
+    if (b.revision !== catalog.state.revision) bad(409, 'Catalog changed; reload before saving');
+    const image = Buffer.from(String(b.base64 || ''), 'base64');
+    const ext = require('./scraper/lib/images').imageExtension(image);
+    if (!ext || image.length > 600000) bad(400, 'Use a JPG, PNG, WebP, GIF, AVIF or BMP under 600 KB');
+    const name = sha(image) + '.' + ext;
+    const dir = path.join(DATA, 'collection-covers'); fs.mkdirSync(dir, { recursive: true });
+    if (!fs.existsSync(path.join(dir, name))) fs.writeFileSync(path.join(dir, name), image, { flag: 'wx' });
+    return catalog.update({ ...catalog.state, collections: catalog.state.collections.map(item => item.id === c.id ? { ...item, coverMode: 'custom', customCover: name } : item) }, new Set(lib.byId.keys()), u.name);
+  },
+  'GET /api/admin/health': async ({ u }) => {
+    admin(u);
+    const sentinel = await fetch(SENTINEL_URL + '/api/state', { headers: SCRAPER_TOKEN ? { authorization: `Bearer ${SCRAPER_TOKEN}` } : {}, signal: AbortSignal.timeout(4000) })
+      .then(async r => { if (!r.ok) throw new Error(); return r.json(); }).catch(() => ({ status: 'unreachable' }));
+    return { reader: { ok: !scanState.error, uptime: Math.round(process.uptime()), scan: scanState }, sentinel };
+  },
   'GET /api/auth': () => ({ firstRun: !Object.keys(db.users).length, signup: OPEN_SIGNUP }),
 
   'POST /api/signup': ({ b, res }) => {
@@ -189,7 +283,7 @@ const routes = {
     const s = (u.settings = u.settings || {});
     if (['last', 'flow', 'page', 'book'].includes(b.startMode)) s.startMode = b.startMode;
     if (['flow', 'page', 'book'].includes(b.lastMode)) s.lastMode = b.lastMode;
-    for (const k of ['adapt', 'skipExtras', 'skipCovers']) if (typeof b[k] === 'boolean') s[k] = b[k];
+    for (const k of ['adapt', 'skipExtras', 'skipCovers', 'flowPageGap', 'readingClock', 'readingStory']) if (typeof b[k] === 'boolean') s[k] = b[k];
     if (b.zoom && typeof b.zoom === 'object') {
       s.zoom = {};
       for (const k of ['flow', 'page', 'book', 'strip']) if (+b.zoom[k] > 0) s.zoom[k] = Math.min(8, Math.max(0.1, +b.zoom[k]));
@@ -206,33 +300,85 @@ const routes = {
 
   'GET /api/library': ({ u }) => {
     const prog = db.progress[u.name] || {}, fol = db.follows[u.name] || {}, notes = (db.notes[u.name] || []).filter(n => !n.read);
+    const showStory = settingsOf(u).readingStory;
+    const mangas = lib.mangas.map(m => {
+      const p = prog[m.id] || {}, meta = m.meta || {}, release = firstReleaseOf(meta);
+      return {
+        id: m.id, title: m.title, cover: url(m.cover), direction: m.direction, chapters: m.chapters.length, updated: m.updated,
+        altTitles: Array.isArray(meta.altTitles) ? meta.altTitles : (meta.altTitles ? [String(meta.altTitles)] : []),
+        authors: Array.isArray(meta.authors) ? meta.authors : (meta.authors ? [String(meta.authors)] : []),
+        tags: Array.isArray(meta.tags) ? meta.tags : (meta.tags ? [String(meta.tags)] : []), status: meta.status || '', year: meta.year || '',
+        firstRelease: release.label, firstReleaseAt: release.time,
+        read: m.chapters.filter(c => p[c.id] && p[c.id].done).length,
+        lastRead: Object.values(p).reduce((t, x) => Math.max(t, x.t || 0), 0),
+        following: !!fol[m.id], unread: notes.filter(n => n.m === m.id).length,
+        reading: showStory ? mangaInsight(mangaReadingOf(u, m), m.chapters.map(ch => ch.id)) : null,
+      };
+    });
     return {
       scanning: !!scanning && !lib.mangas.length,
-      mangas: lib.mangas.map(m => {
-        const p = prog[m.id] || {};
-        return {
-          id: m.id, title: m.title, cover: url(m.cover), direction: m.direction, chapters: m.chapters.length, updated: m.updated,
-          read: m.chapters.filter(c => p[c.id] && p[c.id].done).length,
-          lastRead: Object.values(p).reduce((t, x) => Math.max(t, x.t || 0), 0),
-          following: !!fol[m.id], unread: notes.filter(n => n.m === m.id).length,
-        };
-      }),
+      reading: showStory ? libraryInsight(mangas) : null,
+      mangas,
+      collections: collectionViews(catalog.state, mangas),
     };
   },
 
   'GET /api/manga': ({ u, q }) => {
-    const m = mangaOf(q.get('m')), prog = progressOf(u, m);
+    const m = mangaOf(q.get('m')), prog = progressOf(u, m), showStory = settingsOf(u).readingStory;
+    const records = mangaReadingOf(u, m);
     return {
       id: m.id, title: m.title, meta: m.meta, cover: url(m.cover), direction: m.direction,
+      collections: catalog.state.collections.filter(c => c.members.some(member => member.mangaId === m.id)).map(c => ({ id: c.id, title: c.title })),
+      relations: catalog.state.relations.filter(r => r.state === 'confirmed' && (r.from === m.id || r.to === m.id)).map(r => ({ ...r,
+        otherId: r.from === m.id ? r.to : r.from, otherTitle: lib.byId.get(r.from === m.id ? r.to : r.from)?.title || (r.from === m.id ? r.to : r.from) })),
       following: !!(db.follows[u.name] && db.follows[u.name][m.id]),
       books: m.books.map(b => ({ key: b.key, label: b.label, title: b.title, extra: b.extra })),
       chapters: m.chapters.map(ch => ({
         id: ch.id, book: ch.book, label: ch.label, title: ch.title, extra: ch.extra, progress: prog[ch.id] || null, versions: versionsOf(m, ch, u),
+        reading: showStory ? chapterInsight(records[ch.id]) : null,
       })),
+      reading: showStory ? mangaInsight(records, m.chapters.map(ch => ch.id)) : null,
       continue: continueOf(m, prog, settingsOf(u)),
     };
   },
 
+  'GET /api/reading/export': ({ u }) => {
+    const byManga = db.reading[u.name] || {};
+    const mangas = Object.entries(byManga).map(([id, records]) => {
+      const m = lib.byId.get(id), chapterIds = m ? m.chapters.map(ch => ch.id) : Object.keys(records || {});
+      const chapterMap = m ? m.chapterById : new Map();
+      return {
+        id, title: m ? m.title : id,
+        reading: mangaInsight(records, chapterIds),
+        chapters: chapterIds.map(chapterId => {
+          const chapter = chapterMap.get(chapterId), reading = chapterInsight(records && records[chapterId]);
+          return reading ? { id: chapterId, label: chapter ? chapter.label : chapterId, title: chapter ? chapter.title : '', reading } : null;
+        }).filter(Boolean),
+      };
+    }).filter(item => item.reading.timedChapters || item.reading.completedPasses);
+    return {
+      schema: 1,
+      exportedAt: new Date().toISOString(),
+      user: u.name,
+      reading: libraryInsight(mangas),
+      mangas,
+    };
+  },
+
+  'POST /api/reading/delete': ({ u, b }) => {
+    if (b.all) {
+      if (!checkPw(String(b.password || ''), u.hash)) bad(403, 'The current password is wrong.');
+      delete db.reading[u.name];
+      readingLeases.delete(u.name);
+      save();
+      return { ok: true, deleted: 'all' };
+    }
+    const m = mangaOf(b.m), all = db.reading[u.name];
+    if (all) delete all[m.id];
+    save();
+    return { ok: true, deleted: m.id };
+  },
+
   'GET /api/chapter': ({ u, q }) => {
     const m = mangaOf(q.get('m')), ch = chapterOf(m, q.get('c')), s = settingsOf(u), prog = progressOf(u, m);
     const v = ch.versions.find(x => x.id === q.get('v')) || ranked(m, ch)[0].v;
@@ -247,6 +393,7 @@ const routes = {
       pages: v.pages.map(p => ({ u: url(v.rel + '/' + p.f), w: p.w, h: p.h, t: p.t, r: p.r })),
       prev: neighbour(m, ch, -1, s), next: neighbour(m, ch, 1, s),
       progress: prog[ch.id] || null,
+      reading: s.readingClock ? publicSummary(readingOf(u, m, ch), v.id, eligiblePages(v, s)) : null,
       // for the chapter picker: the whole manga across all books
       books: m.books.map(b => ({ key: b.key, label: b.label, title: b.title })),
       chapters: m.chapters.map(c => ({ id: c.id, book: c.book, label: c.label, title: c.title, extra: c.extra, done: !!(prog[c.id] && prog[c.id].done) })),
@@ -263,6 +410,51 @@ const routes = {
     save();
     return { ok: true };
   },
+  // Separate endpoint: an older server on rollback cannot misread cumulative
+  // visit totals as legacy deltas. Old open tabs retain their original protocol.
+  'POST /api/reading/visit': context => routes['POST /api/reading/pulse']({ ...context, cumulative: true }),
+  'POST /api/reading/pulse': ({ u, b, cumulative = false }) => {
+    const settings = settingsOf(u);
+    if (!settings.readingClock) bad(403, 'The Active Reading Clock is disabled.');
+    const m = mangaOf(b.m), ch = chapterOf(m, b.c);
+    const v = ch.versions.find(x => x.id === b.v) || bad(404, 'Version not found');
+    const session = String(b.session || '');
+    const seq = Number(b.seq);
+    if (!/^[a-zA-Z0-9_-]{8,80}$/.test(session)) bad(400, 'Invalid reading session.');
+    if (!Number.isInteger(seq) || seq < 1 || seq > 1e9) bad(400, 'Invalid reading pulse sequence.');
+
+    const now = Date.now(), eligible = eligiblePages(v, settings), lease = readingLeases.get(u.name);
+    const previous = readingOf(u, m, ch)?.recentSessions?.find(item => item.id === session);
+    // A stale request must not reclaim a lease after a newer exit save. Final
+    // snapshots can also arrive before the initial claim: do not lose that visit.
+    if (cumulative && previous && seq <= previous.seq) return {
+      accepted: true, duplicate: true, leaseMs: 0,
+      reading: publicSummary(readingOf(u, m, ch), v.id, eligible),
+    };
+    const lateFinal = cumulative && b.final;
+    if (!lateFinal && lease && lease.session !== session && now - lease.at < READING_LEASE_MS) {
+      return {
+        accepted: false,
+        reason: 'other-tab',
+        retryAfterMs: READING_LEASE_MS - (now - lease.at),
+        reading: publicSummary(readingOf(u, m, ch), v.id, eligible),
+      };
+    }
+    if (!lateFinal) readingLeases.set(u.name, { session, at: now });
+
+    const byUser = (db.reading[u.name] = db.reading[u.name] || {});
+    const byManga = (byUser[m.id] = byUser[m.id] || {});
+    const record = (byManga[ch.id] = readingOf(u, m, ch) || emptyRecord());
+    const result = (cumulative ? applyVisit : applyPulse)(record, b, { version: v.id, pageCount: v.pages.length, eligible, now });
+    if (b.release && readingLeases.get(u.name)?.session === session) readingLeases.delete(u.name);
+    if (!result.duplicate) save();
+    return {
+      accepted: true,
+      duplicate: result.duplicate,
+      leaseMs: b.release ? 0 : READING_LEASE_MS,
+      reading: publicSummary(record, v.id, eligible),
+    };
+  },
   'POST /api/unread': ({ u, b }) => {
     const m = mangaOf(b.m), mp = db.progress[u.name] && db.progress[u.name][m.id];
     if (mp) delete mp[chapterOf(m, b.c).id];
@@ -311,11 +503,34 @@ const routes = {
     admin(u);
     const name = String(b.name || '');
     if (name === u.name) bad(400, "You can't delete your own account.");
-    delete db.users[name]; delete db.progress[name]; delete db.follows[name]; delete db.notes[name];
+    delete db.users[name]; delete db.progress[name]; delete db.reading[name]; delete db.follows[name]; delete db.notes[name];
     for (const [k, s] of Object.entries(db.sessions)) if (s.u === name) delete db.sessions[k];
     save();
     return { ok: true };
   },
+  'GET /api/admin/scraper': async ({ u }) => {
+    admin(u);
+    try { return { available: true, ...(await scraperRequest('/api/state')) }; }
+    catch (error) { return { available: false, error: error.message, url: SCRAPER_URL }; }
+  },
+  'POST /api/admin/scraper': async ({ u, b }) => {
+    admin(u);
+    const command = String(b.command || '');
+    if (command === 'enqueue') return scraperRequest('/api/queue', {
+      url: b.url, title: b.title, metadataSearch: b.metadataSearch,
+      scope: b.scope, existingPolicy: b.existingPolicy, libraryMangaId: b.libraryMangaId,
+      inspect: b.inspect === true,
+    });
+    if (command === 'remove') return scraperRequest('/api/queue/remove', { id: b.id });
+    if (command === 'clear-completed') return scraperRequest('/api/queue/clear-completed', { source: b.source });
+    if (command === 'reorder') return scraperRequest('/api/queue/reorder', { ids: b.ids });
+    if (command === 'control') return scraperRequest('/api/control', { action: b.action, id: b.id });
+    if (command === 'settings') return scraperRequest('/api/settings', b.settings || {});
+    if (command === 'discover') return scraperRequest('/api/discovery/run', {});
+    if (command === 'auto-update') return scraperRequest('/api/library/auto-update', { id: b.id, enabled: !!b.enabled });
+    if (command === 'publish-legacy-ready') return scraperRequest('/api/legacy/publish-ready', {});
+    bad(400, 'Unknown scraper command');
+  },
 };
 
 // ---------- HTTP ----------
@@ -355,6 +570,13 @@ http.createServer(async (req, res) => {
   try {
     const { pathname, searchParams } = new URL(req.url, 'http://localhost');
     const u = userOf(req);
+    if (pathname === '/health' && req.method === 'GET') return sendJson(res, 200, { ...routes['GET /api/health'](), ok: true, ready: !scanState.error });
+    if (pathname === '/api/collection-cover' && req.method === 'GET') {
+      if (!u) bad(401, 'Please log in');
+      const c = catalog.state.collections.find(c => c.id === searchParams.get('id'));
+      if (!c?.customCover) bad(404, 'Cover not found');
+      return serveFile(req, res, path.join(DATA, 'collection-covers'), c.customCover, 'private, no-cache', IMG);
+    }
     if (pathname.startsWith('/api/')) {
       const route = routes[req.method + ' ' + pathname];
       if (!route) bad(404, 'Unknown endpoint');
@@ -368,10 +590,25 @@ http.createServer(async (req, res) => {
     }
     serveFile(req, res, PUB, pathname === '/' ? 'index.html' : pathname.slice(1), 'no-cache');
   } catch (e) {
-    if (!(e instanceof HttpError)) console.error(e);
-    if (!res.headersSent) sendJson(res, e.code || 500, { error: e instanceof HttpError ? e.message : 'Internal error' });
+    const expected = e instanceof HttpError || e instanceof CatalogError;
+    if (!expected) console.error(e);
+    if (!res.headersSent) sendJson(res, expected ? e.code : 500, { error: expected ? e.message : 'Internal error' });
   }
 }).listen(PORT, () => console.log(`MangaCircus on http://localhost:${PORT}  (library: ${LIB}, data: ${DATA})`));
 
 rescan();
 if (SCAN_MINUTES > 0) setInterval(() => rescan(), SCAN_MINUTES * 60000).unref();
+
+// A completed scraper publish increments a durable revision. Polling it here keeps the
+// reader index fresh without exposing an unauthenticated rescan endpoint to the daemon.
+let scraperRevision = null;
+async function pollScraper() {
+  try {
+    const state = await scraperRequest('/api/state');
+    const revision = Number(state.publishedRevision || 0);
+    if (scraperRevision !== null && revision > scraperRevision) await rescan();
+    scraperRevision = revision;
+  } catch { /* the reader also works when the optional scraper daemon is offline */ }
+}
+pollScraper();
+setInterval(pollScraper, 15000).unref();