Consolidate MangaCircus app, scraper, and Compose baseline

sentinel/daemon.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('fs'), path = require('path'), http = require('http');
+const { secret } = require('../runtime/config');
+const { Monitor } = require('./monitor');
+const root = path.resolve(process.env.SENTINEL_DATA || path.join(process.env.DATA || 'data', 'sentinel'));
+fs.mkdirSync(root, { recursive: true });
+const token = secret('SCRAPER_TOKEN');
+const host = process.env.SENTINEL_HOST || '127.0.0.1', port = +(process.env.SENTINEL_PORT || 8092);
+if (!['127.0.0.1', 'localhost', '::1'].includes(host) && !token) throw new Error('SCRAPER_TOKEN is required for non-loopback Sentinel');
+const reader = process.env.READER_URL || 'http://127.0.0.1:8080';
+const scraper = process.env.SCRAPER_URL || 'http://127.0.0.1:8091';
+const file = path.join(root, 'state.json');
+let saved = {};
+if (fs.existsSync(file)) saved = JSON.parse(fs.readFileSync(file, 'utf8'));
+const monitor = new Monitor({ incidents: saved.incidents || [] });
+const request = async (base, endpoint, body) => {
+  const response = await fetch(base + endpoint, { headers: { ...(token ? { authorization: `Bearer ${token}` } : {}), 'content-type': 'application/json' },
+    signal: AbortSignal.timeout(8000), ...(body ? { method: 'POST', body: JSON.stringify(body) } : {}) });
+  if (!response.ok) throw new Error(`Service HTTP ${response.status}`);
+  return response.json();
+};
+let stopped = false, timer, lastTick = Date.now();
+async function tick() {
+  try {
+    const results = await Promise.allSettled([request(reader, '/health'), request(scraper, '/api/state')]);
+    const state = await monitor.observe(results[0].status === 'fulfilled' ? results[0].value : null,
+      results[1].status === 'fulfilled' ? results[1].value : null, input => request(scraper, '/api/recovery', input));
+    fs.writeFileSync(file + '.tmp', JSON.stringify(state, null, 2) + '\n'); fs.renameSync(file + '.tmp', file);
+    lastTick = Date.now();
+    for (const incident of state.incidents.filter(i => i.state === 'open' && !i.logged)) {
+      console.log(JSON.stringify({ ...incident, targetService: incident.service, service: 'sentinel', level: 'warning' })); incident.logged = true;
+    }
+  } catch (error) { console.error('[sentinel]', error.message); }
+  if (!stopped) timer = setTimeout(tick, 30000);
+}
+const server = http.createServer((req, res) => {
+  const endpoint = new URL(req.url, 'http://localhost').pathname;
+  let code = 200, value;
+  if (endpoint === '/health') { code = Date.now() - lastTick < 120000 ? 200 : 503; value = { ok: code === 200, service: 'sentinel', checkedAt: monitor.state.checkedAt }; }
+  else if (token && req.headers.authorization !== `Bearer ${token}`) { code = 401; value = { error: 'Unauthorized' }; }
+  else if (endpoint === '/api/state' && req.method === 'GET') value = monitor.state;
+  else if (endpoint === '/metrics' && req.method === 'GET') {
+    res.writeHead(200, { 'content-type': 'text/plain; version=0.0.4' });
+    return res.end(`mangacircus_reader_up ${Number(!!monitor.state.services.reader)}\nmangacircus_scraper_up ${Number(!!monitor.state.services.scraper)}\nmangacircus_open_incidents ${monitor.incidents.filter(i => i.state === 'open').length}\n`);
+  } else { code = 404; value = { error: 'Not found' }; }
+  res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(value));
+});
+server.listen(port, host, () => console.log(JSON.stringify({ service: 'sentinel', event: 'started', port })));
+tick();
+for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => { stopped = true; clearTimeout(timer); server.close(() => process.exit(0)); });

sentinel/monitor.js

@@ -0,0 +1,62 @@
+'use strict';
+const { progressKey } = require('../scraper/lib/failures');
+
+// No source requests, filesystem checkpoint edits, shell execution or Docker socket.
+class Monitor {
+  constructor(options = {}) {
+    this.now = options.now || Date.now;
+    this.stallMs = options.stallMs || 900000;
+    this.observations = new Map();
+    this.incidents = options.incidents || [];
+    this.state = { status: 'starting', services: {}, incidents: this.incidents };
+  }
+  incident(key, message, extra = {}) {
+    let incident = this.incidents.find(i => i.key === key && i.state === 'open');
+    if (!incident) {
+      incident = { key, message, state: 'open', at: new Date(this.now()).toISOString(), ...extra };
+      this.incidents.push(incident);
+      if (this.incidents.length > 200) this.incidents.splice(0, this.incidents.length - 200);
+    }
+    return incident;
+  }
+  resolve(key) {
+    for (const i of this.incidents) if (i.key === key && i.state === 'open') { i.state = 'resolved'; i.resolvedAt = new Date(this.now()).toISOString(); }
+  }
+  async observe(reader, scraper, recover) {
+    const now = this.now();
+    this.state.checkedAt = new Date(now).toISOString();
+    this.state.services = { reader: !!reader?.ok, scraper: !!scraper };
+    if (reader?.ready === false) this.incident('reader:scan', 'Reader is alive but its library scan failed; inspect storage and scan diagnostics', { service: 'reader' });
+    else if (reader?.ok) this.resolve('reader:scan');
+    for (const [service, healthy] of Object.entries(this.state.services)) {
+      if (healthy) this.resolve(`service:${service}`);
+      else this.incident(`service:${service}`, `${service} is unreachable or unhealthy; inspect its supervisor`, { service });
+    }
+    if (!scraper) { this.state.status = 'degraded'; return this.state; }
+    const ids = new Set();
+    for (const entry of scraper.active || []) {
+      const id = entry.current.id; ids.add(id);
+      const key = progressKey(entry.job), prior = this.observations.get(id);
+      if (!prior || prior.key !== key) { this.observations.set(id, { key, at: now }); this.resolve(`stall:${id}`); continue; }
+      if (scraper.paused || scraper.sourcePaused?.[entry.source]) continue;
+      const allowance = Math.max(this.stallMs, Number(entry.job?.rate?.chapterMaxMs || 0) + 180000);
+      if (now - prior.at < allowance) continue;
+      const incident = this.incident(`stall:${id}`, 'Job progress has stopped; recovery is limited to one checkpoint-safe attempt', { jobId: id, source: entry.source, title: entry.current.title });
+      if (!incident.attempted && recover) {
+        // Persist attempted in the daemon via stopReason/stallRecoveries; monitor records audit too.
+        incident.attempted = true;
+        try { incident.recovery = await recover({ id, progressKey: key }); }
+        catch { incident.recovery = { changed: false, error: 'Recovery API unavailable' }; }
+      }
+    }
+    for (const id of this.observations.keys()) if (!ids.has(id)) { this.observations.delete(id); this.resolve(`stall:${id}`); }
+    const failures = (scraper.queue || []).filter(item => ['error', 'quarantined'].includes(item.status));
+    for (const item of failures) this.incident(`job:${item.id}`, item.failure?.message || item.error || 'Job needs attention', { jobId: item.id, source: item.source, kind: item.failure?.kind || 'unknown' });
+    for (const i of this.incidents) if (i.key.startsWith('job:') && !failures.some(item => i.key === `job:${item.id}`)) this.resolve(i.key);
+    this.state.sourcePaused = scraper.sourcePaused || {};
+    this.state.active = (scraper.active || []).map(e => ({ source: e.source, id: e.current.id, title: e.current.title, completed: e.job?.completed, total: e.job?.total, updated: e.job?.updated }));
+    this.state.status = this.incidents.some(i => i.state === 'open') ? 'attention' : scraper.paused ? 'paused' : 'healthy';
+    return this.state;
+  }
+}
+module.exports = { Monitor };

sentinel/test/monitor.test.js

@@ -0,0 +1,26 @@
+'use strict';
+const test = require('node:test'), assert = require('node:assert/strict');
+const { Monitor } = require('../monitor');
+test('Sentinel does not disturb progressing or deliberately paused jobs', async () => {
+  let now = 0, recoveries = 0;
+  const monitor = new Monitor({ now: () => now, stallMs: 100 });
+  const state = { paused: false, active: [{ source: 'a', current: { id: 'job', title: 'Manga' }, job: { completed: 0, current: { downloadedPages: 1 } } }], queue: [] };
+  const recover = async () => { recoveries++; return { changed: true }; };
+  await monitor.observe({ ok: true }, state, recover);
+  now = 500000; state.active[0].job.current.downloadedPages = 2;
+  await monitor.observe({ ok: true }, state, recover);
+  now = 1000000; state.paused = true;
+  await monitor.observe({ ok: true }, state, recover);
+  assert.equal(recoveries, 0);
+  state.paused = false; await monitor.observe({ ok: true }, state, recover); await monitor.observe({ ok: true }, state, recover);
+  assert.equal(recoveries, 1);
+  state.active[0].job.completed = 1; await monitor.observe({ ok: true }, state, recover);
+  assert.equal(monitor.incidents.find(i => i.key === 'stall:job').state, 'resolved');
+});
+test('Sentinel reports service failures without restart loops and resolves recovery', async () => {
+  const monitor = new Monitor();
+  await monitor.observe(null, null); await monitor.observe(null, null);
+  assert.equal(monitor.incidents.length, 2);
+  await monitor.observe({ ok: true }, { active: [], queue: [], paused: false });
+  assert.equal(monitor.state.status, 'healthy'); assert.ok(monitor.incidents.every(i => i.state === 'resolved'));
+});

sentinel/test/supervisor.test.js

@@ -0,0 +1,28 @@
+'use strict';
+const test = require('node:test'), assert = require('node:assert/strict');
+const fs = require('fs'), os = require('os'), path = require('path'), { EventEmitter } = require('events');
+const { Supervisor } = require('../../runtime/supervisor');
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+test('supervisor isolates a child crash and persists its bounded restart budget', async () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-supervisor-')), file = path.join(dir, 'runtime.json');
+  let starts = 0;
+  const spawn = () => { const child = new EventEmitter(); child.pid = ++starts; child.kill = () => { child.emit('exit', 0); child.emit('close', 0); }; return child; };
+  const supervisor = new Supervisor('reader', { file, spawn, retryBaseMs: 1 });
+  try {
+    supervisor.start();
+    for (let n = 0; n < 3; n++) {
+      supervisor.child.emit('close', 1, null);
+      for (let i = 0; i < 50 && supervisor.state.status === 'retry_wait'; i++) await delay(2);
+    }
+    assert.equal(starts, 3); assert.equal(supervisor.state.status, 'parked'); assert.equal(supervisor.child, null);
+    const restored = new Supervisor('reader', { file, spawn, retryBaseMs: 1 }); restored.start();
+    assert.equal(starts, 3); assert.equal(restored.state.status, 'parked'); await restored.close();
+  } finally { await supervisor.close(); fs.rmSync(dir, { recursive: true, force: true }); }
+});
+test('intentional supervisor shutdown does not consume the crash budget', async () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-supervisor-stop-'));
+  const child = new EventEmitter(); child.pid = 1; child.kill = () => { child.emit('exit', 0); child.emit('close', 0); };
+  const supervisor = new Supervisor('scraper', { file: path.join(dir, 'runtime.json'), spawn: () => child });
+  try { supervisor.start(); await supervisor.close(); assert.equal(supervisor.state.status, 'stopped'); assert.equal(supervisor.state.failures.length, 0); }
+  finally { await supervisor.close(); fs.rmSync(dir, { recursive: true, force: true }); }
+});