Fold the sentinel into the scraper and drop the sentinel service (BL-0046)

The stall watchdog now runs inside the scraper (ScraperService.checkStalls,
every 30 s from the work loop): same rule as before, one checkpoint-safe
restart, quarantine on the second stall, missing checkpoint is no evidence,
and paused time no longer counts. Stalls are recorded as scraper incidents
(failure.kind 'stall'). The private /api/recovery endpoint is gone.

Dropped with the service: reader/scraper-unreachable incidents (supervisor
and Docker handle crashes; the health page shows "Scraper: unreachable"),
duplicate job-failure incidents and the unused /metrics endpoint.
The stack is now reader + scraper (+ optional peer-gateway).
The supervisor test moved to runtime/test (npm run test:runtime stays).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scraper/service.js

@@ -124,6 +124,9 @@ class ScraperService {
     this.resolveAdapter = options.adapterFor || adapterFor;
     this.adapterList = options.adapters || adapters;
     this.retryDelayMs = options.retryDelayMs ?? 300000;
+    this.stallMs = options.stallMs ?? 900000;
+    this.stallObservations = new Map();
+    this.stallCheckedAt = 0;
     this.runtimeHeartbeat = Date.now();
   }
 
@@ -738,22 +741,32 @@ class ScraperService {
     throw new Error('Unknown control action');
   }
 
-  async recover(input) {
-    const item = this.state.queue.find(entry => entry.id === input.id);
-    if (!item) throw new Error('Job not found');
-    // Compare observations so a stale watcher cannot disturb a progressing worker.
-    if (Object.values(this.state.currentIds).includes(item.id)) {
+  // Stall watchdog: an active job whose checkpoint has not moved for stallMs (or one
+  // chapter delay + 3 min, if longer) is stopped at its next checkpoint and requeued
+  // once; a second stall quarantines it. A missing checkpoint proves nothing, and time
+  // spent paused does not count.
+  async checkStalls(now = Date.now()) {
+    const activeIds = new Set(Object.values(this.state.currentIds || {}));
+    for (const id of this.stallObservations.keys()) if (!activeIds.has(id)) this.stallObservations.delete(id);
+    for (const item of this.state.queue) {
+      if (!activeIds.has(item.id)) continue;
       const job = await this.jobState(item);
-      if (!job) return { changed: false };
-      if (progressKey(job) !== input.progressKey || this.state.paused || item.stopReason) return { changed: false };
-      if (Date.now() - Date.parse(job?.updated || item.started) < 900000) return { changed: false };
-      if ((item.stallRecoveries || 0) >= 1) {
-        item.stopReason = 'quarantined';
-      } else { item.stallRecoveries = (item.stallRecoveries || 0) + 1; item.stopReason = 'recovering'; }
-      await this.event('warning', `Sentinel requested recovery at the next checkpoint for ${item.title}`, { id: item.id, source: item.source });
-      return { changed: true };
+      if (!job) { this.stallObservations.delete(item.id); continue; }
+      const key = progressKey(job), prior = this.stallObservations.get(item.id);
+      if (!prior || prior.key !== key) { this.stallObservations.set(item.id, { key, at: now }); continue; }
+      if (this.state.paused || this.state.sourcePaused[item.source] || item.stopReason) { prior.at = now; continue; }
+      if (now - prior.at < Math.max(this.stallMs, Number(job.rate?.chapterMaxMs || 0) + 180000)) continue;
+      this.stallObservations.delete(item.id);
+      const minutes = Math.round((now - prior.at) / 60000), quarantine = (item.stallRecoveries || 0) >= 1;
+      const failure = { kind: 'stall', scope: 'job', retryable: !quarantine, message: quarantine
+        ? `No progress for ${minutes} minutes after a restart; quarantined`
+        : `No progress for ${minutes} minutes; restarting from the last checkpoint` };
+      if (quarantine) { item.stopReason = 'quarantined'; item.failure = failure; item.error = failure.message; }
+      else { item.stallRecoveries = (item.stallRecoveries || 0) + 1; item.stopReason = 'recovering'; }
+      this.state.incidents.push({ id: crypto.randomUUID(), at: new Date(now).toISOString(), jobId: item.id, source: item.source, title: item.title, failure, state: 'open' });
+      this.state.incidents = this.state.incidents.slice(-200);
+      await this.event('warning', `${item.title}: ${failure.message}`, { id: item.id, title: item.title, source: item.source });
     }
-    return { changed: false };
   }
 
   recomputeStatus() {
@@ -876,6 +889,10 @@ class ScraperService {
       const prior = this.state.status;
       this.recomputeStatus();
       if (prior !== this.state.status) await this.persist();
+      if (Date.now() - this.stallCheckedAt >= 30000) {
+        this.stallCheckedAt = Date.now();
+        await this.checkStalls().catch(error => console.error('[scraper] stall check failed:', safeMessage(error)));
+      }
       await sleep(500);
     }
   }