Add built-in backup of the data folder (BL-0044)

The reader owns a scheduled backup into BACKUP (Compose: BACKUP_PATH mounted at
/backup, off when unset). A run pauses the scraper queue through its control
API, waits (bounded) until no job is active, writes a verified tar.gz of DATA
with db.json from memory and scraper/queue.json stored paused, excludes
runtime/lock/pid/temp files, resumes only a queue it paused, then applies
count- or day-based retention to its own archive names only. Schedule,
retention, "Back up now" and last result live in Settings > Service health.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.dockerignore

@@ -6,6 +6,7 @@
 !reading-direction.js
 !chapter-identity.js
 !catalog.js
+!backup.js
 !reading-analytics.js
 !reading-access.js
 !public/

.env.example

@@ -9,6 +9,9 @@ DATA_PATH=./data
 SCRAPER_DATA_PATH=./data/scraper
 SENTINEL_DATA_PATH=./data/sentinel
 SCRAPER_TOKEN_PATH=./.secrets/scraper_token
+# Built-in backups of DATA_PATH (docs/backup.md): an existing folder, ideally on another
+# disk/NAS, writable by uid 1000. Unset = backups off (./backup is then mounted unused).
+# BACKUP_PATH=./backup
 TZ=Europe/Berlin
 COOKIE_SECURE=0
 # Optional server-enforced reading windows. Day keys: 0=Sunday through 6=Saturday.

.gitignore

@@ -3,6 +3,8 @@
 /data/
 # the manga library: demo content is generated by `node tools/make-dummy.js`, a real one is yours alone
 /library/
+# built-in backups (BACKUP_PATH default)
+/backup/
 
 # --- Node ---
 node_modules/

Dockerfile

@@ -3,7 +3,7 @@ RUN apk add --no-cache tzdata openssl
 WORKDIR /app
 ARG VCS_REF=development
 LABEL org.opencontainers.image.title="MangaCircus" org.opencontainers.image.revision=$VCS_REF
-COPY package.json server.js library.js reading-direction.js chapter-identity.js reading-analytics.js reading-access.js catalog.js ./
+COPY package.json server.js library.js reading-direction.js chapter-identity.js reading-analytics.js reading-access.js catalog.js backup.js ./
 COPY public ./public
 COPY greatest ./greatest
 COPY discovery ./discovery

README.md

@@ -74,6 +74,7 @@ see administration; personal settings do not poll scraper services. See
 | `SCRAPER_TOKEN` | empty       | Shared daemon token; required when it listens beyond loopback |
 | `SCRAPER_TOKEN_FILE` | empty | Alternative secret file (used in Compose) |
 | `SENTINEL_URL` | `http://127.0.0.1:8092` | Private observation service URL |
+| `BACKUP`        | –           | Existing folder for built-in backups of `DATA`; unset = off. Compose: set `BACKUP_PATH`. Schedule and retention under Settings → Service health ([backup](docs/backup.md)) |
 
 Serve it behind a reverse proxy with HTTPS if it is reachable from the internet. All pages use relative URLs, so a sub-path like `/manga/` works.
 
@@ -113,6 +114,7 @@ whole-series completeness. See the scraper service and library format contracts
 | [docs/architecture.md](docs/architecture.md) | Components, data model, API, security, the reader's layout engine, deployment, design decisions |
 | [docs/local-deployment.md](docs/local-deployment.md) | Current local Compose deployment, Tailscale access, persistent state, checks, restart, and recovery |
 | [docs/settings-navigation.md](docs/settings-navigation.md) | User/Admin settings sections, scraper tabs, routes and UI checks |
+| [docs/backup.md](docs/backup.md) | Built-in scheduled backup of `data/`, what it contains, retention and the restore procedure |
 | [docs/reading-insights-concept.md](docs/reading-insights-concept.md) | Active-time presentation, private summaries, deletion/export controls, and the future journal design |
 | [docs/scraper-guide.md](docs/scraper-guide.md) | Rule book (agent-readable) for scrapers and other tools that write into the library |
 | [scraper/AGENT_RUNBOOK.md](scraper/AGENT_RUNBOOK.md) | Conservative scraper workflow and host-specific procedures for automated agents |

backup.js

@@ -0,0 +1,398 @@
+'use strict';
+/*
+ * Built-in backup of the reader's DATA tree into BACKUP (see docs/backup.md).
+ *
+ * One run: pause the scraper queue through its control API, wait (bounded) until no
+ * job is active, write a gzip'd tar of DATA into BACKUP, resume the queue only if this
+ * run paused it, then apply retention. No container stops. db.json comes from the
+ * reader's in-memory state; scraper/queue.json is stored with `paused: true` so a
+ * restored copy never starts downloading by itself. Runtime/lock/pid/temp files are
+ * left out. Retention only ever deletes `circus-YYYYMMDD-HHMMSS.tar.gz` files directly
+ * inside BACKUP. No dependencies: the tar writer/reader below is plain ustar + PAX.
+ */
+const fs = require('fs'), fsp = fs.promises, path = require('path'), zlib = require('zlib');
+const { Readable } = require('stream'), { pipeline } = require('stream/promises');
+
+const ARCHIVE = /^circus-(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})\.tar\.gz$/;
+const PART = /^circus-\d{8}-\d{6}\.tar\.gz\.part$/;
+const EXCLUDE = [/^runtime.*\.json$/, /\.lock$/, /\.pid$/, /\.tmp$/];
+const DEFAULT_SETTINGS = { enabled: false, time: '03:30', retention: { mode: 'count', value: 14 } };
+const RETENTION_LIMITS = { count: [1, 365], days: [1, 3650] };
+const CATCH_UP_MS = 24 * 3600e3;
+
+class BackupError extends Error { constructor(code, msg) { super(msg); this.code = code; } }
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+const pad = n => String(n).padStart(2, '0');
+const stampOf = d => `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`;
+const archiveTime = name => { const m = ARCHIVE.exec(name); return m ? new Date(+m[1], m[2] - 1, +m[3], +m[4], +m[5], +m[6]).getTime() : NaN; };
+const message = e => String((e && e.message) || e).slice(0, 500);
+
+function validateSettings(input, current = DEFAULT_SETTINGS) {
+  const next = { ...current, retention: { ...current.retention } };
+  if (input.enabled !== undefined) {
+    if (typeof input.enabled !== 'boolean') throw new BackupError(400, 'enabled must be true or false');
+    next.enabled = input.enabled;
+  }
+  if (input.time !== undefined) {
+    if (!/^([01]\d|2[0-3]):[0-5]\d$/.test(String(input.time))) throw new BackupError(400, 'The backup time must be HH:MM');
+    next.time = String(input.time);
+  }
+  if (input.retention !== undefined) {
+    const mode = input.retention && input.retention.mode, value = Number(input.retention && input.retention.value);
+    const limits = RETENTION_LIMITS[mode];
+    if (!limits) throw new BackupError(400, 'Retention is "count" or "days"');
+    if (!Number.isInteger(value) || value < limits[0] || value > limits[1]) throw new BackupError(400, `Keep ${limits[0]}–${limits[1]} ${mode === 'count' ? 'backups' : 'days'}`);
+    next.retention = { mode, value };
+  }
+  return next;
+}
+
+// Next local occurrence of HH:MM strictly after `now` (process TZ).
+function nextOccurrence(time, now) {
+  const [hh, mm] = time.split(':').map(Number), d = new Date(now);
+  const at = new Date(d.getFullYear(), d.getMonth(), d.getDate(), hh, mm, 0, 0);
+  if (at.getTime() <= now) at.setDate(at.getDate() + 1);
+  return at.getTime();
+}
+
+// ---------- tar (ustar + PAX path records), enough for `tar -xzf` everywhere ----------
+function octal(buf, offset, length, value) { buf.write(value.toString(8).padStart(length - 1, '0') + '\0', offset, length, 'ascii'); }
+function rawHeader(name, { size = 0, mode = 0o644, uid = 0, gid = 0, mtime = 0, type = '0' }) {
+  const b = Buffer.alloc(512);
+  b.write(name, 0, 100, 'utf8');
+  octal(b, 100, 8, mode & 0o7777); octal(b, 108, 8, uid & 0o7777777); octal(b, 116, 8, gid & 0o7777777);
+  octal(b, 124, 12, size); octal(b, 136, 12, Math.max(0, Math.floor(mtime / 1000)));
+  b.write('        ', 148, 8, 'ascii');
+  b.write(type, 156, 1, 'ascii');
+  b.write('ustar\0' + '00', 257, 8, 'ascii');
+  let sum = 0; for (const byte of b) sum += byte;
+  b.write(sum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii');
+  return b;
+}
+const padding = size => Buffer.alloc((512 - (size % 512)) % 512);
+function paxRecord(key, value) {
+  const body = ` ${key}=${value}\n`, bytes = Buffer.byteLength(body);
+  let length = bytes + String(bytes).length;
+  if (String(length).length !== String(bytes).length) length = bytes + String(length).length;
+  return `${length}${body}`;
+}
+function tarEntry(name, data, meta) {
+  const parts = [];
+  // Names longer than the ustar field or non-ASCII go into a PAX header first.
+  if (Buffer.byteLength(name) > 100 || /[^\x20-\x7e]/.test(name)) {
+    const pax = Buffer.from(paxRecord('path', name), 'utf8');
+    parts.push(rawHeader('PaxHeader/' + name.replace(/[^\x20-\x7e]/g, '_').slice(-80), { size: pax.length, type: 'x', mtime: meta.mtime }), pax, padding(pax.length));
+    name = name.replace(/[^\x20-\x7e]/g, '_').slice(-100);
+  }
+  parts.push(rawHeader(name, { ...meta, size: data ? data.length : 0 }));
+  if (data && data.length) parts.push(data, padding(data.length));
+  return parts;
+}
+
+// Streaming reader for verification and tests: [{ name, type, size, mode, mtime, data? }].
+async function readArchive(file, { contents = false } = {}) {
+  const entries = []; let pending = Buffer.alloc(0), current = null, paxPath = null, ended = false;
+  const consume = chunk => {
+    pending = pending.length ? Buffer.concat([pending, chunk]) : chunk;
+    for (;;) {
+      if (ended) return;
+      if (current) {
+        if (pending.length < current.need) return;
+        const body = pending.subarray(0, current.size);
+        if (current.entry.type === 'x') paxPath = /(?:^|\n)\d+ path=([^\n]*)\n/.exec(body.toString('utf8'))?.[1] ?? null;
+        else { if (contents) current.entry.data = Buffer.from(body); entries.push(current.entry); }
+        pending = pending.subarray(current.need); current = null; continue;
+      }
+      if (pending.length < 512) return;
+      const h = pending.subarray(0, 512); pending = pending.subarray(512);
+      if (h.every(byte => byte === 0)) { ended = true; return; }
+      let sum = 0; for (let i = 0; i < 512; i++) sum += i >= 148 && i < 156 ? 32 : h[i];
+      if (sum !== parseInt(h.toString('ascii', 148, 156), 8)) throw new Error('Corrupt tar header');
+      const field = (o, l) => h.toString('utf8', o, o + l).replace(/\0.*$/s, '');
+      const size = parseInt(field(124, 12), 8) || 0, type = field(156, 1) || '0';
+      const prefix = field(345, 155), name = prefix ? prefix + '/' + field(0, 100) : field(0, 100);
+      const entry = { name: type === 'x' ? name : (paxPath ?? name), type, size, mode: parseInt(field(100, 8), 8), mtime: parseInt(field(136, 12), 8) * 1000 };
+      if (type !== 'x') paxPath = null;
+      current = { entry, size, need: size + padding(size).length };
+    }
+  };
+  await pipeline(fs.createReadStream(file), zlib.createGunzip(), async function* (source) { for await (const chunk of source) consume(chunk); });
+  if (!ended) throw new Error('Archive ends without the tar end marker');
+  return entries;
+}
+
+class BackupService {
+  /**
+   * dataRoot     DATA (read)            backupRoot  BACKUP (rw) or '' = feature off
+   * stateFile    settings + history     snapshotDb  () => db.json text from memory
+   * scraper      { state(), control(action) } or null
+   */
+  constructor({ dataRoot, backupRoot, stateFile, snapshotDb, scraper = null, now = Date.now, log = console,
+    idleTimeoutMs = 15 * 60e3, pollMs = 2000, resumeRetryMs = 5000 }) {
+    this.dataRoot = path.resolve(dataRoot);
+    this.backupRoot = backupRoot ? path.resolve(backupRoot) : '';
+    this.stateFile = stateFile || path.join(this.dataRoot, 'backup.json');
+    this.snapshotDb = snapshotDb; this.scraper = scraper; this.now = now; this.log = log;
+    this.idleTimeoutMs = idleTimeoutMs; this.pollMs = pollMs; this.resumeRetryMs = resumeRetryMs;
+    this.state = { version: 1, settings: { ...DEFAULT_SETTINGS }, nextAt: null, last: null, history: [], pending: null };
+    this.running = null; this.archives = []; this.targetError = null; this.timer = null;
+  }
+
+  load() {
+    try { Object.assign(this.state, JSON.parse(fs.readFileSync(this.stateFile, 'utf8'))); }
+    catch (e) { if (e.code !== 'ENOENT') this.log.warn(`[backup] ignoring unreadable ${this.stateFile}: ${e.message}`); }
+    this.state.settings = validateSettings(this.state.settings || {}, DEFAULT_SETTINGS);
+    this.state.history = Array.isArray(this.state.history) ? this.state.history.slice(-10) : [];
+    return this;
+  }
+
+  persist() {
+    fs.writeFileSync(this.stateFile + '.tmp', JSON.stringify(this.state, null, 2) + '\n');
+    fs.renameSync(this.stateFile + '.tmp', this.stateFile);
+  }
+
+  // A run interrupted by a restart may have left the queue paused on its behalf.
+  async recoverInterrupted() {
+    const pending = this.state.pending;
+    if (!pending) return;
+    this.state.pending = null;
+    const result = { at: pending.startedAt, trigger: pending.trigger, error: 'Interrupted by a reader restart', warnings: [] };
+    if (pending.pausedByUs) {
+      try { await this.resume(); result.warnings.push('Queue resumed after the interrupted backup'); }
+      catch (e) { result.warnings.push(`Queue left paused: ${message(e)}`); }
+    }
+    this.record(result);
+  }
+
+  async refreshTarget() {
+    if (!this.backupRoot) { this.targetError = null; this.archives = []; return; }
+    try {
+      const st = await fsp.stat(this.backupRoot);
+      if (!st.isDirectory()) throw new Error('not a directory');
+      await fsp.access(this.backupRoot, fs.constants.W_OK);
+      const names = (await fsp.readdir(this.backupRoot)).filter(name => ARCHIVE.test(name)).sort();
+      this.archives = await Promise.all(names.map(async name => ({ name, size: (await fsp.stat(path.join(this.backupRoot, name))).size, at: archiveTime(name) })));
+      this.targetError = null;
+    } catch (e) { this.targetError = `Backup folder ${this.backupRoot} is not usable: ${e.code || message(e)}`; this.archives = []; }
+  }
+
+  publicState() {
+    const s = this.state;
+    return {
+      configured: !!this.backupRoot, path: this.backupRoot || null, targetError: this.targetError,
+      settings: s.settings, nextAt: s.settings.enabled && this.backupRoot ? s.nextAt : null,
+      running: this.running && { ...this.running }, last: s.last, history: s.history.slice().reverse(),
+      archives: { count: this.archives.length, bytes: this.archives.reduce((n, a) => n + a.size, 0),
+        newest: this.archives.length ? this.archives[this.archives.length - 1].name : null },
+    };
+  }
+
+  updateSettings(input) {
+    this.state.settings = validateSettings(input || {}, this.state.settings);
+    this.state.nextAt = this.state.settings.enabled ? nextOccurrence(this.state.settings.time, this.now()) : null;
+    this.persist();
+    return this.publicState();
+  }
+
+  start(intervalMs = 30000) {
+    this.load();
+    this.refreshTarget().then(() => this.recoverInterrupted()).catch(e => this.log.error('[backup]', message(e)));
+    this.timer = setInterval(() => this.tick(), intervalMs);
+    this.timer.unref?.();
+    return this;
+  }
+
+  stop() { clearInterval(this.timer); }
+
+  // Scheduled runs: catch up a missed slot from the last day, never older ones
+  // (e.g. right after restoring an old backup.json).
+  async tick() {
+    const s = this.state.settings, now = this.now();
+    if (!s.enabled || !this.backupRoot || this.running) return null;
+    if (!this.state.nextAt) { this.state.nextAt = nextOccurrence(s.time, now); this.persist(); return null; }
+    if (now < this.state.nextAt) return null;
+    const due = now - this.state.nextAt < CATCH_UP_MS;
+    this.state.nextAt = nextOccurrence(s.time, now); this.persist();
+    return due ? this.run('schedule').catch(() => null) : null;
+  }
+
+  /** Starts a run and resolves with its result; rejects at once when no run is possible. */
+  run(trigger = 'manual') {
+    if (!this.backupRoot) return Promise.reject(new BackupError(409, 'Backups are off: set BACKUP to a mounted folder'));
+    if (this.running) return Promise.reject(new BackupError(409, 'A backup is already running'));
+    this.running = { startedAt: this.now(), trigger, phase: 'starting' };
+    return this.perform(trigger).finally(() => { this.running = null; });
+  }
+
+  async perform(trigger) {
+    const started = this.now(), result = { at: started, trigger, warnings: [] };
+    let part = null, pausedByUs = false;
+    try {
+      await this.refreshTarget();
+      if (this.targetError) throw new Error(this.targetError);
+      await this.removeStaleParts();
+      // 1. Pause the queue (unless already paused) and wait for the active chapter.
+      let scraperState = null;
+      if (this.scraper) {
+        try { scraperState = await this.scraper.state(); }
+        catch (e) { result.warnings.push(`Scraper unreachable, queue not paused: ${message(e)}`); }
+      }
+      if (scraperState) {
+        result.queueWasPaused = !!scraperState.paused;
+        if (!scraperState.paused) {
+          this.state.pending = { startedAt: started, trigger, pausedByUs: true }; this.persist();
+          await this.scraper.control('pause'); pausedByUs = true;
+        }
+        this.running.phase = 'waiting for the scraper to finish its current step';
+        await this.waitIdle(scraperState);
+      }
+      // 2. Snapshot DATA into BACKUP/<name>.part, then rename.
+      this.running.phase = 'writing archive';
+      let stamp = started, name, file;
+      do { name = `circus-${stampOf(new Date(stamp))}.tar.gz`; file = path.join(this.backupRoot, name); stamp += 1000; }
+      while (fs.existsSync(file)); // two runs within one second must not overwrite each other
+      part = file + '.part';
+      const stats = await this.writeArchive(part);
+      this.running.phase = 'verifying archive';
+      const entries = await readArchive(part);
+      const files = entries.filter(e => e.type === '0').length;
+      if (files !== stats.files) throw new Error(`Verification read ${files} files, expected ${stats.files}`);
+      await fsp.rename(part, file); part = null;
+      Object.assign(result, { file: name, size: (await fsp.stat(file)).size, files, dataBytes: stats.bytes });
+    } catch (e) {
+      result.error = message(e);
+      if (part) await fsp.rm(part, { force: true }).catch(() => {});
+    } finally {
+      // 3. Resume only what this run paused.
+      if (pausedByUs) {
+        this.running.phase = 'resuming queue';
+        try { await this.resume(); result.queueResumed = true; }
+        catch (e) { result.warnings.push(`Queue could not be resumed, resume it in Settings → Scraper: ${message(e)}`); }
+      }
+      if (this.state.pending) this.state.pending = null;
+    }
+    // 4. Retention only after a successful run, so failures never thin out old copies.
+    if (!result.error) {
+      this.running.phase = 'applying retention';
+      try { result.deleted = await this.applyRetention(result.file); }
+      catch (e) { result.warnings.push(`Retention failed: ${message(e)}`); }
+    }
+    result.durationMs = this.now() - started;
+    await this.refreshTarget();
+    this.record(result);
+    if (result.error) this.log.error(`[backup] failed: ${result.error}`);
+    else this.log.log(`[backup] ${result.file} ${result.size} bytes in ${result.durationMs} ms`);
+    return result;
+  }
+
+  record(result) {
+    this.state.last = result;
+    this.state.history = [...this.state.history, result].slice(-10);
+    this.persist();
+  }
+
+  async waitIdle(initial) {
+    const deadline = this.now() + this.idleTimeoutMs;
+    let state = initial, lastError = null;
+    for (;;) {
+      if (state && !(state.active || []).length && state.status !== 'pausing') return;
+      if (this.now() >= deadline) {
+        throw new Error(`The scraper did not become idle within ${Math.round(this.idleTimeoutMs / 60000)} min${lastError ? ` (${lastError})` : ''}; no backup written`);
+      }
+      await delay(this.pollMs);
+      try { state = await this.scraper.state(); lastError = null; }
+      catch (e) { state = null; lastError = message(e); }
+    }
+  }
+
+  async resume() {
+    let error;
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try { await this.scraper.control('resume'); return; }
+      catch (e) { error = e; if (attempt < 2) await delay(this.resumeRetryMs); }
+    }
+    throw error;
+  }
+
+  // Files of DATA in a stable order, skipping excluded names, symlinks and a nested BACKUP.
+  async collect(dir = this.dataRoot, rel = '') {
+    const out = [];
+    const entries = (await fsp.readdir(dir, { withFileTypes: true })).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const entry of entries) {
+      const full = path.join(dir, entry.name), relative = rel ? rel + '/' + entry.name : entry.name;
+      if (entry.isDirectory()) {
+        if (full === this.backupRoot) continue;
+        out.push({ full, relative, dir: true });
+        out.push(...await this.collect(full, relative));
+      } else if (entry.isFile() && !EXCLUDE.some(re => re.test(entry.name))) out.push({ full, relative, dir: false });
+    }
+    return out;
+  }
+
+  // Contents of one file as it goes into the archive.
+  async contentOf(item) {
+    if (item.relative === 'db.json' && this.snapshotDb) return Buffer.from(this.snapshotDb());
+    if (item.relative === path.relative(this.dataRoot, this.stateFile).split(path.sep).join('/')) {
+      const { pending, ...rest } = this.state; void pending;
+      return Buffer.from(JSON.stringify(rest, null, 2) + '\n');
+    }
+    const data = await fsp.readFile(item.full);
+    if (item.relative === 'scraper/queue.json') {
+      const queue = JSON.parse(data.toString('utf8'));
+      queue.paused = true; queue.status = 'paused';
+      return Buffer.from(JSON.stringify(queue, null, 2) + '\n');
+    }
+    return data;
+  }
+
+  async writeArchive(part) {
+    const items = await this.collect(), stats = { files: 0, bytes: 0 };
+    // The in-memory db is archived even before its first debounced write.
+    if (this.snapshotDb && !items.some(item => item.relative === 'db.json')) items.unshift({ full: path.join(this.dataRoot, 'db.json'), relative: 'db.json', dir: false, memory: true });
+    const manifest = { kind: 'mangacircus-data-backup', version: 1, created: new Date(this.now()).toISOString(),
+      excluded: 'runtime*.json, *.lock, *.pid, *.tmp', queuePaused: true };
+    const self = this;
+    async function* tar() {
+      yield* tarEntry('.circus-backup.json', Buffer.from(JSON.stringify(manifest, null, 2) + '\n'), { mtime: self.now() });
+      for (const item of items) {
+        let st;
+        try { st = item.memory ? { mode: 0o644, uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0, mtimeMs: self.now() } : await fsp.stat(item.full); }
+        catch (e) { if (e.code === 'ENOENT') continue; throw e; }
+        const meta = { mode: st.mode, uid: st.uid, gid: st.gid, mtime: st.mtimeMs };
+        if (item.dir) { yield* tarEntry(item.relative + '/', null, { ...meta, type: '5' }); continue; }
+        let data;
+        try { data = await self.contentOf(item); } catch (e) { if (e.code === 'ENOENT') continue; throw new Error(`${item.relative}: ${message(e)}`); }
+        stats.files++; stats.bytes += data.length;
+        yield* tarEntry(item.relative, data, meta);
+      }
+      yield Buffer.alloc(1024);
+    }
+    await pipeline(Readable.from(tar()), zlib.createGzip(), fs.createWriteStream(part, { flags: 'wx' }));
+    const handle = await fsp.open(part, 'r+'); // fsync before the rename makes it visible
+    try { await handle.sync(); } finally { await handle.close(); }
+    stats.files++; // the manifest
+    return stats;
+  }
+
+  async removeStaleParts() {
+    for (const name of await fsp.readdir(this.backupRoot)) if (PART.test(name)) await fsp.rm(path.join(this.backupRoot, name), { force: true });
+  }
+
+  // Deletes only our own archive names directly inside BACKUP; never the newest one.
+  async applyRetention(keepName) {
+    const { mode, value } = this.state.settings.retention;
+    const names = (await fsp.readdir(this.backupRoot)).filter(name => ARCHIVE.test(name)).sort();
+    let doomed;
+    if (mode === 'count') doomed = names.slice(0, Math.max(0, names.length - value));
+    else { const cutoff = this.now() - value * 864e5; doomed = names.filter(name => archiveTime(name) < cutoff); }
+    doomed = doomed.filter(name => name !== keepName && name !== names[names.length - 1]);
+    for (const name of doomed) {
+      const target = path.join(this.backupRoot, name);
+      if (path.dirname(target) !== this.backupRoot) continue;
+      await fsp.rm(target);
+    }
+    return doomed.length;
+  }
+}
+
+module.exports = { BackupService, BackupError, readArchive, validateSettings, nextOccurrence, DEFAULT_SETTINGS, ARCHIVE };

docker-compose.yml

@@ -36,10 +36,17 @@ services:
         source: ${DATA_PATH:-./data}
         target: /data
         bind: {create_host_path: false}
+      # Built-in backups (docs/backup.md). The folder must exist and be writable by
+      # uid 1000; backups stay off unless BACKUP_PATH is set.
+      - type: bind
+        source: ${BACKUP_PATH:-./backup}
+        target: /backup
+        bind: {create_host_path: false}
     environment:
       PORT: "8080"
       DATA: /data
       LIBRARY: /library
+      BACKUP: ${BACKUP_PATH:+/backup}
       TZ: ${TZ:-Europe/Berlin}
       SIGNUP: closed
       SCAN_INTERVAL: "10"

docs/architecture.md

@@ -18,6 +18,7 @@ Related documents:
 - [peering.md](peering.md): optional private server-to-server catalog and transfer.
 - [discovery.md](discovery.md): reader-owned, opt-in deterministic discovery feed and its limits.
 - [source-modules.md](source-modules.md): adding and maintaining first-party hosters inside the single scraper process.
+- [backup.md](backup.md): built-in scheduled backup of `data/` and the restore procedure.
 
 ## 1. Principles
 
@@ -67,6 +68,7 @@ survives restarts, so restarting does not itself force another complete walk.
 |------|------:|----------------|
 | `server.js` | ~350 | HTTP server and router, auth and sessions, JSON storage, API handlers, scan scheduling, notifications and webhooks |
 | `library.js` | ~170 | Folder scanner, name parsing, image-size reader, scan cache |
+| `backup.js` | ~330 | Built-in `DATA` backup: queue pause/resume, tar.gz writer and verifier, schedule, retention |
 | `public/common.js` | ~85 | Browser helpers: `h()` element builder, `api()` fetch wrapper, top navigation with the notification bell |
 | `public/index.html` + `library.js` | ~60 | Library overview (cover grid, filter, sort) |
 | `public/manga.html` + `manga.js` | ~110 | Manga page: info, follow, chapter list with book separators, versions and votes, read/unread toggle |
@@ -157,7 +159,11 @@ This keeps ids human-readable and derivable without a database. The price: renam
 - `data/library-index.json` holds a disposable, versioned index snapshot and shallow
   directory stamps. The library folders remain authoritative; a missing or invalid
   snapshot triggers a full scan. Rollback-compatible `scan-cache.json` is retained.
-- **Backups:** copy `data/db.json`. The atomic writes make a copy at any time consistent.
+- `data/backup.json` holds the built-in backup schedule, retention and recent results.
+- **Backups:** the reader's built-in backup ([backup.md](backup.md)) archives the whole
+  `data/` tree into `BACKUP` on a schedule, with `db.json` taken from memory and the
+  queue paused around the copy. A plain copy of `data/db.json` also works: atomic
+  writes make it consistent at any time.
 
 ## 6. Server
 
@@ -199,6 +205,7 @@ All requests and responses are JSON. POSTs require `Content-Type: application/js
 | `POST /api/admin/rescan` `{full}` | admin | Scan now, and wait for the result |
 | `POST /api/admin/user` `{name, password, admin}` | admin | Create a user or reset a password |
 | `POST /api/admin/user/delete` `{name}` | admin | Delete a user and their data |
+| `GET /api/admin/backup` · `POST /api/admin/backup` `{command:"settings"\|"run"}` | admin | Built-in backup status, schedule/retention, "back up now" ([backup.md](backup.md)) |
 
 ### 6.3 Authentication and security
 
@@ -342,7 +349,8 @@ docker compose up -d --build
 - **Volumes:**
   - `/library`: read-only in the reader and writable in the scraper. Files must be readable—and scraper targets writable—by uid 1000.
   - `/data`: reader state must be writable; the scraper also needs its writable `/data/scraper` state.
-- **Configuration:** process wiring uses environment variables (`PORT`, `LIBRARY`, `DATA`, `SIGNUP`, `SCAN_INTERVAL`, `FULL_SCAN_INTERVAL`, `COOKIE_SECURE`, `SCRAPER_URL`, and `SCRAPER_TOKEN`). Queue pacing and automatic-update schedule are persisted in `data/scraper/queue.json` and edited through the authenticated admin UI.
+  - `/backup`: reader only, writable target of the built-in backup (`BACKUP_PATH`; off when unset).
+- **Configuration:** process wiring uses environment variables (`PORT`, `LIBRARY`, `DATA`, `BACKUP`, `SIGNUP`, `SCAN_INTERVAL`, `FULL_SCAN_INTERVAL`, `COOKIE_SECURE`, `SCRAPER_URL`, and `SCRAPER_TOKEN`). Queue pacing and automatic-update schedule are persisted in `data/scraper/queue.json` and edited through the authenticated admin UI.
 - **Without Docker:** `npm run start:all` with Node ≥ 18, or supervise `npm start` and `npm run start:scraper` independently. On the current Windows machine that supervision is already owned by `\MangaCircus Supervisor`.
 - **Behind a reverse proxy:** terminate TLS, set `COOKIE_SECURE=1`. A sub-path works.
 - **Upgrades:** replace the code and restart. Reader state has no explicit migration;
@@ -366,6 +374,7 @@ docker compose up -d --build
 |----------|--------------|-----|
 | Index the folder tree in place | Import into a DB / managed storage | Stays readable by any viewer and writable by any tool. Nothing to keep in sync |
 | No dependencies, no framework, no build | Express, React, bundler | Minimal code, maximum longevity, runs anywhere Node runs |
+| Built-in backup in the reader, queue paused, no container stop | Host job that stops the stack; backup in Sentinel | Only the reader sees all of `/data` and the in-memory `db.json`; the scraper's pause API gives a consistent point without downtime |
 | JSON file for user data | SQLite (`node:sqlite` is still experimental in Node 22/24), Postgres | Zero dependencies, trivial backup, enough for the target size. Easy to swap: all access goes through the `db` object |
 | Page roles as file-name tags | Page lists in JSON | Visible in any file browser, survives copying, no second source of truth. Unknown tags are harmless |
 | Image sizes read on the server | Measure in the browser after load | Layout before load: exact scroll restore, zoom anchoring, auto-detected double pages, no layout jumps |

docs/backup.md

@@ -0,0 +1,114 @@
+# Built-in backup
+
+The reader can back up its own `DATA` folder on a schedule set in
+**Settings → Admin → Service health → Backups**, without stopping any container.
+The library is not part of it (it is plain files; back it up with the storage that
+holds it). Implementation: [`backup.js`](../backup.js); tests: `test/backup.test.js`.
+
+## Turning it on
+
+| Where | Setting |
+|---|---|
+| Native | `BACKUP=/path/to/backups` for `server.js`. Unset = feature off |
+| Compose | `BACKUP_PATH=/host/folder` in `.env`. The reader mounts it rw at `/backup` and gets `BACKUP=/backup`; without `BACKUP_PATH` the variable stays empty and backups are off |
+
+The folder must already exist and be writable by uid 1000 (`create_host_path: false`:
+a missing folder makes `docker compose up` fail instead of silently creating an empty
+one on the wrong disk). Use another disk or a NAS share, never a folder inside `DATA`.
+Without `BACKUP_PATH`, Compose still mounts the unused default `./backup`, so that
+folder must exist too (`node tools/prepare-compose.js` creates it). Upgrading an
+existing deployment therefore needs either `BACKUP_PATH` in `.env` or an empty
+`backup/` beside `docker-compose.yml`.
+
+Archives contain password hashes, sessions and the peering private key: keep the
+target private.
+
+### Why the reader owns it
+
+The reader is the only service that mounts all of `/data` (the scraper sees only
+`/data/scraper`, Sentinel only `/data/sentinel`), it holds `db.json` in memory and owns
+its atomic writer, it already talks to the scraper's private control API with the
+shared token, and the admin settings UI lives there. Running it in Sentinel would need
+a full read-write `/data` mount there and break its "writes only `data/sentinel/`"
+contract (see [architecture.md §3](architecture.md#3-components)).
+
+## Settings and status
+
+Admin only, under Service health (`GET/POST /api/admin/backup`, also included in
+`GET /api/admin/health` as `backup`):
+
+- **Back up daily at HH:MM** (container time zone `TZ`) and an enable switch.
+- **Keep the newest** N backups (`count`, 1–365) or N days (`days`, 1–3650).
+- **Back up now** starts a run immediately; the page polls every 2 s while it runs.
+- Status: target folder, number/size of archives, the running phase, last backup
+  (time, trigger, size, duration, file), its error and warnings, next scheduled run.
+
+Settings and the last 10 results persist atomically in `data/backup.json`. A slot
+missed while the reader was down runs once at the next start if it is less than a
+day old; older slots (for example after restoring an old `backup.json`) are skipped.
+
+```text
+POST /api/admin/backup {command:"settings", settings:{enabled, time:"HH:MM", retention:{mode:"count"|"days", value}}}
+POST /api/admin/backup {command:"run"}       -> 409 when off or already running
+```
+
+## What one run does
+
+1. Ask the scraper for its state (`GET /api/state`). If the queue runs, pause it
+   (`POST /api/control {action:"pause"}`) and remember that this run paused it.
+2. Wait until no job is active (the worker stops at its next page/chapter boundary),
+   polling every 2 s for at most 15 minutes. On timeout the run fails, writes nothing
+   and resumes the queue if it paused it. An unreachable scraper does not block the
+   backup; that is recorded as a warning.
+3. Write `BACKUP/circus-YYYYMMDD-HHMMSS.tar.gz.part`: a gzip'd tar of `DATA` with
+   paths relative to `DATA`, fsync it, read it back to verify, rename it to `.tar.gz`.
+   - `db.json` is serialized from the reader's in-memory state at that moment.
+   - `scraper/queue.json` is stored with `paused: true` (status `paused`), so a
+     restored copy never starts downloading by itself. The live file is not touched.
+   - `scan-cache.json`, `library-index.json`, `catalog.json`, peering, Sentinel state,
+     logs and everything else under `DATA` is copied as found; all of them are
+     written by atomic rename, so each file is complete.
+   - Left out: `runtime*.json`, `*.lock` (`daemon.lock`, `supervisor.lock`), `*.pid`,
+     `*.tmp`, symlinks, and the `BACKUP` folder itself if it lies inside `DATA`.
+   - `.circus-backup.json` at the archive root records when and how it was made.
+4. Resume the queue only if this run paused it (three attempts; a failure is shown
+   as a warning: resume it under Settings → Scraper).
+5. After a successful run only, apply retention. It deletes only files named
+   `circus-YYYYMMDD-HHMMSS.tar.gz` directly inside `BACKUP` (age from the name),
+   never the newest one, never other files or subfolders. Stale
+   `circus-*.tar.gz.part` files from an interrupted run are removed at the next run.
+
+If the reader restarts during a run, `backup.json` still carries the run's
+"queue paused by backup" marker; the next start resumes the queue and records the
+run as interrupted. The marker is never part of an archive.
+
+Measured on a copy of a production data folder (54 MB, 1000 files): 5.8 MB archive in
+about 5 s including verification. Pauses last until the current chapter is done
+plus those seconds.
+
+## Restore
+
+Restoring replaces the user data with the state of the archive. Reading progress
+and queue changes made after that backup are lost.
+
+1. Stop the stack (or at least reader, scraper and Sentinel):
+   `docker compose stop reader scraper sentinel`.
+2. Move the current data folder aside instead of deleting it:
+   `mv /srv/mangacircus/data /srv/mangacircus/data.before-restore`.
+3. Unpack the archive into an empty data folder owned by uid 1000:
+   ```bash
+   mkdir /srv/mangacircus/data
+   tar -xzf /path/to/circus-YYYYMMDD-HHMMSS.tar.gz -C /srv/mangacircus/data
+   chown -R 1000:1000 /srv/mangacircus/data
+   ```
+   (`--same-owner` is the default for root; the archive keeps the original uid/gid.)
+4. Start the reader first: `docker compose up -d --no-build reader`. Check `/health`,
+   log in, check progress and library. The reader rebuilds a missing or stale library
+   index by itself.
+5. Start `scraper sentinel`. The restored queue is paused: check it under
+   Settings → Scraper and resume it when it looks right.
+
+The scraper token is not in `DATA`; Compose keeps it under `.secrets/` (or the
+deployment's `SCRAPER_TOKEN_PATH`). Restore tests should unpack into a separate
+folder and run an isolated stack (no second writer against the live library), as
+a restore-test script does for the host-level archives.

docs/settings-navigation.md

@@ -59,6 +59,10 @@ download times by manga/source, and a collapsed recent-check log. Successful
 no-change nightly checks update this summary and leave the queue. Sources owns
 the per-hoster request and chapter delay settings.
 
+Service health also holds the **Backups** card: built-in backup status (last run,
+size, duration, error), **Back up now**, and the daily schedule and retention form,
+whose draft survives the page's polling. See [backup.md](backup.md).
+
 Reading-time collection controls, export and deletion live under Reading time &
 privacy; reader layout controls stay under Reading. Rescan controls and user
 administration are separate. Server & library describes existing maintenance,

public/health.js

@@ -4,10 +4,13 @@
   try { me = await api('api/me'); } catch (e) { return showError(e); }
   topNav(me);
   if (!me.admin) return showError(new Error('Administrator access required.'));
-  const main = settingsShell(me, 'admin/health').content;
+  const shell = settingsShell(me, 'admin/health').content;
+  const main = h('div'), backup = backupCard();
+  shell.append(main, backup.card);
   async function load() {
     try {
       const [health, scraper] = await Promise.all([api('api/admin/health'), api('api/admin/scraper')]);
+      backup.draw(health.backup);
       main.textContent = '';
       main.append(h('section', { class: 'card' },
         h('p', {}, `Reader: ${health.reader.ok ? 'healthy' : 'scan error'} · Sentinel: ${health.sentinel.status} · Scraper: ${scraper.status || 'unavailable'}`),
@@ -28,7 +31,56 @@
       main.append(h('details', { class: 'card', open: incidents.some(i => i.state === 'open') }, h('summary', {}, 'Incident & recovery history'), incidents.slice(-50).map(i =>
         h('p', {}, `${i.at} · ${i.state} · ${i.message || i.failure?.message || ''}${i.resolution ? ' · ' + i.resolution : ''}`))));
     } catch (e) { main.textContent = ''; main.append(h('p', { class: 'error', role: 'alert' }, e.message)); }
-    setTimeout(load, document.hidden ? 30000 : 10000);
+    setTimeout(load, document.hidden ? 30000 : backup.busy() ? 2000 : 10000);
   }
   load();
+
+  // Built-in backup: status redraws on every poll; the schedule form keeps its draft.
+  function backupCard() {
+    const mb = n => (n / 1048576).toFixed(n < 10485760 ? 1 : 0) + ' MB';
+    const secs = ms => ms < 60000 ? Math.round(ms / 1000) + ' s' : Math.round(ms / 60000) + ' min';
+    const status = h('div'), msg = h('p', { class: 'small', role: 'status' });
+    const enabled = h('input', { type: 'checkbox' }), time = h('input', { type: 'time', required: true });
+    const mode = h('select', {}, h('option', { value: 'count' }, 'backups'), h('option', { value: 'days' }, 'days'));
+    const keep = h('input', { type: 'number', min: '1', step: '1', style: 'width:6em' });
+    let state = null, filled = false;
+    const fill = s => { enabled.checked = s.enabled; time.value = s.time; mode.value = s.retention.mode; keep.value = s.retention.value; filled = true; };
+    const command = async (body, button) => {
+      button.disabled = true; msg.textContent = ''; msg.className = 'small';
+      try { const next = await api('api/admin/backup', body); if (body.command === 'settings') { fill(next.settings); msg.textContent = 'Saved.'; } draw(next); }
+      catch (e) { msg.textContent = e.message; msg.className = 'small error'; }
+      finally { button.disabled = false; }
+    };
+    const runButton = h('button', { class: 'btn primary', onclick: e => command({ command: 'run' }, e.target) }, 'Back up now');
+    const saveButton = h('button', { class: 'btn', onclick: e => command({ command: 'settings', settings: {
+      enabled: enabled.checked, time: time.value, retention: { mode: mode.value, value: Number(keep.value) } } }, e.target) }, 'Save schedule');
+    const form = h('div', { class: 'field' }, h('span', {}, 'Schedule'),
+      h('div', { class: 'row' }, h('label', { class: 'row' }, enabled, 'Back up daily at'), time),
+      h('div', { class: 'row', style: 'margin-top:8px' }, h('span', {}, 'Keep the newest'), keep, mode),
+      h('div', { class: 'row', style: 'margin-top:8px' }, saveButton, runButton), msg);
+    const card = h('section', { class: 'card', id: 'backup' }, h('h2', {}, 'Backups'), status, form);
+    function draw(next) {
+      if (!next) return;
+      state = next;
+      if (!filled) fill(next.settings);
+      const last = next.last, lines = [];
+      if (!next.configured) lines.push(h('p', { class: 'muted' }, 'Backups are off: mount a folder and set BACKUP_PATH (Compose) or BACKUP (native). See docs/backup.md.'));
+      else {
+        lines.push(h('p', {}, `Target: ${next.path} · ${next.archives.count} archive${next.archives.count === 1 ? '' : 's'}, ${mb(next.archives.bytes)}`));
+        if (next.targetError) lines.push(h('p', { class: 'error' }, next.targetError));
+      }
+      if (next.running) lines.push(h('p', { class: 'tag' }, `Running (${next.running.trigger}): ${next.running.phase}…`));
+      if (last) {
+        lines.push(h('p', {}, `Last backup: ${new Date(last.at).toLocaleString()} (${last.trigger})`,
+          last.error ? '' : ` · ${mb(last.size)} · ${secs(last.durationMs || 0)} · ${last.file}`));
+        if (last.error) lines.push(h('p', { class: 'error', role: 'alert' }, `Failed: ${last.error}`));
+        for (const warning of last.warnings || []) lines.push(h('p', { class: 'small error' }, warning));
+      } else if (next.configured) lines.push(h('p', { class: 'muted' }, 'No backup yet.'));
+      lines.push(h('p', { class: 'muted small' }, next.nextAt ? `Next scheduled backup: ${new Date(next.nextAt).toLocaleString()}` : 'No scheduled backup.'));
+      status.textContent = ''; status.append(...lines);
+      runButton.disabled = !next.configured || !!next.running;
+      form.hidden = !next.configured && !next.settings.enabled;
+    }
+    return { card, draw, busy: () => !!state?.running };
+  }
 })();

server.js

@@ -10,6 +10,7 @@
  *   COOKIE_SECURE=1      mark the session cookie Secure (when served over HTTPS)
  *   SCRAPER_URL=http://127.0.0.1:8091   scraper daemon API
  *   SCRAPER_TOKEN=...                    shared daemon token (optional on loopback)
+ *   BACKUP=/backup       folder for built-in DATA backups (unset = backups off)
  */
 const http = require('http'), fs = require('fs'), path = require('path'), crypto = require('crypto');
 const { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex } = require('./library');
@@ -26,6 +27,7 @@ const {
   applyPulse, applyVisit, chapterInsight, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary,
 } = require('./reading-analytics');
 const { parseSchedule, publicSchedule, loadQuestionBank, createReadingAccess } = require('./reading-access');
+const { BackupService, BackupError } = require('./backup');
 
 const PORT = +process.env.PORT || 8080;
 const LIB = path.resolve(process.env.LIBRARY || 'library');
@@ -65,6 +67,9 @@ function save() { saveTimer = saveTimer || setTimeout(flush, 500); }
 function flush() { clearTimeout(saveTimer); saveTimer = null; writeAtomic(DB_FILE, JSON.stringify(db)); }
 for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => { flush(); process.exit(0); });
 db.readingAccess ||= {};
+// The reader owns backups: it alone mounts all of DATA and holds db.json in memory.
+const backup = new BackupService({ dataRoot: DATA, backupRoot: process.env.BACKUP || '', snapshotDb: () => JSON.stringify(db),
+  scraper: { state: () => scraperRequest('/api/state'), control: action => scraperRequest('/api/control', { action }) } }).start();
 const readingSchedule = parseSchedule(process.env.READING_WINDOWS_JSON || '', process.env.READING_TIMEZONE || process.env.TZ || 'Europe/Berlin');
 const readingAccess = createReadingAccess({ schedule: readingSchedule,
   bank: loadQuestionBank(process.env.READING_QUIZ_CATALOG_PATH || ''),
@@ -377,7 +382,19 @@ const routes = {
     admin(u);
     const sentinel = await fetch(SENTINEL_URL + '/api/state', { headers: SCRAPER_TOKEN ? { authorization: `Bearer ${SCRAPER_TOKEN}` } : {}, signal: AbortSignal.timeout(4000) })
       .then(async r => { if (!r.ok) throw new Error(); return r.json(); }).catch(() => ({ status: 'unreachable' }));
-    return { reader: { ok: !scanState.error, uptime: Math.round(process.uptime()), scan: scanState }, sentinel };
+    return { reader: { ok: !scanState.error, uptime: Math.round(process.uptime()), scan: scanState }, sentinel, backup: backup.publicState() };
+  },
+  'GET /api/admin/backup': ({ u }) => { admin(u); return backup.publicState(); },
+  'POST /api/admin/backup': ({ u, b }) => {
+    admin(u);
+    try {
+      if (b.command === 'settings') return backup.updateSettings(b.settings);
+      if (b.command !== 'run') bad(400, 'Unknown backup command');
+      if (!backup.backupRoot) bad(409, 'Backups are off: set BACKUP to a mounted folder');
+      if (backup.running) bad(409, 'A backup is already running');
+      backup.run('manual').catch(() => {}); // the result lands in last/history
+      return backup.publicState();
+    } catch (e) { if (e instanceof BackupError) bad(e.code, e.message); throw e; }
   },
   'GET /api/auth': () => ({ firstRun: !Object.keys(db.users).length, signup: OPEN_SIGNUP }),
 

test/backup.test.js

@@ -0,0 +1,276 @@
+'use strict';
+const test = require('node:test'), assert = require('node:assert/strict');
+const fs = require('fs'), os = require('os'), path = require('path'), net = require('net');
+const { spawn, spawnSync } = require('child_process');
+const { BackupService, readArchive, validateSettings, nextOccurrence } = require('../backup');
+const root = path.resolve(__dirname, '..');
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+const quiet = { log() {}, warn() {}, error() {} };
+const LONG = 'collection-covers/' + 'a-very-long-folder-name-that-keeps-going-'.repeat(3) + '/cover-with-a-long-name.png';
+
+function fixture() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-backup-'));
+  const data = path.join(dir, 'data'), target = path.join(dir, 'backup');
+  const write = (rel, text) => { fs.mkdirSync(path.dirname(path.join(data, rel)), { recursive: true }); fs.writeFileSync(path.join(data, rel), text); };
+  write('db.json', '{"stale":true}');
+  write('scan-cache.json', '{"cache":1}');
+  write('library-index.json', '{"version":1}');
+  write('catalog.json', '{"collections":[]}');
+  write('scraper/queue.json', JSON.stringify({ version: 3, paused: false, status: 'idle', queue: [{ id: 'a' }] }));
+  write('scraper/events.log', '{"level":"info"}\n');
+  write('sentinel/state.json', '{}');
+  write(LONG, 'png-bytes');
+  // Must not be archived.
+  write('runtime-reader.json', '{}'); write('scraper/runtime.json', '{}'); write('sentinel/runtime.json', '{}');
+  write('scraper/daemon.lock', '1'); write('scraper/supervisor.lock', '1'); write('reader.pid', '1'); write('db.json.tmp', '{}');
+  fs.mkdirSync(target);
+  return { dir, data, target, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) };
+}
+
+function fakeScraper({ paused = false, busyPolls = 0, failResume = false } = {}) {
+  const calls = [];
+  let polls = 0;
+  return {
+    calls,
+    get paused() { return paused; },
+    async state() { polls++; return { paused, status: paused && polls <= busyPolls ? 'pausing' : paused ? 'paused' : 'downloading', active: polls <= busyPolls ? [{ source: 'x' }] : [] }; },
+    async control(action) {
+      calls.push(action);
+      if (action === 'resume' && failResume) throw new Error('scraper gone');
+      paused = action === 'pause';
+    },
+  };
+}
+
+const service = (f, options = {}) => new BackupService({ dataRoot: f.data, backupRoot: f.target, log: quiet, pollMs: 5, resumeRetryMs: 1,
+  snapshotDb: () => JSON.stringify({ users: { admin: {} }, live: true }), ...options }).load();
+
+test('settings validation and schedule computation', () => {
+  assert.deepEqual(validateSettings({ enabled: true, time: '04:15', retention: { mode: 'days', value: 30 } }),
+    { enabled: true, time: '04:15', retention: { mode: 'days', value: 30 } });
+  assert.throws(() => validateSettings({ time: '24:00' }), /HH:MM/);
+  assert.throws(() => validateSettings({ retention: { mode: 'count', value: 0 } }), /Keep/);
+  assert.throws(() => validateSettings({ retention: { mode: 'weeks', value: 2 } }), /count/);
+  assert.throws(() => validateSettings({ enabled: 'yes' }), /enabled/);
+  const now = new Date(2026, 9, 22, 3, 0).getTime();
+  assert.equal(nextOccurrence('04:00', now), new Date(2026, 9, 22, 4, 0).getTime());
+  assert.equal(nextOccurrence('03:00', now), new Date(2026, 9, 23, 3, 0).getTime());
+});
+
+test('a backup pauses a running queue, waits for idle, archives a consistent DATA snapshot and resumes', async () => {
+  const f = fixture();
+  try {
+    const scraper = fakeScraper({ busyPolls: 3 });
+    const backup = service(f, { scraper });
+    const result = await backup.run('manual');
+    assert.equal(result.error, undefined, result.error);
+    assert.deepEqual(scraper.calls, ['pause', 'resume']);
+    assert.equal(result.queueWasPaused, false); assert.equal(result.queueResumed, true);
+    assert.match(result.file, /^circus-\d{8}-\d{6}\.tar\.gz$/);
+    assert.deepEqual(fs.readdirSync(f.target), [result.file]);
+    const entries = await readArchive(path.join(f.target, result.file), { contents: true });
+    const byName = new Map(entries.map(e => [e.name, e]));
+    const text = name => byName.get(name).data.toString('utf8');
+    // db.json comes from memory, not from the stale file on disk.
+    assert.deepEqual(JSON.parse(text('db.json')), { users: { admin: {} }, live: true });
+    // The archived queue is paused; the live one is untouched by the snapshot.
+    const queue = JSON.parse(text('scraper/queue.json'));
+    assert.equal(queue.paused, true); assert.deepEqual(queue.queue, [{ id: 'a' }]);
+    assert.equal(JSON.parse(fs.readFileSync(path.join(f.data, 'scraper/queue.json'), 'utf8')).paused, false);
+    for (const name of ['scan-cache.json', 'library-index.json', 'catalog.json', 'scraper/events.log', 'sentinel/state.json', LONG, '.circus-backup.json'])
+      assert.ok(byName.has(name), `${name} missing`);
+    assert.equal(text(LONG), 'png-bytes');
+    for (const name of ['runtime-reader.json', 'scraper/runtime.json', 'sentinel/runtime.json', 'scraper/daemon.lock', 'scraper/supervisor.lock', 'reader.pid', 'db.json.tmp'])
+      assert.ok(!byName.has(name), `${name} must be excluded`);
+    assert.ok(byName.get('scraper/').type === '5');
+    // The persisted state records the result but never a pending marker inside the archive.
+    assert.equal(JSON.parse(text('backup.json')).pending, undefined);
+    assert.equal(backup.publicState().last.file, result.file);
+    assert.equal(backup.publicState().archives.count, 1);
+  } finally { f.cleanup(); }
+});
+
+test('an already paused queue stays paused and is not resumed', async () => {
+  const f = fixture();
+  try {
+    const scraper = fakeScraper({ paused: true });
+    const result = await service(f, { scraper }).run('schedule');
+    assert.equal(result.error, undefined);
+    assert.deepEqual(scraper.calls, []);
+    assert.equal(result.queueWasPaused, true); assert.equal(scraper.paused, true);
+  } finally { f.cleanup(); }
+});
+
+test('a scraper that never becomes idle aborts the backup, leaves no archive and resumes the queue', async () => {
+  const f = fixture();
+  try {
+    const scraper = fakeScraper({ busyPolls: Infinity });
+    const result = await service(f, { scraper, idleTimeoutMs: 40 }).run('manual');
+    assert.match(result.error, /did not become idle/);
+    assert.deepEqual(scraper.calls, ['pause', 'resume']);
+    assert.deepEqual(fs.readdirSync(f.target), []);
+  } finally { f.cleanup(); }
+});
+
+test('an unreachable scraper does not block the backup and is reported', async () => {
+  const f = fixture();
+  try {
+    const scraper = { async state() { throw new Error('connect ECONNREFUSED'); }, async control() { throw new Error('unused'); } };
+    const result = await service(f, { scraper }).run('manual');
+    assert.equal(result.error, undefined);
+    assert.match(result.warnings[0], /unreachable/);
+    const queue = (await readArchive(path.join(f.target, result.file), { contents: true })).find(e => e.name === 'scraper/queue.json');
+    assert.equal(JSON.parse(queue.data).paused, true);
+  } finally { f.cleanup(); }
+});
+
+test('a failed resume is surfaced as a warning', async () => {
+  const f = fixture();
+  try {
+    const result = await service(f, { scraper: fakeScraper({ failResume: true }) }).run('manual');
+    assert.equal(result.error, undefined);
+    assert.match(result.warnings.join(' '), /could not be resumed/);
+  } finally { f.cleanup(); }
+});
+
+test('retention keeps the newest archives and never touches other files', async () => {
+  const f = fixture();
+  try {
+    const old = ['circus-20260101-030000.tar.gz', 'circus-20260102-030000.tar.gz', 'circus-20260103-030000.tar.gz'];
+    const foreign = ['circus-20250101-0300.tar.zst', 'notes.txt', 'circus-20250101-030000.tar.gz.bak'];
+    for (const name of [...old, ...foreign]) fs.writeFileSync(path.join(f.target, name), 'x');
+    fs.mkdirSync(path.join(f.target, 'circus-20250101-030000.tar.gz-dir'));
+    fs.writeFileSync(path.join(f.target, 'circus-20250101-030000.tar.gz.part'), 'stale');
+    const backup = service(f);
+    backup.updateSettings({ retention: { mode: 'count', value: 2 } });
+    const result = await backup.run('manual');
+    assert.equal(result.error, undefined); assert.equal(result.deleted, 2);
+    assert.deepEqual(fs.readdirSync(f.target).sort(), [...foreign, 'circus-20250101-030000.tar.gz-dir', old[2], result.file].sort());
+    // Days mode works from the timestamp in the name.
+    backup.updateSettings({ retention: { mode: 'days', value: 1 } });
+    const second = await backup.run('manual');
+    assert.ok(!fs.existsSync(path.join(f.target, old[2])));
+    assert.ok(fs.existsSync(path.join(f.target, second.file)));
+    for (const name of foreign) assert.ok(fs.existsSync(path.join(f.target, name)));
+  } finally { f.cleanup(); }
+});
+
+test('backups are off without BACKUP, and a missing folder is reported, never created', async () => {
+  const f = fixture();
+  try {
+    const off = service(f, { backupRoot: '' });
+    await assert.rejects(off.run('manual'), /Backups are off/);
+    assert.equal(off.publicState().configured, false);
+    const missing = path.join(f.dir, 'not-mounted');
+    const result = await service(f, { backupRoot: missing }).run('manual');
+    assert.match(result.error, /not usable/);
+    assert.ok(!fs.existsSync(missing));
+  } finally { f.cleanup(); }
+});
+
+test('only one run at a time', async () => {
+  const f = fixture();
+  try {
+    const backup = service(f, { scraper: fakeScraper({ busyPolls: 5 }) });
+    const first = backup.run('manual');
+    await assert.rejects(backup.run('manual'), /already running/);
+    assert.equal((await first).error, undefined);
+  } finally { f.cleanup(); }
+});
+
+test('the schedule runs once per slot, catches up a recent miss and skips stale slots', async () => {
+  const f = fixture();
+  try {
+    let now = new Date(2026, 9, 22, 2, 0).getTime();
+    const backup = service(f, { now: () => now });
+    backup.updateSettings({ enabled: true, time: '03:30' });
+    assert.equal(backup.state.nextAt, new Date(2026, 9, 22, 3, 30).getTime());
+    assert.equal(await backup.tick(), null);
+    now = new Date(2026, 9, 22, 9, 0).getTime(); // reader was down at 03:30
+    const result = await backup.tick();
+    assert.equal(result.trigger, 'schedule'); assert.equal(result.error, undefined);
+    assert.equal(backup.state.nextAt, new Date(2026, 9, 23, 3, 30).getTime());
+    assert.equal(await backup.tick(), null);
+    now = new Date(2026, 9, 30, 9, 0).getTime(); // e.g. an old backup.json was restored
+    assert.equal(await backup.tick(), null);
+    assert.equal(backup.state.nextAt, new Date(2026, 9, 31, 3, 30).getTime());
+    assert.equal(fs.readdirSync(f.target).length, 1);
+  } finally { f.cleanup(); }
+});
+
+test('a run interrupted by a restart resumes the queue it paused', async () => {
+  const f = fixture();
+  try {
+    fs.writeFileSync(path.join(f.data, 'backup.json'), JSON.stringify({ version: 1, pending: { startedAt: 1, trigger: 'schedule', pausedByUs: true } }));
+    const scraper = fakeScraper({ paused: true });
+    const backup = service(f, { scraper });
+    await backup.recoverInterrupted();
+    assert.deepEqual(scraper.calls, ['resume']);
+    assert.match(backup.state.last.error, /Interrupted/);
+    assert.equal(JSON.parse(fs.readFileSync(path.join(f.data, 'backup.json'), 'utf8')).pending, null);
+  } finally { f.cleanup(); }
+});
+
+test('system tar extracts the archive', { skip: spawnSync('tar', ['--version']).error ? 'no tar binary' : false }, async () => {
+  const f = fixture();
+  try {
+    const result = await service(f).run('manual');
+    const out = path.join(f.dir, 'restored'); fs.mkdirSync(out);
+    const tar = spawnSync('tar', ['-xzf', path.join(f.target, result.file), '-C', out], { encoding: 'utf8' });
+    assert.equal(tar.status, 0, tar.stderr);
+    assert.equal(fs.readFileSync(path.join(out, LONG), 'utf8'), 'png-bytes');
+    assert.equal(JSON.parse(fs.readFileSync(path.join(out, 'scraper/queue.json'), 'utf8')).paused, true);
+    assert.ok(!fs.existsSync(path.join(out, 'scraper/daemon.lock')));
+  } finally { f.cleanup(); }
+});
+
+async function port() {
+  const server = net.createServer();
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  const result = server.address().port; await new Promise(resolve => server.close(resolve)); return result;
+}
+
+test('reader and scraper: admin backup API pauses and resumes the real queue', { timeout: 30000 }, async () => {
+  const f = fixture(), children = [], output = [];
+  try {
+    const library = path.join(f.dir, 'library'); fs.mkdirSync(library);
+    fs.writeFileSync(path.join(f.data, 'scraper', 'queue.json'), JSON.stringify({ paused: false, queue: [], settings: { autoUpdateEnabled: false } }));
+    for (const name of ['runtime.json', 'daemon.lock', 'supervisor.lock']) fs.rmSync(path.join(f.data, 'scraper', name), { force: true });
+    for (const name of ['db.json', 'catalog.json', 'library-index.json', 'scan-cache.json']) fs.rmSync(path.join(f.data, name));
+    const [rp, sp] = await Promise.all([port(), port()]);
+    const base = `http://127.0.0.1:${rp}`, token = 'backup-test-only';
+    const env = { ...process.env, DATA: f.data, LIBRARY: library, SCRAPER_DATA: path.join(f.data, 'scraper'), BACKUP: f.target,
+      PORT: String(rp), SCRAPER_PORT: String(sp), SCRAPER_HOST: '127.0.0.1', SCRAPER_URL: `http://127.0.0.1:${sp}`, SENTINEL_URL: 'http://127.0.0.1:9',
+      SCRAPER_TOKEN: token, SCRAPER_TOKEN_FILE: '', SCAN_INTERVAL: '0' };
+    for (const script of ['server.js', 'scraper/daemon.js']) {
+      const child = spawn(process.execPath, [script], { cwd: root, env, windowsHide: true }); children.push(child);
+      for (const stream of [child.stdout, child.stderr]) stream.on('data', chunk => output.push(chunk.toString()));
+    }
+    for (const url of [base, `http://127.0.0.1:${sp}`]) {
+      let ready = false;
+      for (let n = 0; n < 80 && !ready; n++) { ready = await fetch(url + '/health').then(r => r.ok).catch(() => false); if (!ready) await delay(100); }
+      assert.ok(ready, output.join(''));
+    }
+    const signup = await fetch(base + '/api/signup', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'admin', password: 'local-test-password' }) });
+    const cookie = signup.headers.get('set-cookie').split(';')[0];
+    const request = (endpoint, body) => fetch(base + endpoint, { headers: { cookie, 'content-type': 'application/json' }, ...(body ? { method: 'POST', body: JSON.stringify(body) } : {}) });
+    assert.equal((await request('/api/admin/backup', { command: 'settings', settings: { time: '25:00' } })).status, 400);
+    const saved = await (await request('/api/admin/backup', { command: 'settings', settings: { enabled: true, time: '04:00', retention: { mode: 'count', value: 3 } } })).json();
+    assert.equal(saved.settings.retention.value, 3); assert.ok(saved.nextAt > Date.now());
+    assert.equal((await request('/api/admin/backup', { command: 'run' })).status, 200);
+    let state;
+    for (let n = 0; n < 100; n++) { state = await (await request('/api/admin/backup')).json(); if (state.last && !state.running) break; await delay(100); }
+    assert.equal(state.last.error, undefined, state.last.error);
+    assert.equal(state.last.queueWasPaused, false); assert.equal(state.last.queueResumed, true);
+    const scraperState = await (await request('/api/admin/scraper')).json();
+    assert.equal(scraperState.paused, false);
+    const entries = await readArchive(path.join(f.target, state.last.file), { contents: true });
+    const db = JSON.parse(entries.find(e => e.name === 'db.json').data);
+    assert.ok(db.users.admin);
+    assert.equal(JSON.parse(entries.find(e => e.name === 'scraper/queue.json').data).paused, true);
+    assert.ok((await (await request('/api/admin/health')).json()).backup.last.file);
+  } finally {
+    await Promise.all(children.map(child => new Promise(resolve => { if (child.exitCode !== null) return resolve(); child.once('exit', resolve); child.kill(); })));
+    f.cleanup();
+  }
+});

tools/prepare-compose.js

@@ -2,7 +2,7 @@
 'use strict';
 const fs = require('fs'), path = require('path'), crypto = require('crypto');
 const root = path.resolve(__dirname, '..');
-for (const relative of ['data', 'data/scraper', 'data/sentinel', 'library', '.secrets']) fs.mkdirSync(path.join(root, relative), { recursive: true });
+for (const relative of ['data', 'data/scraper', 'data/sentinel', 'library', 'backup', '.secrets']) fs.mkdirSync(path.join(root, relative), { recursive: true });
 const file = path.join(root, '.secrets', 'scraper_token');
 if (!fs.existsSync(file)) fs.writeFileSync(file, crypto.randomBytes(32).toString('hex') + '\n', { flag: 'wx', mode: 0o600 });
 if (!fs.existsSync(path.join(root, '.env'))) fs.copyFileSync(path.join(root, '.env.example'), path.join(root, '.env'));

tools/test-compose.js

@@ -6,10 +6,10 @@ const assert = require('assert/strict'), { spawn } = require('child_process');
 const { Canvas } = require('./png');
 const root = path.resolve(__dirname, '..'), dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-compose-'));
 const project = 'mc-qa-' + crypto.randomBytes(3).toString('hex'), port = process.env.QA_PORT || '8180';
-const library = path.join(dir, 'library'), data = path.join(dir, 'data'), tokenFile = path.join(dir, 'token');
+const library = path.join(dir, 'library'), data = path.join(dir, 'data'), backupDir = path.join(dir, 'backup'), tokenFile = path.join(dir, 'token');
 const directionQa = process.argv.includes('--direction-ui');
 const peerGatewayQa = process.argv.includes('--peer-gateway');
-fs.mkdirSync(path.join(data, 'scraper'), { recursive: true }); fs.mkdirSync(path.join(data, 'sentinel'));
+fs.mkdirSync(path.join(data, 'scraper'), { recursive: true }); fs.mkdirSync(path.join(data, 'sentinel')); fs.mkdirSync(backupDir);
 if (peerGatewayQa) {
   const peerData = path.join(data, 'peering');
   const peerUsage = path.join(data, 'peering-usage');
@@ -48,7 +48,7 @@ for (const [index, name] of ['Moon Garden', 'Before the Rain', 'Iron Harbor'].en
     status: index ? 'completed' : 'ongoing', tags: ['Fantasy', 'Demo'] }));
 }
 const env = { ...process.env, LIBRARY_PATH: library, DATA_PATH: data, SCRAPER_DATA_PATH: path.join(data, 'scraper'), SENTINEL_DATA_PATH: path.join(data, 'sentinel'),
-  SCRAPER_TOKEN_PATH: tokenFile, READER_PORT: port, READER_BIND: '127.0.0.1', COOKIE_SECURE: '0', APP_IMAGE: process.env.APP_IMAGE || 'mangacircus:local',
+  SCRAPER_TOKEN_PATH: tokenFile, BACKUP_PATH: backupDir, READER_PORT: port, READER_BIND: '127.0.0.1', COOKIE_SECURE: '0', APP_IMAGE: process.env.APP_IMAGE || 'mangacircus:local',
   READER_IMAGE: process.env.APP_IMAGE || 'mangacircus:local',
   ...(readingGateQa ? { READING_WINDOWS_JSON: '{"0":[],"1":[],"2":[],"3":[],"4":[],"5":[],"6":[]}',
     READING_TIMEZONE: 'Europe/Berlin', READING_QUIZ_UNLOCK_MINUTES: '60', READING_QUIZ_CATALOG_PATH: '/data/reading-quiz.json' }
@@ -93,6 +93,14 @@ let keep = false;
     let inventory;
     await until(async () => { inventory = await (await api('/api/library')).json(); return inventory.mangas?.length === 3; }, 'Fixture inventory missing');
     const byTitle = title => inventory.mangas.find(m => m.title === title).id;
+    // Built-in backup through the real control API: archive written, paused queue left paused.
+    const queueBefore = JSON.parse(fs.readFileSync(path.join(data, 'scraper', 'queue.json'), 'utf8'));
+    assert.equal((await api('/api/admin/backup', { command: 'run' })).status, 200);
+    let backupState;
+    await until(async () => { backupState = await (await api('/api/admin/backup')).json(); return backupState.last && !backupState.running; }, 'Backup did not finish');
+    assert.equal(backupState.last.error, undefined, backupState.last.error);
+    assert.ok(fs.existsSync(path.join(backupDir, backupState.last.file)), 'Backup archive missing on the host');
+    assert.equal(JSON.parse(fs.readFileSync(path.join(data, 'scraper', 'queue.json'), 'utf8')).paused, queueBefore.paused);
     if (readingGateQa) {
       const manga = byTitle('Moon Garden'), other = byTitle('Before the Rain');
       const chapter = (await (await api('/api/manga?m=' + encodeURIComponent(manga))).json()).chapters[0].id;
@@ -243,7 +251,7 @@ let keep = false;
     keep = process.argv.includes('--keep');
     if (keep) {
       const envFile = path.join(dir, 'qa.env');
-      fs.writeFileSync(envFile, ['LIBRARY_PATH', 'DATA_PATH', 'SCRAPER_DATA_PATH', 'SENTINEL_DATA_PATH', 'SCRAPER_TOKEN_PATH', 'READER_PORT', 'READER_BIND', 'COOKIE_SECURE', 'APP_IMAGE',
+      fs.writeFileSync(envFile, ['LIBRARY_PATH', 'DATA_PATH', 'SCRAPER_DATA_PATH', 'SENTINEL_DATA_PATH', 'SCRAPER_TOKEN_PATH', 'BACKUP_PATH', 'READER_PORT', 'READER_BIND', 'COOKIE_SECURE', 'APP_IMAGE',
         ...(readingGateQa ? ['READING_WINDOWS_JSON', 'READING_TIMEZONE', 'READING_QUIZ_UNLOCK_MINUTES', 'READING_QUIZ_CATALOG_PATH'] : [])]
         .map(k => `${k}=${k === 'READING_WINDOWS_JSON' ? `'${env[k]}'` : env[k].replaceAll('\\', '/')}`).join('\n') + '\n');
       console.log(JSON.stringify({ project, url, envFile, fixture: dir }));