diff --git a/.dockerignore b/.dockerignore index 99021b0..c84f518 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ !reading-direction.js !chapter-identity.js !catalog.js +!backup.js !reading-analytics.js !reading-access.js !public/ diff --git a/.env.example b/.env.example index 23c02e4..5556f69 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,9 @@ DATA_PATH=./data SCRAPER_DATA_PATH=./data/scraper SENTINEL_DATA_PATH=./data/sentinel SCRAPER_TOKEN_PATH=./.secrets/scraper_token +# Built-in backups of DATA_PATH (docs/backup.md): an existing folder, ideally on another +# disk/NAS, writable by uid 1000. Unset = backups off (./backup is then mounted unused). +# BACKUP_PATH=./backup TZ=Europe/Berlin COOKIE_SECURE=0 # Optional server-enforced reading windows. Day keys: 0=Sunday through 6=Saturday. diff --git a/.gitignore b/.gitignore index ac61e02..51cc483 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ /data/ # the manga library: demo content is generated by `node tools/make-dummy.js`, a real one is yours alone /library/ +# built-in backups (BACKUP_PATH default) +/backup/ # --- Node --- node_modules/ diff --git a/Dockerfile b/Dockerfile index 69cc4b9..786aa00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apk add --no-cache tzdata openssl WORKDIR /app ARG VCS_REF=development LABEL org.opencontainers.image.title="MangaCircus" org.opencontainers.image.revision=$VCS_REF -COPY package.json server.js library.js reading-direction.js chapter-identity.js reading-analytics.js reading-access.js catalog.js ./ +COPY package.json server.js library.js reading-direction.js chapter-identity.js reading-analytics.js reading-access.js catalog.js backup.js ./ COPY public ./public COPY greatest ./greatest COPY discovery ./discovery diff --git a/README.md b/README.md index d305bf6..1678569 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ see administration; personal settings do not poll scraper services. See | `SCRAPER_TOKEN` | empty | Shared daemon token; required when it listens beyond loopback | | `SCRAPER_TOKEN_FILE` | empty | Alternative secret file (used in Compose) | | `SENTINEL_URL` | `http://127.0.0.1:8092` | Private observation service URL | +| `BACKUP` | – | Existing folder for built-in backups of `DATA`; unset = off. Compose: set `BACKUP_PATH`. Schedule and retention under Settings → Service health ([backup](docs/backup.md)) | Serve it behind a reverse proxy with HTTPS if it is reachable from the internet. All pages use relative URLs, so a sub-path like `/manga/` works. @@ -113,6 +114,7 @@ whole-series completeness. See the scraper service and library format contracts | [docs/architecture.md](docs/architecture.md) | Components, data model, API, security, the reader's layout engine, deployment, design decisions | | [docs/local-deployment.md](docs/local-deployment.md) | Current local Compose deployment, Tailscale access, persistent state, checks, restart, and recovery | | [docs/settings-navigation.md](docs/settings-navigation.md) | User/Admin settings sections, scraper tabs, routes and UI checks | +| [docs/backup.md](docs/backup.md) | Built-in scheduled backup of `data/`, what it contains, retention and the restore procedure | | [docs/reading-insights-concept.md](docs/reading-insights-concept.md) | Active-time presentation, private summaries, deletion/export controls, and the future journal design | | [docs/scraper-guide.md](docs/scraper-guide.md) | Rule book (agent-readable) for scrapers and other tools that write into the library | | [scraper/AGENT_RUNBOOK.md](scraper/AGENT_RUNBOOK.md) | Conservative scraper workflow and host-specific procedures for automated agents | diff --git a/backup.js b/backup.js new file mode 100644 index 0000000..58ed5a1 --- /dev/null +++ b/backup.js @@ -0,0 +1,398 @@ +'use strict'; +/* + * Built-in backup of the reader's DATA tree into BACKUP (see docs/backup.md). + * + * One run: pause the scraper queue through its control API, wait (bounded) until no + * job is active, write a gzip'd tar of DATA into BACKUP, resume the queue only if this + * run paused it, then apply retention. No container stops. db.json comes from the + * reader's in-memory state; scraper/queue.json is stored with `paused: true` so a + * restored copy never starts downloading by itself. Runtime/lock/pid/temp files are + * left out. Retention only ever deletes `circus-YYYYMMDD-HHMMSS.tar.gz` files directly + * inside BACKUP. No dependencies: the tar writer/reader below is plain ustar + PAX. + */ +const fs = require('fs'), fsp = fs.promises, path = require('path'), zlib = require('zlib'); +const { Readable } = require('stream'), { pipeline } = require('stream/promises'); + +const ARCHIVE = /^circus-(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})\.tar\.gz$/; +const PART = /^circus-\d{8}-\d{6}\.tar\.gz\.part$/; +const EXCLUDE = [/^runtime.*\.json$/, /\.lock$/, /\.pid$/, /\.tmp$/]; +const DEFAULT_SETTINGS = { enabled: false, time: '03:30', retention: { mode: 'count', value: 14 } }; +const RETENTION_LIMITS = { count: [1, 365], days: [1, 3650] }; +const CATCH_UP_MS = 24 * 3600e3; + +class BackupError extends Error { constructor(code, msg) { super(msg); this.code = code; } } +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); +const pad = n => String(n).padStart(2, '0'); +const stampOf = d => `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`; +const archiveTime = name => { const m = ARCHIVE.exec(name); return m ? new Date(+m[1], m[2] - 1, +m[3], +m[4], +m[5], +m[6]).getTime() : NaN; }; +const message = e => String((e && e.message) || e).slice(0, 500); + +function validateSettings(input, current = DEFAULT_SETTINGS) { + const next = { ...current, retention: { ...current.retention } }; + if (input.enabled !== undefined) { + if (typeof input.enabled !== 'boolean') throw new BackupError(400, 'enabled must be true or false'); + next.enabled = input.enabled; + } + if (input.time !== undefined) { + if (!/^([01]\d|2[0-3]):[0-5]\d$/.test(String(input.time))) throw new BackupError(400, 'The backup time must be HH:MM'); + next.time = String(input.time); + } + if (input.retention !== undefined) { + const mode = input.retention && input.retention.mode, value = Number(input.retention && input.retention.value); + const limits = RETENTION_LIMITS[mode]; + if (!limits) throw new BackupError(400, 'Retention is "count" or "days"'); + if (!Number.isInteger(value) || value < limits[0] || value > limits[1]) throw new BackupError(400, `Keep ${limits[0]}–${limits[1]} ${mode === 'count' ? 'backups' : 'days'}`); + next.retention = { mode, value }; + } + return next; +} + +// Next local occurrence of HH:MM strictly after `now` (process TZ). +function nextOccurrence(time, now) { + const [hh, mm] = time.split(':').map(Number), d = new Date(now); + const at = new Date(d.getFullYear(), d.getMonth(), d.getDate(), hh, mm, 0, 0); + if (at.getTime() <= now) at.setDate(at.getDate() + 1); + return at.getTime(); +} + +// ---------- tar (ustar + PAX path records), enough for `tar -xzf` everywhere ---------- +function octal(buf, offset, length, value) { buf.write(value.toString(8).padStart(length - 1, '0') + '\0', offset, length, 'ascii'); } +function rawHeader(name, { size = 0, mode = 0o644, uid = 0, gid = 0, mtime = 0, type = '0' }) { + const b = Buffer.alloc(512); + b.write(name, 0, 100, 'utf8'); + octal(b, 100, 8, mode & 0o7777); octal(b, 108, 8, uid & 0o7777777); octal(b, 116, 8, gid & 0o7777777); + octal(b, 124, 12, size); octal(b, 136, 12, Math.max(0, Math.floor(mtime / 1000))); + b.write(' ', 148, 8, 'ascii'); + b.write(type, 156, 1, 'ascii'); + b.write('ustar\0' + '00', 257, 8, 'ascii'); + let sum = 0; for (const byte of b) sum += byte; + b.write(sum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii'); + return b; +} +const padding = size => Buffer.alloc((512 - (size % 512)) % 512); +function paxRecord(key, value) { + const body = ` ${key}=${value}\n`, bytes = Buffer.byteLength(body); + let length = bytes + String(bytes).length; + if (String(length).length !== String(bytes).length) length = bytes + String(length).length; + return `${length}${body}`; +} +function tarEntry(name, data, meta) { + const parts = []; + // Names longer than the ustar field or non-ASCII go into a PAX header first. + if (Buffer.byteLength(name) > 100 || /[^\x20-\x7e]/.test(name)) { + const pax = Buffer.from(paxRecord('path', name), 'utf8'); + parts.push(rawHeader('PaxHeader/' + name.replace(/[^\x20-\x7e]/g, '_').slice(-80), { size: pax.length, type: 'x', mtime: meta.mtime }), pax, padding(pax.length)); + name = name.replace(/[^\x20-\x7e]/g, '_').slice(-100); + } + parts.push(rawHeader(name, { ...meta, size: data ? data.length : 0 })); + if (data && data.length) parts.push(data, padding(data.length)); + return parts; +} + +// Streaming reader for verification and tests: [{ name, type, size, mode, mtime, data? }]. +async function readArchive(file, { contents = false } = {}) { + const entries = []; let pending = Buffer.alloc(0), current = null, paxPath = null, ended = false; + const consume = chunk => { + pending = pending.length ? Buffer.concat([pending, chunk]) : chunk; + for (;;) { + if (ended) return; + if (current) { + if (pending.length < current.need) return; + const body = pending.subarray(0, current.size); + if (current.entry.type === 'x') paxPath = /(?:^|\n)\d+ path=([^\n]*)\n/.exec(body.toString('utf8'))?.[1] ?? null; + else { if (contents) current.entry.data = Buffer.from(body); entries.push(current.entry); } + pending = pending.subarray(current.need); current = null; continue; + } + if (pending.length < 512) return; + const h = pending.subarray(0, 512); pending = pending.subarray(512); + if (h.every(byte => byte === 0)) { ended = true; return; } + let sum = 0; for (let i = 0; i < 512; i++) sum += i >= 148 && i < 156 ? 32 : h[i]; + if (sum !== parseInt(h.toString('ascii', 148, 156), 8)) throw new Error('Corrupt tar header'); + const field = (o, l) => h.toString('utf8', o, o + l).replace(/\0.*$/s, ''); + const size = parseInt(field(124, 12), 8) || 0, type = field(156, 1) || '0'; + const prefix = field(345, 155), name = prefix ? prefix + '/' + field(0, 100) : field(0, 100); + const entry = { name: type === 'x' ? name : (paxPath ?? name), type, size, mode: parseInt(field(100, 8), 8), mtime: parseInt(field(136, 12), 8) * 1000 }; + if (type !== 'x') paxPath = null; + current = { entry, size, need: size + padding(size).length }; + } + }; + await pipeline(fs.createReadStream(file), zlib.createGunzip(), async function* (source) { for await (const chunk of source) consume(chunk); }); + if (!ended) throw new Error('Archive ends without the tar end marker'); + return entries; +} + +class BackupService { + /** + * dataRoot DATA (read) backupRoot BACKUP (rw) or '' = feature off + * stateFile settings + history snapshotDb () => db.json text from memory + * scraper { state(), control(action) } or null + */ + constructor({ dataRoot, backupRoot, stateFile, snapshotDb, scraper = null, now = Date.now, log = console, + idleTimeoutMs = 15 * 60e3, pollMs = 2000, resumeRetryMs = 5000 }) { + this.dataRoot = path.resolve(dataRoot); + this.backupRoot = backupRoot ? path.resolve(backupRoot) : ''; + this.stateFile = stateFile || path.join(this.dataRoot, 'backup.json'); + this.snapshotDb = snapshotDb; this.scraper = scraper; this.now = now; this.log = log; + this.idleTimeoutMs = idleTimeoutMs; this.pollMs = pollMs; this.resumeRetryMs = resumeRetryMs; + this.state = { version: 1, settings: { ...DEFAULT_SETTINGS }, nextAt: null, last: null, history: [], pending: null }; + this.running = null; this.archives = []; this.targetError = null; this.timer = null; + } + + load() { + try { Object.assign(this.state, JSON.parse(fs.readFileSync(this.stateFile, 'utf8'))); } + catch (e) { if (e.code !== 'ENOENT') this.log.warn(`[backup] ignoring unreadable ${this.stateFile}: ${e.message}`); } + this.state.settings = validateSettings(this.state.settings || {}, DEFAULT_SETTINGS); + this.state.history = Array.isArray(this.state.history) ? this.state.history.slice(-10) : []; + return this; + } + + persist() { + fs.writeFileSync(this.stateFile + '.tmp', JSON.stringify(this.state, null, 2) + '\n'); + fs.renameSync(this.stateFile + '.tmp', this.stateFile); + } + + // A run interrupted by a restart may have left the queue paused on its behalf. + async recoverInterrupted() { + const pending = this.state.pending; + if (!pending) return; + this.state.pending = null; + const result = { at: pending.startedAt, trigger: pending.trigger, error: 'Interrupted by a reader restart', warnings: [] }; + if (pending.pausedByUs) { + try { await this.resume(); result.warnings.push('Queue resumed after the interrupted backup'); } + catch (e) { result.warnings.push(`Queue left paused: ${message(e)}`); } + } + this.record(result); + } + + async refreshTarget() { + if (!this.backupRoot) { this.targetError = null; this.archives = []; return; } + try { + const st = await fsp.stat(this.backupRoot); + if (!st.isDirectory()) throw new Error('not a directory'); + await fsp.access(this.backupRoot, fs.constants.W_OK); + const names = (await fsp.readdir(this.backupRoot)).filter(name => ARCHIVE.test(name)).sort(); + this.archives = await Promise.all(names.map(async name => ({ name, size: (await fsp.stat(path.join(this.backupRoot, name))).size, at: archiveTime(name) }))); + this.targetError = null; + } catch (e) { this.targetError = `Backup folder ${this.backupRoot} is not usable: ${e.code || message(e)}`; this.archives = []; } + } + + publicState() { + const s = this.state; + return { + configured: !!this.backupRoot, path: this.backupRoot || null, targetError: this.targetError, + settings: s.settings, nextAt: s.settings.enabled && this.backupRoot ? s.nextAt : null, + running: this.running && { ...this.running }, last: s.last, history: s.history.slice().reverse(), + archives: { count: this.archives.length, bytes: this.archives.reduce((n, a) => n + a.size, 0), + newest: this.archives.length ? this.archives[this.archives.length - 1].name : null }, + }; + } + + updateSettings(input) { + this.state.settings = validateSettings(input || {}, this.state.settings); + this.state.nextAt = this.state.settings.enabled ? nextOccurrence(this.state.settings.time, this.now()) : null; + this.persist(); + return this.publicState(); + } + + start(intervalMs = 30000) { + this.load(); + this.refreshTarget().then(() => this.recoverInterrupted()).catch(e => this.log.error('[backup]', message(e))); + this.timer = setInterval(() => this.tick(), intervalMs); + this.timer.unref?.(); + return this; + } + + stop() { clearInterval(this.timer); } + + // Scheduled runs: catch up a missed slot from the last day, never older ones + // (e.g. right after restoring an old backup.json). + async tick() { + const s = this.state.settings, now = this.now(); + if (!s.enabled || !this.backupRoot || this.running) return null; + if (!this.state.nextAt) { this.state.nextAt = nextOccurrence(s.time, now); this.persist(); return null; } + if (now < this.state.nextAt) return null; + const due = now - this.state.nextAt < CATCH_UP_MS; + this.state.nextAt = nextOccurrence(s.time, now); this.persist(); + return due ? this.run('schedule').catch(() => null) : null; + } + + /** Starts a run and resolves with its result; rejects at once when no run is possible. */ + run(trigger = 'manual') { + if (!this.backupRoot) return Promise.reject(new BackupError(409, 'Backups are off: set BACKUP to a mounted folder')); + if (this.running) return Promise.reject(new BackupError(409, 'A backup is already running')); + this.running = { startedAt: this.now(), trigger, phase: 'starting' }; + return this.perform(trigger).finally(() => { this.running = null; }); + } + + async perform(trigger) { + const started = this.now(), result = { at: started, trigger, warnings: [] }; + let part = null, pausedByUs = false; + try { + await this.refreshTarget(); + if (this.targetError) throw new Error(this.targetError); + await this.removeStaleParts(); + // 1. Pause the queue (unless already paused) and wait for the active chapter. + let scraperState = null; + if (this.scraper) { + try { scraperState = await this.scraper.state(); } + catch (e) { result.warnings.push(`Scraper unreachable, queue not paused: ${message(e)}`); } + } + if (scraperState) { + result.queueWasPaused = !!scraperState.paused; + if (!scraperState.paused) { + this.state.pending = { startedAt: started, trigger, pausedByUs: true }; this.persist(); + await this.scraper.control('pause'); pausedByUs = true; + } + this.running.phase = 'waiting for the scraper to finish its current step'; + await this.waitIdle(scraperState); + } + // 2. Snapshot DATA into BACKUP/.part, then rename. + this.running.phase = 'writing archive'; + let stamp = started, name, file; + do { name = `circus-${stampOf(new Date(stamp))}.tar.gz`; file = path.join(this.backupRoot, name); stamp += 1000; } + while (fs.existsSync(file)); // two runs within one second must not overwrite each other + part = file + '.part'; + const stats = await this.writeArchive(part); + this.running.phase = 'verifying archive'; + const entries = await readArchive(part); + const files = entries.filter(e => e.type === '0').length; + if (files !== stats.files) throw new Error(`Verification read ${files} files, expected ${stats.files}`); + await fsp.rename(part, file); part = null; + Object.assign(result, { file: name, size: (await fsp.stat(file)).size, files, dataBytes: stats.bytes }); + } catch (e) { + result.error = message(e); + if (part) await fsp.rm(part, { force: true }).catch(() => {}); + } finally { + // 3. Resume only what this run paused. + if (pausedByUs) { + this.running.phase = 'resuming queue'; + try { await this.resume(); result.queueResumed = true; } + catch (e) { result.warnings.push(`Queue could not be resumed, resume it in Settings → Scraper: ${message(e)}`); } + } + if (this.state.pending) this.state.pending = null; + } + // 4. Retention only after a successful run, so failures never thin out old copies. + if (!result.error) { + this.running.phase = 'applying retention'; + try { result.deleted = await this.applyRetention(result.file); } + catch (e) { result.warnings.push(`Retention failed: ${message(e)}`); } + } + result.durationMs = this.now() - started; + await this.refreshTarget(); + this.record(result); + if (result.error) this.log.error(`[backup] failed: ${result.error}`); + else this.log.log(`[backup] ${result.file} ${result.size} bytes in ${result.durationMs} ms`); + return result; + } + + record(result) { + this.state.last = result; + this.state.history = [...this.state.history, result].slice(-10); + this.persist(); + } + + async waitIdle(initial) { + const deadline = this.now() + this.idleTimeoutMs; + let state = initial, lastError = null; + for (;;) { + if (state && !(state.active || []).length && state.status !== 'pausing') return; + if (this.now() >= deadline) { + throw new Error(`The scraper did not become idle within ${Math.round(this.idleTimeoutMs / 60000)} min${lastError ? ` (${lastError})` : ''}; no backup written`); + } + await delay(this.pollMs); + try { state = await this.scraper.state(); lastError = null; } + catch (e) { state = null; lastError = message(e); } + } + } + + async resume() { + let error; + for (let attempt = 0; attempt < 3; attempt++) { + try { await this.scraper.control('resume'); return; } + catch (e) { error = e; if (attempt < 2) await delay(this.resumeRetryMs); } + } + throw error; + } + + // Files of DATA in a stable order, skipping excluded names, symlinks and a nested BACKUP. + async collect(dir = this.dataRoot, rel = '') { + const out = []; + const entries = (await fsp.readdir(dir, { withFileTypes: true })).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + for (const entry of entries) { + const full = path.join(dir, entry.name), relative = rel ? rel + '/' + entry.name : entry.name; + if (entry.isDirectory()) { + if (full === this.backupRoot) continue; + out.push({ full, relative, dir: true }); + out.push(...await this.collect(full, relative)); + } else if (entry.isFile() && !EXCLUDE.some(re => re.test(entry.name))) out.push({ full, relative, dir: false }); + } + return out; + } + + // Contents of one file as it goes into the archive. + async contentOf(item) { + if (item.relative === 'db.json' && this.snapshotDb) return Buffer.from(this.snapshotDb()); + if (item.relative === path.relative(this.dataRoot, this.stateFile).split(path.sep).join('/')) { + const { pending, ...rest } = this.state; void pending; + return Buffer.from(JSON.stringify(rest, null, 2) + '\n'); + } + const data = await fsp.readFile(item.full); + if (item.relative === 'scraper/queue.json') { + const queue = JSON.parse(data.toString('utf8')); + queue.paused = true; queue.status = 'paused'; + return Buffer.from(JSON.stringify(queue, null, 2) + '\n'); + } + return data; + } + + async writeArchive(part) { + const items = await this.collect(), stats = { files: 0, bytes: 0 }; + // The in-memory db is archived even before its first debounced write. + if (this.snapshotDb && !items.some(item => item.relative === 'db.json')) items.unshift({ full: path.join(this.dataRoot, 'db.json'), relative: 'db.json', dir: false, memory: true }); + const manifest = { kind: 'mangacircus-data-backup', version: 1, created: new Date(this.now()).toISOString(), + excluded: 'runtime*.json, *.lock, *.pid, *.tmp', queuePaused: true }; + const self = this; + async function* tar() { + yield* tarEntry('.circus-backup.json', Buffer.from(JSON.stringify(manifest, null, 2) + '\n'), { mtime: self.now() }); + for (const item of items) { + let st; + try { st = item.memory ? { mode: 0o644, uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0, mtimeMs: self.now() } : await fsp.stat(item.full); } + catch (e) { if (e.code === 'ENOENT') continue; throw e; } + const meta = { mode: st.mode, uid: st.uid, gid: st.gid, mtime: st.mtimeMs }; + if (item.dir) { yield* tarEntry(item.relative + '/', null, { ...meta, type: '5' }); continue; } + let data; + try { data = await self.contentOf(item); } catch (e) { if (e.code === 'ENOENT') continue; throw new Error(`${item.relative}: ${message(e)}`); } + stats.files++; stats.bytes += data.length; + yield* tarEntry(item.relative, data, meta); + } + yield Buffer.alloc(1024); + } + await pipeline(Readable.from(tar()), zlib.createGzip(), fs.createWriteStream(part, { flags: 'wx' })); + const handle = await fsp.open(part, 'r+'); // fsync before the rename makes it visible + try { await handle.sync(); } finally { await handle.close(); } + stats.files++; // the manifest + return stats; + } + + async removeStaleParts() { + for (const name of await fsp.readdir(this.backupRoot)) if (PART.test(name)) await fsp.rm(path.join(this.backupRoot, name), { force: true }); + } + + // Deletes only our own archive names directly inside BACKUP; never the newest one. + async applyRetention(keepName) { + const { mode, value } = this.state.settings.retention; + const names = (await fsp.readdir(this.backupRoot)).filter(name => ARCHIVE.test(name)).sort(); + let doomed; + if (mode === 'count') doomed = names.slice(0, Math.max(0, names.length - value)); + else { const cutoff = this.now() - value * 864e5; doomed = names.filter(name => archiveTime(name) < cutoff); } + doomed = doomed.filter(name => name !== keepName && name !== names[names.length - 1]); + for (const name of doomed) { + const target = path.join(this.backupRoot, name); + if (path.dirname(target) !== this.backupRoot) continue; + await fsp.rm(target); + } + return doomed.length; + } +} + +module.exports = { BackupService, BackupError, readArchive, validateSettings, nextOccurrence, DEFAULT_SETTINGS, ARCHIVE }; diff --git a/docker-compose.yml b/docker-compose.yml index 6f7dcf6..66e34ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,10 +36,17 @@ services: source: ${DATA_PATH:-./data} target: /data bind: {create_host_path: false} + # Built-in backups (docs/backup.md). The folder must exist and be writable by + # uid 1000; backups stay off unless BACKUP_PATH is set. + - type: bind + source: ${BACKUP_PATH:-./backup} + target: /backup + bind: {create_host_path: false} environment: PORT: "8080" DATA: /data LIBRARY: /library + BACKUP: ${BACKUP_PATH:+/backup} TZ: ${TZ:-Europe/Berlin} SIGNUP: closed SCAN_INTERVAL: "10" diff --git a/docs/architecture.md b/docs/architecture.md index 8dc1384..8ab383f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,6 +18,7 @@ Related documents: - [peering.md](peering.md): optional private server-to-server catalog and transfer. - [discovery.md](discovery.md): reader-owned, opt-in deterministic discovery feed and its limits. - [source-modules.md](source-modules.md): adding and maintaining first-party hosters inside the single scraper process. +- [backup.md](backup.md): built-in scheduled backup of `data/` and the restore procedure. ## 1. Principles @@ -67,6 +68,7 @@ survives restarts, so restarting does not itself force another complete walk. |------|------:|----------------| | `server.js` | ~350 | HTTP server and router, auth and sessions, JSON storage, API handlers, scan scheduling, notifications and webhooks | | `library.js` | ~170 | Folder scanner, name parsing, image-size reader, scan cache | +| `backup.js` | ~330 | Built-in `DATA` backup: queue pause/resume, tar.gz writer and verifier, schedule, retention | | `public/common.js` | ~85 | Browser helpers: `h()` element builder, `api()` fetch wrapper, top navigation with the notification bell | | `public/index.html` + `library.js` | ~60 | Library overview (cover grid, filter, sort) | | `public/manga.html` + `manga.js` | ~110 | Manga page: info, follow, chapter list with book separators, versions and votes, read/unread toggle | @@ -157,7 +159,11 @@ This keeps ids human-readable and derivable without a database. The price: renam - `data/library-index.json` holds a disposable, versioned index snapshot and shallow directory stamps. The library folders remain authoritative; a missing or invalid snapshot triggers a full scan. Rollback-compatible `scan-cache.json` is retained. -- **Backups:** copy `data/db.json`. The atomic writes make a copy at any time consistent. +- `data/backup.json` holds the built-in backup schedule, retention and recent results. +- **Backups:** the reader's built-in backup ([backup.md](backup.md)) archives the whole + `data/` tree into `BACKUP` on a schedule, with `db.json` taken from memory and the + queue paused around the copy. A plain copy of `data/db.json` also works: atomic + writes make it consistent at any time. ## 6. Server @@ -199,6 +205,7 @@ All requests and responses are JSON. POSTs require `Content-Type: application/js | `POST /api/admin/rescan` `{full}` | admin | Scan now, and wait for the result | | `POST /api/admin/user` `{name, password, admin}` | admin | Create a user or reset a password | | `POST /api/admin/user/delete` `{name}` | admin | Delete a user and their data | +| `GET /api/admin/backup` · `POST /api/admin/backup` `{command:"settings"\|"run"}` | admin | Built-in backup status, schedule/retention, "back up now" ([backup.md](backup.md)) | ### 6.3 Authentication and security @@ -342,7 +349,8 @@ docker compose up -d --build - **Volumes:** - `/library`: read-only in the reader and writable in the scraper. Files must be readable—and scraper targets writable—by uid 1000. - `/data`: reader state must be writable; the scraper also needs its writable `/data/scraper` state. -- **Configuration:** process wiring uses environment variables (`PORT`, `LIBRARY`, `DATA`, `SIGNUP`, `SCAN_INTERVAL`, `FULL_SCAN_INTERVAL`, `COOKIE_SECURE`, `SCRAPER_URL`, and `SCRAPER_TOKEN`). Queue pacing and automatic-update schedule are persisted in `data/scraper/queue.json` and edited through the authenticated admin UI. + - `/backup`: reader only, writable target of the built-in backup (`BACKUP_PATH`; off when unset). +- **Configuration:** process wiring uses environment variables (`PORT`, `LIBRARY`, `DATA`, `BACKUP`, `SIGNUP`, `SCAN_INTERVAL`, `FULL_SCAN_INTERVAL`, `COOKIE_SECURE`, `SCRAPER_URL`, and `SCRAPER_TOKEN`). Queue pacing and automatic-update schedule are persisted in `data/scraper/queue.json` and edited through the authenticated admin UI. - **Without Docker:** `npm run start:all` with Node ≥ 18, or supervise `npm start` and `npm run start:scraper` independently. On the current Windows machine that supervision is already owned by `\MangaCircus Supervisor`. - **Behind a reverse proxy:** terminate TLS, set `COOKIE_SECURE=1`. A sub-path works. - **Upgrades:** replace the code and restart. Reader state has no explicit migration; @@ -366,6 +374,7 @@ docker compose up -d --build |----------|--------------|-----| | Index the folder tree in place | Import into a DB / managed storage | Stays readable by any viewer and writable by any tool. Nothing to keep in sync | | No dependencies, no framework, no build | Express, React, bundler | Minimal code, maximum longevity, runs anywhere Node runs | +| Built-in backup in the reader, queue paused, no container stop | Host job that stops the stack; backup in Sentinel | Only the reader sees all of `/data` and the in-memory `db.json`; the scraper's pause API gives a consistent point without downtime | | JSON file for user data | SQLite (`node:sqlite` is still experimental in Node 22/24), Postgres | Zero dependencies, trivial backup, enough for the target size. Easy to swap: all access goes through the `db` object | | Page roles as file-name tags | Page lists in JSON | Visible in any file browser, survives copying, no second source of truth. Unknown tags are harmless | | Image sizes read on the server | Measure in the browser after load | Layout before load: exact scroll restore, zoom anchoring, auto-detected double pages, no layout jumps | diff --git a/docs/backup.md b/docs/backup.md new file mode 100644 index 0000000..130a1f1 --- /dev/null +++ b/docs/backup.md @@ -0,0 +1,114 @@ +# Built-in backup + +The reader can back up its own `DATA` folder on a schedule set in +**Settings → Admin → Service health → Backups**, without stopping any container. +The library is not part of it (it is plain files; back it up with the storage that +holds it). Implementation: [`backup.js`](../backup.js); tests: `test/backup.test.js`. + +## Turning it on + +| Where | Setting | +|---|---| +| Native | `BACKUP=/path/to/backups` for `server.js`. Unset = feature off | +| Compose | `BACKUP_PATH=/host/folder` in `.env`. The reader mounts it rw at `/backup` and gets `BACKUP=/backup`; without `BACKUP_PATH` the variable stays empty and backups are off | + +The folder must already exist and be writable by uid 1000 (`create_host_path: false`: +a missing folder makes `docker compose up` fail instead of silently creating an empty +one on the wrong disk). Use another disk or a NAS share, never a folder inside `DATA`. +Without `BACKUP_PATH`, Compose still mounts the unused default `./backup`, so that +folder must exist too (`node tools/prepare-compose.js` creates it). Upgrading an +existing deployment therefore needs either `BACKUP_PATH` in `.env` or an empty +`backup/` beside `docker-compose.yml`. + +Archives contain password hashes, sessions and the peering private key: keep the +target private. + +### Why the reader owns it + +The reader is the only service that mounts all of `/data` (the scraper sees only +`/data/scraper`, Sentinel only `/data/sentinel`), it holds `db.json` in memory and owns +its atomic writer, it already talks to the scraper's private control API with the +shared token, and the admin settings UI lives there. Running it in Sentinel would need +a full read-write `/data` mount there and break its "writes only `data/sentinel/`" +contract (see [architecture.md §3](architecture.md#3-components)). + +## Settings and status + +Admin only, under Service health (`GET/POST /api/admin/backup`, also included in +`GET /api/admin/health` as `backup`): + +- **Back up daily at HH:MM** (container time zone `TZ`) and an enable switch. +- **Keep the newest** N backups (`count`, 1–365) or N days (`days`, 1–3650). +- **Back up now** starts a run immediately; the page polls every 2 s while it runs. +- Status: target folder, number/size of archives, the running phase, last backup + (time, trigger, size, duration, file), its error and warnings, next scheduled run. + +Settings and the last 10 results persist atomically in `data/backup.json`. A slot +missed while the reader was down runs once at the next start if it is less than a +day old; older slots (for example after restoring an old `backup.json`) are skipped. + +```text +POST /api/admin/backup {command:"settings", settings:{enabled, time:"HH:MM", retention:{mode:"count"|"days", value}}} +POST /api/admin/backup {command:"run"} -> 409 when off or already running +``` + +## What one run does + +1. Ask the scraper for its state (`GET /api/state`). If the queue runs, pause it + (`POST /api/control {action:"pause"}`) and remember that this run paused it. +2. Wait until no job is active (the worker stops at its next page/chapter boundary), + polling every 2 s for at most 15 minutes. On timeout the run fails, writes nothing + and resumes the queue if it paused it. An unreachable scraper does not block the + backup; that is recorded as a warning. +3. Write `BACKUP/circus-YYYYMMDD-HHMMSS.tar.gz.part`: a gzip'd tar of `DATA` with + paths relative to `DATA`, fsync it, read it back to verify, rename it to `.tar.gz`. + - `db.json` is serialized from the reader's in-memory state at that moment. + - `scraper/queue.json` is stored with `paused: true` (status `paused`), so a + restored copy never starts downloading by itself. The live file is not touched. + - `scan-cache.json`, `library-index.json`, `catalog.json`, peering, Sentinel state, + logs and everything else under `DATA` is copied as found; all of them are + written by atomic rename, so each file is complete. + - Left out: `runtime*.json`, `*.lock` (`daemon.lock`, `supervisor.lock`), `*.pid`, + `*.tmp`, symlinks, and the `BACKUP` folder itself if it lies inside `DATA`. + - `.circus-backup.json` at the archive root records when and how it was made. +4. Resume the queue only if this run paused it (three attempts; a failure is shown + as a warning: resume it under Settings → Scraper). +5. After a successful run only, apply retention. It deletes only files named + `circus-YYYYMMDD-HHMMSS.tar.gz` directly inside `BACKUP` (age from the name), + never the newest one, never other files or subfolders. Stale + `circus-*.tar.gz.part` files from an interrupted run are removed at the next run. + +If the reader restarts during a run, `backup.json` still carries the run's +"queue paused by backup" marker; the next start resumes the queue and records the +run as interrupted. The marker is never part of an archive. + +Measured on a copy of a production data folder (54 MB, 1000 files): 5.8 MB archive in +about 5 s including verification. Pauses last until the current chapter is done +plus those seconds. + +## Restore + +Restoring replaces the user data with the state of the archive. Reading progress +and queue changes made after that backup are lost. + +1. Stop the stack (or at least reader, scraper and Sentinel): + `docker compose stop reader scraper sentinel`. +2. Move the current data folder aside instead of deleting it: + `mv /srv/mangacircus/data /srv/mangacircus/data.before-restore`. +3. Unpack the archive into an empty data folder owned by uid 1000: + ```bash + mkdir /srv/mangacircus/data + tar -xzf /path/to/circus-YYYYMMDD-HHMMSS.tar.gz -C /srv/mangacircus/data + chown -R 1000:1000 /srv/mangacircus/data + ``` + (`--same-owner` is the default for root; the archive keeps the original uid/gid.) +4. Start the reader first: `docker compose up -d --no-build reader`. Check `/health`, + log in, check progress and library. The reader rebuilds a missing or stale library + index by itself. +5. Start `scraper sentinel`. The restored queue is paused: check it under + Settings → Scraper and resume it when it looks right. + +The scraper token is not in `DATA`; Compose keeps it under `.secrets/` (or the +deployment's `SCRAPER_TOKEN_PATH`). Restore tests should unpack into a separate +folder and run an isolated stack (no second writer against the live library), as +a restore-test script does for the host-level archives. diff --git a/docs/settings-navigation.md b/docs/settings-navigation.md index 99dbae3..316885e 100644 --- a/docs/settings-navigation.md +++ b/docs/settings-navigation.md @@ -59,6 +59,10 @@ download times by manga/source, and a collapsed recent-check log. Successful no-change nightly checks update this summary and leave the queue. Sources owns the per-hoster request and chapter delay settings. +Service health also holds the **Backups** card: built-in backup status (last run, +size, duration, error), **Back up now**, and the daily schedule and retention form, +whose draft survives the page's polling. See [backup.md](backup.md). + Reading-time collection controls, export and deletion live under Reading time & privacy; reader layout controls stay under Reading. Rescan controls and user administration are separate. Server & library describes existing maintenance, diff --git a/public/health.js b/public/health.js index f1ad95d..3536650 100644 --- a/public/health.js +++ b/public/health.js @@ -4,10 +4,13 @@ try { me = await api('api/me'); } catch (e) { return showError(e); } topNav(me); if (!me.admin) return showError(new Error('Administrator access required.')); - const main = settingsShell(me, 'admin/health').content; + const shell = settingsShell(me, 'admin/health').content; + const main = h('div'), backup = backupCard(); + shell.append(main, backup.card); async function load() { try { const [health, scraper] = await Promise.all([api('api/admin/health'), api('api/admin/scraper')]); + backup.draw(health.backup); main.textContent = ''; main.append(h('section', { class: 'card' }, h('p', {}, `Reader: ${health.reader.ok ? 'healthy' : 'scan error'} · Sentinel: ${health.sentinel.status} · Scraper: ${scraper.status || 'unavailable'}`), @@ -28,7 +31,56 @@ main.append(h('details', { class: 'card', open: incidents.some(i => i.state === 'open') }, h('summary', {}, 'Incident & recovery history'), incidents.slice(-50).map(i => h('p', {}, `${i.at} · ${i.state} · ${i.message || i.failure?.message || ''}${i.resolution ? ' · ' + i.resolution : ''}`)))); } catch (e) { main.textContent = ''; main.append(h('p', { class: 'error', role: 'alert' }, e.message)); } - setTimeout(load, document.hidden ? 30000 : 10000); + setTimeout(load, document.hidden ? 30000 : backup.busy() ? 2000 : 10000); } load(); + + // Built-in backup: status redraws on every poll; the schedule form keeps its draft. + function backupCard() { + const mb = n => (n / 1048576).toFixed(n < 10485760 ? 1 : 0) + ' MB'; + const secs = ms => ms < 60000 ? Math.round(ms / 1000) + ' s' : Math.round(ms / 60000) + ' min'; + const status = h('div'), msg = h('p', { class: 'small', role: 'status' }); + const enabled = h('input', { type: 'checkbox' }), time = h('input', { type: 'time', required: true }); + const mode = h('select', {}, h('option', { value: 'count' }, 'backups'), h('option', { value: 'days' }, 'days')); + const keep = h('input', { type: 'number', min: '1', step: '1', style: 'width:6em' }); + let state = null, filled = false; + const fill = s => { enabled.checked = s.enabled; time.value = s.time; mode.value = s.retention.mode; keep.value = s.retention.value; filled = true; }; + const command = async (body, button) => { + button.disabled = true; msg.textContent = ''; msg.className = 'small'; + try { const next = await api('api/admin/backup', body); if (body.command === 'settings') { fill(next.settings); msg.textContent = 'Saved.'; } draw(next); } + catch (e) { msg.textContent = e.message; msg.className = 'small error'; } + finally { button.disabled = false; } + }; + const runButton = h('button', { class: 'btn primary', onclick: e => command({ command: 'run' }, e.target) }, 'Back up now'); + const saveButton = h('button', { class: 'btn', onclick: e => command({ command: 'settings', settings: { + enabled: enabled.checked, time: time.value, retention: { mode: mode.value, value: Number(keep.value) } } }, e.target) }, 'Save schedule'); + const form = h('div', { class: 'field' }, h('span', {}, 'Schedule'), + h('div', { class: 'row' }, h('label', { class: 'row' }, enabled, 'Back up daily at'), time), + h('div', { class: 'row', style: 'margin-top:8px' }, h('span', {}, 'Keep the newest'), keep, mode), + h('div', { class: 'row', style: 'margin-top:8px' }, saveButton, runButton), msg); + const card = h('section', { class: 'card', id: 'backup' }, h('h2', {}, 'Backups'), status, form); + function draw(next) { + if (!next) return; + state = next; + if (!filled) fill(next.settings); + const last = next.last, lines = []; + if (!next.configured) lines.push(h('p', { class: 'muted' }, 'Backups are off: mount a folder and set BACKUP_PATH (Compose) or BACKUP (native). See docs/backup.md.')); + else { + lines.push(h('p', {}, `Target: ${next.path} · ${next.archives.count} archive${next.archives.count === 1 ? '' : 's'}, ${mb(next.archives.bytes)}`)); + if (next.targetError) lines.push(h('p', { class: 'error' }, next.targetError)); + } + if (next.running) lines.push(h('p', { class: 'tag' }, `Running (${next.running.trigger}): ${next.running.phase}…`)); + if (last) { + lines.push(h('p', {}, `Last backup: ${new Date(last.at).toLocaleString()} (${last.trigger})`, + last.error ? '' : ` · ${mb(last.size)} · ${secs(last.durationMs || 0)} · ${last.file}`)); + if (last.error) lines.push(h('p', { class: 'error', role: 'alert' }, `Failed: ${last.error}`)); + for (const warning of last.warnings || []) lines.push(h('p', { class: 'small error' }, warning)); + } else if (next.configured) lines.push(h('p', { class: 'muted' }, 'No backup yet.')); + lines.push(h('p', { class: 'muted small' }, next.nextAt ? `Next scheduled backup: ${new Date(next.nextAt).toLocaleString()}` : 'No scheduled backup.')); + status.textContent = ''; status.append(...lines); + runButton.disabled = !next.configured || !!next.running; + form.hidden = !next.configured && !next.settings.enabled; + } + return { card, draw, busy: () => !!state?.running }; + } })(); diff --git a/server.js b/server.js index 6020aac..35c816b 100644 --- a/server.js +++ b/server.js @@ -10,6 +10,7 @@ * COOKIE_SECURE=1 mark the session cookie Secure (when served over HTTPS) * SCRAPER_URL=http://127.0.0.1:8091 scraper daemon API * SCRAPER_TOKEN=... shared daemon token (optional on loopback) + * BACKUP=/backup folder for built-in DATA backups (unset = backups off) */ const http = require('http'), fs = require('fs'), path = require('path'), crypto = require('crypto'); const { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex } = require('./library'); @@ -26,6 +27,7 @@ const { applyPulse, applyVisit, chapterInsight, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary, } = require('./reading-analytics'); const { parseSchedule, publicSchedule, loadQuestionBank, createReadingAccess } = require('./reading-access'); +const { BackupService, BackupError } = require('./backup'); const PORT = +process.env.PORT || 8080; const LIB = path.resolve(process.env.LIBRARY || 'library'); @@ -65,6 +67,9 @@ function save() { saveTimer = saveTimer || setTimeout(flush, 500); } function flush() { clearTimeout(saveTimer); saveTimer = null; writeAtomic(DB_FILE, JSON.stringify(db)); } for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => { flush(); process.exit(0); }); db.readingAccess ||= {}; +// The reader owns backups: it alone mounts all of DATA and holds db.json in memory. +const backup = new BackupService({ dataRoot: DATA, backupRoot: process.env.BACKUP || '', snapshotDb: () => JSON.stringify(db), + scraper: { state: () => scraperRequest('/api/state'), control: action => scraperRequest('/api/control', { action }) } }).start(); const readingSchedule = parseSchedule(process.env.READING_WINDOWS_JSON || '', process.env.READING_TIMEZONE || process.env.TZ || 'Europe/Berlin'); const readingAccess = createReadingAccess({ schedule: readingSchedule, bank: loadQuestionBank(process.env.READING_QUIZ_CATALOG_PATH || ''), @@ -377,7 +382,19 @@ const routes = { admin(u); const sentinel = await fetch(SENTINEL_URL + '/api/state', { headers: SCRAPER_TOKEN ? { authorization: `Bearer ${SCRAPER_TOKEN}` } : {}, signal: AbortSignal.timeout(4000) }) .then(async r => { if (!r.ok) throw new Error(); return r.json(); }).catch(() => ({ status: 'unreachable' })); - return { reader: { ok: !scanState.error, uptime: Math.round(process.uptime()), scan: scanState }, sentinel }; + return { reader: { ok: !scanState.error, uptime: Math.round(process.uptime()), scan: scanState }, sentinel, backup: backup.publicState() }; + }, + 'GET /api/admin/backup': ({ u }) => { admin(u); return backup.publicState(); }, + 'POST /api/admin/backup': ({ u, b }) => { + admin(u); + try { + if (b.command === 'settings') return backup.updateSettings(b.settings); + if (b.command !== 'run') bad(400, 'Unknown backup command'); + if (!backup.backupRoot) bad(409, 'Backups are off: set BACKUP to a mounted folder'); + if (backup.running) bad(409, 'A backup is already running'); + backup.run('manual').catch(() => {}); // the result lands in last/history + return backup.publicState(); + } catch (e) { if (e instanceof BackupError) bad(e.code, e.message); throw e; } }, 'GET /api/auth': () => ({ firstRun: !Object.keys(db.users).length, signup: OPEN_SIGNUP }), diff --git a/test/backup.test.js b/test/backup.test.js new file mode 100644 index 0000000..4fad568 --- /dev/null +++ b/test/backup.test.js @@ -0,0 +1,276 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('fs'), os = require('os'), path = require('path'), net = require('net'); +const { spawn, spawnSync } = require('child_process'); +const { BackupService, readArchive, validateSettings, nextOccurrence } = require('../backup'); +const root = path.resolve(__dirname, '..'); +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); +const quiet = { log() {}, warn() {}, error() {} }; +const LONG = 'collection-covers/' + 'a-very-long-folder-name-that-keeps-going-'.repeat(3) + '/cover-with-a-long-name.png'; + +function fixture() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-backup-')); + const data = path.join(dir, 'data'), target = path.join(dir, 'backup'); + const write = (rel, text) => { fs.mkdirSync(path.dirname(path.join(data, rel)), { recursive: true }); fs.writeFileSync(path.join(data, rel), text); }; + write('db.json', '{"stale":true}'); + write('scan-cache.json', '{"cache":1}'); + write('library-index.json', '{"version":1}'); + write('catalog.json', '{"collections":[]}'); + write('scraper/queue.json', JSON.stringify({ version: 3, paused: false, status: 'idle', queue: [{ id: 'a' }] })); + write('scraper/events.log', '{"level":"info"}\n'); + write('sentinel/state.json', '{}'); + write(LONG, 'png-bytes'); + // Must not be archived. + write('runtime-reader.json', '{}'); write('scraper/runtime.json', '{}'); write('sentinel/runtime.json', '{}'); + write('scraper/daemon.lock', '1'); write('scraper/supervisor.lock', '1'); write('reader.pid', '1'); write('db.json.tmp', '{}'); + fs.mkdirSync(target); + return { dir, data, target, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) }; +} + +function fakeScraper({ paused = false, busyPolls = 0, failResume = false } = {}) { + const calls = []; + let polls = 0; + return { + calls, + get paused() { return paused; }, + async state() { polls++; return { paused, status: paused && polls <= busyPolls ? 'pausing' : paused ? 'paused' : 'downloading', active: polls <= busyPolls ? [{ source: 'x' }] : [] }; }, + async control(action) { + calls.push(action); + if (action === 'resume' && failResume) throw new Error('scraper gone'); + paused = action === 'pause'; + }, + }; +} + +const service = (f, options = {}) => new BackupService({ dataRoot: f.data, backupRoot: f.target, log: quiet, pollMs: 5, resumeRetryMs: 1, + snapshotDb: () => JSON.stringify({ users: { admin: {} }, live: true }), ...options }).load(); + +test('settings validation and schedule computation', () => { + assert.deepEqual(validateSettings({ enabled: true, time: '04:15', retention: { mode: 'days', value: 30 } }), + { enabled: true, time: '04:15', retention: { mode: 'days', value: 30 } }); + assert.throws(() => validateSettings({ time: '24:00' }), /HH:MM/); + assert.throws(() => validateSettings({ retention: { mode: 'count', value: 0 } }), /Keep/); + assert.throws(() => validateSettings({ retention: { mode: 'weeks', value: 2 } }), /count/); + assert.throws(() => validateSettings({ enabled: 'yes' }), /enabled/); + const now = new Date(2026, 9, 22, 3, 0).getTime(); + assert.equal(nextOccurrence('04:00', now), new Date(2026, 9, 22, 4, 0).getTime()); + assert.equal(nextOccurrence('03:00', now), new Date(2026, 9, 23, 3, 0).getTime()); +}); + +test('a backup pauses a running queue, waits for idle, archives a consistent DATA snapshot and resumes', async () => { + const f = fixture(); + try { + const scraper = fakeScraper({ busyPolls: 3 }); + const backup = service(f, { scraper }); + const result = await backup.run('manual'); + assert.equal(result.error, undefined, result.error); + assert.deepEqual(scraper.calls, ['pause', 'resume']); + assert.equal(result.queueWasPaused, false); assert.equal(result.queueResumed, true); + assert.match(result.file, /^circus-\d{8}-\d{6}\.tar\.gz$/); + assert.deepEqual(fs.readdirSync(f.target), [result.file]); + const entries = await readArchive(path.join(f.target, result.file), { contents: true }); + const byName = new Map(entries.map(e => [e.name, e])); + const text = name => byName.get(name).data.toString('utf8'); + // db.json comes from memory, not from the stale file on disk. + assert.deepEqual(JSON.parse(text('db.json')), { users: { admin: {} }, live: true }); + // The archived queue is paused; the live one is untouched by the snapshot. + const queue = JSON.parse(text('scraper/queue.json')); + assert.equal(queue.paused, true); assert.deepEqual(queue.queue, [{ id: 'a' }]); + assert.equal(JSON.parse(fs.readFileSync(path.join(f.data, 'scraper/queue.json'), 'utf8')).paused, false); + for (const name of ['scan-cache.json', 'library-index.json', 'catalog.json', 'scraper/events.log', 'sentinel/state.json', LONG, '.circus-backup.json']) + assert.ok(byName.has(name), `${name} missing`); + assert.equal(text(LONG), 'png-bytes'); + for (const name of ['runtime-reader.json', 'scraper/runtime.json', 'sentinel/runtime.json', 'scraper/daemon.lock', 'scraper/supervisor.lock', 'reader.pid', 'db.json.tmp']) + assert.ok(!byName.has(name), `${name} must be excluded`); + assert.ok(byName.get('scraper/').type === '5'); + // The persisted state records the result but never a pending marker inside the archive. + assert.equal(JSON.parse(text('backup.json')).pending, undefined); + assert.equal(backup.publicState().last.file, result.file); + assert.equal(backup.publicState().archives.count, 1); + } finally { f.cleanup(); } +}); + +test('an already paused queue stays paused and is not resumed', async () => { + const f = fixture(); + try { + const scraper = fakeScraper({ paused: true }); + const result = await service(f, { scraper }).run('schedule'); + assert.equal(result.error, undefined); + assert.deepEqual(scraper.calls, []); + assert.equal(result.queueWasPaused, true); assert.equal(scraper.paused, true); + } finally { f.cleanup(); } +}); + +test('a scraper that never becomes idle aborts the backup, leaves no archive and resumes the queue', async () => { + const f = fixture(); + try { + const scraper = fakeScraper({ busyPolls: Infinity }); + const result = await service(f, { scraper, idleTimeoutMs: 40 }).run('manual'); + assert.match(result.error, /did not become idle/); + assert.deepEqual(scraper.calls, ['pause', 'resume']); + assert.deepEqual(fs.readdirSync(f.target), []); + } finally { f.cleanup(); } +}); + +test('an unreachable scraper does not block the backup and is reported', async () => { + const f = fixture(); + try { + const scraper = { async state() { throw new Error('connect ECONNREFUSED'); }, async control() { throw new Error('unused'); } }; + const result = await service(f, { scraper }).run('manual'); + assert.equal(result.error, undefined); + assert.match(result.warnings[0], /unreachable/); + const queue = (await readArchive(path.join(f.target, result.file), { contents: true })).find(e => e.name === 'scraper/queue.json'); + assert.equal(JSON.parse(queue.data).paused, true); + } finally { f.cleanup(); } +}); + +test('a failed resume is surfaced as a warning', async () => { + const f = fixture(); + try { + const result = await service(f, { scraper: fakeScraper({ failResume: true }) }).run('manual'); + assert.equal(result.error, undefined); + assert.match(result.warnings.join(' '), /could not be resumed/); + } finally { f.cleanup(); } +}); + +test('retention keeps the newest archives and never touches other files', async () => { + const f = fixture(); + try { + const old = ['circus-20260101-030000.tar.gz', 'circus-20260102-030000.tar.gz', 'circus-20260103-030000.tar.gz']; + const foreign = ['circus-20250101-0300.tar.zst', 'notes.txt', 'circus-20250101-030000.tar.gz.bak']; + for (const name of [...old, ...foreign]) fs.writeFileSync(path.join(f.target, name), 'x'); + fs.mkdirSync(path.join(f.target, 'circus-20250101-030000.tar.gz-dir')); + fs.writeFileSync(path.join(f.target, 'circus-20250101-030000.tar.gz.part'), 'stale'); + const backup = service(f); + backup.updateSettings({ retention: { mode: 'count', value: 2 } }); + const result = await backup.run('manual'); + assert.equal(result.error, undefined); assert.equal(result.deleted, 2); + assert.deepEqual(fs.readdirSync(f.target).sort(), [...foreign, 'circus-20250101-030000.tar.gz-dir', old[2], result.file].sort()); + // Days mode works from the timestamp in the name. + backup.updateSettings({ retention: { mode: 'days', value: 1 } }); + const second = await backup.run('manual'); + assert.ok(!fs.existsSync(path.join(f.target, old[2]))); + assert.ok(fs.existsSync(path.join(f.target, second.file))); + for (const name of foreign) assert.ok(fs.existsSync(path.join(f.target, name))); + } finally { f.cleanup(); } +}); + +test('backups are off without BACKUP, and a missing folder is reported, never created', async () => { + const f = fixture(); + try { + const off = service(f, { backupRoot: '' }); + await assert.rejects(off.run('manual'), /Backups are off/); + assert.equal(off.publicState().configured, false); + const missing = path.join(f.dir, 'not-mounted'); + const result = await service(f, { backupRoot: missing }).run('manual'); + assert.match(result.error, /not usable/); + assert.ok(!fs.existsSync(missing)); + } finally { f.cleanup(); } +}); + +test('only one run at a time', async () => { + const f = fixture(); + try { + const backup = service(f, { scraper: fakeScraper({ busyPolls: 5 }) }); + const first = backup.run('manual'); + await assert.rejects(backup.run('manual'), /already running/); + assert.equal((await first).error, undefined); + } finally { f.cleanup(); } +}); + +test('the schedule runs once per slot, catches up a recent miss and skips stale slots', async () => { + const f = fixture(); + try { + let now = new Date(2026, 9, 22, 2, 0).getTime(); + const backup = service(f, { now: () => now }); + backup.updateSettings({ enabled: true, time: '03:30' }); + assert.equal(backup.state.nextAt, new Date(2026, 9, 22, 3, 30).getTime()); + assert.equal(await backup.tick(), null); + now = new Date(2026, 9, 22, 9, 0).getTime(); // reader was down at 03:30 + const result = await backup.tick(); + assert.equal(result.trigger, 'schedule'); assert.equal(result.error, undefined); + assert.equal(backup.state.nextAt, new Date(2026, 9, 23, 3, 30).getTime()); + assert.equal(await backup.tick(), null); + now = new Date(2026, 9, 30, 9, 0).getTime(); // e.g. an old backup.json was restored + assert.equal(await backup.tick(), null); + assert.equal(backup.state.nextAt, new Date(2026, 9, 31, 3, 30).getTime()); + assert.equal(fs.readdirSync(f.target).length, 1); + } finally { f.cleanup(); } +}); + +test('a run interrupted by a restart resumes the queue it paused', async () => { + const f = fixture(); + try { + fs.writeFileSync(path.join(f.data, 'backup.json'), JSON.stringify({ version: 1, pending: { startedAt: 1, trigger: 'schedule', pausedByUs: true } })); + const scraper = fakeScraper({ paused: true }); + const backup = service(f, { scraper }); + await backup.recoverInterrupted(); + assert.deepEqual(scraper.calls, ['resume']); + assert.match(backup.state.last.error, /Interrupted/); + assert.equal(JSON.parse(fs.readFileSync(path.join(f.data, 'backup.json'), 'utf8')).pending, null); + } finally { f.cleanup(); } +}); + +test('system tar extracts the archive', { skip: spawnSync('tar', ['--version']).error ? 'no tar binary' : false }, async () => { + const f = fixture(); + try { + const result = await service(f).run('manual'); + const out = path.join(f.dir, 'restored'); fs.mkdirSync(out); + const tar = spawnSync('tar', ['-xzf', path.join(f.target, result.file), '-C', out], { encoding: 'utf8' }); + assert.equal(tar.status, 0, tar.stderr); + assert.equal(fs.readFileSync(path.join(out, LONG), 'utf8'), 'png-bytes'); + assert.equal(JSON.parse(fs.readFileSync(path.join(out, 'scraper/queue.json'), 'utf8')).paused, true); + assert.ok(!fs.existsSync(path.join(out, 'scraper/daemon.lock'))); + } finally { f.cleanup(); } +}); + +async function port() { + const server = net.createServer(); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const result = server.address().port; await new Promise(resolve => server.close(resolve)); return result; +} + +test('reader and scraper: admin backup API pauses and resumes the real queue', { timeout: 30000 }, async () => { + const f = fixture(), children = [], output = []; + try { + const library = path.join(f.dir, 'library'); fs.mkdirSync(library); + fs.writeFileSync(path.join(f.data, 'scraper', 'queue.json'), JSON.stringify({ paused: false, queue: [], settings: { autoUpdateEnabled: false } })); + for (const name of ['runtime.json', 'daemon.lock', 'supervisor.lock']) fs.rmSync(path.join(f.data, 'scraper', name), { force: true }); + for (const name of ['db.json', 'catalog.json', 'library-index.json', 'scan-cache.json']) fs.rmSync(path.join(f.data, name)); + const [rp, sp] = await Promise.all([port(), port()]); + const base = `http://127.0.0.1:${rp}`, token = 'backup-test-only'; + const env = { ...process.env, DATA: f.data, LIBRARY: library, SCRAPER_DATA: path.join(f.data, 'scraper'), BACKUP: f.target, + PORT: String(rp), SCRAPER_PORT: String(sp), SCRAPER_HOST: '127.0.0.1', SCRAPER_URL: `http://127.0.0.1:${sp}`, SENTINEL_URL: 'http://127.0.0.1:9', + SCRAPER_TOKEN: token, SCRAPER_TOKEN_FILE: '', SCAN_INTERVAL: '0' }; + for (const script of ['server.js', 'scraper/daemon.js']) { + const child = spawn(process.execPath, [script], { cwd: root, env, windowsHide: true }); children.push(child); + for (const stream of [child.stdout, child.stderr]) stream.on('data', chunk => output.push(chunk.toString())); + } + for (const url of [base, `http://127.0.0.1:${sp}`]) { + let ready = false; + for (let n = 0; n < 80 && !ready; n++) { ready = await fetch(url + '/health').then(r => r.ok).catch(() => false); if (!ready) await delay(100); } + assert.ok(ready, output.join('')); + } + const signup = await fetch(base + '/api/signup', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'admin', password: 'local-test-password' }) }); + const cookie = signup.headers.get('set-cookie').split(';')[0]; + const request = (endpoint, body) => fetch(base + endpoint, { headers: { cookie, 'content-type': 'application/json' }, ...(body ? { method: 'POST', body: JSON.stringify(body) } : {}) }); + assert.equal((await request('/api/admin/backup', { command: 'settings', settings: { time: '25:00' } })).status, 400); + const saved = await (await request('/api/admin/backup', { command: 'settings', settings: { enabled: true, time: '04:00', retention: { mode: 'count', value: 3 } } })).json(); + assert.equal(saved.settings.retention.value, 3); assert.ok(saved.nextAt > Date.now()); + assert.equal((await request('/api/admin/backup', { command: 'run' })).status, 200); + let state; + for (let n = 0; n < 100; n++) { state = await (await request('/api/admin/backup')).json(); if (state.last && !state.running) break; await delay(100); } + assert.equal(state.last.error, undefined, state.last.error); + assert.equal(state.last.queueWasPaused, false); assert.equal(state.last.queueResumed, true); + const scraperState = await (await request('/api/admin/scraper')).json(); + assert.equal(scraperState.paused, false); + const entries = await readArchive(path.join(f.target, state.last.file), { contents: true }); + const db = JSON.parse(entries.find(e => e.name === 'db.json').data); + assert.ok(db.users.admin); + assert.equal(JSON.parse(entries.find(e => e.name === 'scraper/queue.json').data).paused, true); + assert.ok((await (await request('/api/admin/health')).json()).backup.last.file); + } finally { + await Promise.all(children.map(child => new Promise(resolve => { if (child.exitCode !== null) return resolve(); child.once('exit', resolve); child.kill(); }))); + f.cleanup(); + } +}); diff --git a/tools/prepare-compose.js b/tools/prepare-compose.js index 100e227..c5a3b81 100644 --- a/tools/prepare-compose.js +++ b/tools/prepare-compose.js @@ -2,7 +2,7 @@ 'use strict'; const fs = require('fs'), path = require('path'), crypto = require('crypto'); const root = path.resolve(__dirname, '..'); -for (const relative of ['data', 'data/scraper', 'data/sentinel', 'library', '.secrets']) fs.mkdirSync(path.join(root, relative), { recursive: true }); +for (const relative of ['data', 'data/scraper', 'data/sentinel', 'library', 'backup', '.secrets']) fs.mkdirSync(path.join(root, relative), { recursive: true }); const file = path.join(root, '.secrets', 'scraper_token'); if (!fs.existsSync(file)) fs.writeFileSync(file, crypto.randomBytes(32).toString('hex') + '\n', { flag: 'wx', mode: 0o600 }); if (!fs.existsSync(path.join(root, '.env'))) fs.copyFileSync(path.join(root, '.env.example'), path.join(root, '.env')); diff --git a/tools/test-compose.js b/tools/test-compose.js index 3b38120..a694c32 100644 --- a/tools/test-compose.js +++ b/tools/test-compose.js @@ -6,10 +6,10 @@ const assert = require('assert/strict'), { spawn } = require('child_process'); const { Canvas } = require('./png'); const root = path.resolve(__dirname, '..'), dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-compose-')); const project = 'mc-qa-' + crypto.randomBytes(3).toString('hex'), port = process.env.QA_PORT || '8180'; -const library = path.join(dir, 'library'), data = path.join(dir, 'data'), tokenFile = path.join(dir, 'token'); +const library = path.join(dir, 'library'), data = path.join(dir, 'data'), backupDir = path.join(dir, 'backup'), tokenFile = path.join(dir, 'token'); const directionQa = process.argv.includes('--direction-ui'); const peerGatewayQa = process.argv.includes('--peer-gateway'); -fs.mkdirSync(path.join(data, 'scraper'), { recursive: true }); fs.mkdirSync(path.join(data, 'sentinel')); +fs.mkdirSync(path.join(data, 'scraper'), { recursive: true }); fs.mkdirSync(path.join(data, 'sentinel')); fs.mkdirSync(backupDir); if (peerGatewayQa) { const peerData = path.join(data, 'peering'); const peerUsage = path.join(data, 'peering-usage'); @@ -48,7 +48,7 @@ for (const [index, name] of ['Moon Garden', 'Before the Rain', 'Iron Harbor'].en status: index ? 'completed' : 'ongoing', tags: ['Fantasy', 'Demo'] })); } const env = { ...process.env, LIBRARY_PATH: library, DATA_PATH: data, SCRAPER_DATA_PATH: path.join(data, 'scraper'), SENTINEL_DATA_PATH: path.join(data, 'sentinel'), - SCRAPER_TOKEN_PATH: tokenFile, READER_PORT: port, READER_BIND: '127.0.0.1', COOKIE_SECURE: '0', APP_IMAGE: process.env.APP_IMAGE || 'mangacircus:local', + SCRAPER_TOKEN_PATH: tokenFile, BACKUP_PATH: backupDir, READER_PORT: port, READER_BIND: '127.0.0.1', COOKIE_SECURE: '0', APP_IMAGE: process.env.APP_IMAGE || 'mangacircus:local', READER_IMAGE: process.env.APP_IMAGE || 'mangacircus:local', ...(readingGateQa ? { READING_WINDOWS_JSON: '{"0":[],"1":[],"2":[],"3":[],"4":[],"5":[],"6":[]}', READING_TIMEZONE: 'Europe/Berlin', READING_QUIZ_UNLOCK_MINUTES: '60', READING_QUIZ_CATALOG_PATH: '/data/reading-quiz.json' } @@ -93,6 +93,14 @@ let keep = false; let inventory; await until(async () => { inventory = await (await api('/api/library')).json(); return inventory.mangas?.length === 3; }, 'Fixture inventory missing'); const byTitle = title => inventory.mangas.find(m => m.title === title).id; + // Built-in backup through the real control API: archive written, paused queue left paused. + const queueBefore = JSON.parse(fs.readFileSync(path.join(data, 'scraper', 'queue.json'), 'utf8')); + assert.equal((await api('/api/admin/backup', { command: 'run' })).status, 200); + let backupState; + await until(async () => { backupState = await (await api('/api/admin/backup')).json(); return backupState.last && !backupState.running; }, 'Backup did not finish'); + assert.equal(backupState.last.error, undefined, backupState.last.error); + assert.ok(fs.existsSync(path.join(backupDir, backupState.last.file)), 'Backup archive missing on the host'); + assert.equal(JSON.parse(fs.readFileSync(path.join(data, 'scraper', 'queue.json'), 'utf8')).paused, queueBefore.paused); if (readingGateQa) { const manga = byTitle('Moon Garden'), other = byTitle('Before the Rain'); const chapter = (await (await api('/api/manga?m=' + encodeURIComponent(manga))).json()).chapters[0].id; @@ -243,7 +251,7 @@ let keep = false; keep = process.argv.includes('--keep'); if (keep) { const envFile = path.join(dir, 'qa.env'); - fs.writeFileSync(envFile, ['LIBRARY_PATH', 'DATA_PATH', 'SCRAPER_DATA_PATH', 'SENTINEL_DATA_PATH', 'SCRAPER_TOKEN_PATH', 'READER_PORT', 'READER_BIND', 'COOKIE_SECURE', 'APP_IMAGE', + fs.writeFileSync(envFile, ['LIBRARY_PATH', 'DATA_PATH', 'SCRAPER_DATA_PATH', 'SENTINEL_DATA_PATH', 'SCRAPER_TOKEN_PATH', 'BACKUP_PATH', 'READER_PORT', 'READER_BIND', 'COOKIE_SECURE', 'APP_IMAGE', ...(readingGateQa ? ['READING_WINDOWS_JSON', 'READING_TIMEZONE', 'READING_QUIZ_UNLOCK_MINUTES', 'READING_QUIZ_CATALOG_PATH'] : [])] .map(k => `${k}=${k === 'READING_WINDOWS_JSON' ? `'${env[k]}'` : env[k].replaceAll('\\', '/')}`).join('\n') + '\n'); console.log(JSON.stringify({ project, url, envFile, fixture: dir }));