Add built-in backup of the data folder (BL-0044)
The reader owns a scheduled backup into BACKUP (Compose: BACKUP_PATH mounted at /backup, off when unset). A run pauses the scraper queue through its control API, waits (bounded) until no job is active, writes a verified tar.gz of DATA with db.json from memory and scraper/queue.json stored paused, excludes runtime/lock/pid/temp files, resumes only a queue it paused, then applies count- or day-based retention to its own archive names only. Schedule, retention, "Back up now" and last result live in Settings > Service health. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
MaashkinoozheandClaude Opus 5.5 committed
1 parent
fcc50d576c
commit
086f6353f8
15 files changed
+904
-11
No files matched your search
@@ -6,6 +6,7 @@
|
||||
!reading-direction.js
|
||||
!chapter-identity.js
|
||||
!catalog.js
|
||||
!backup.js
|
||||
!reading-analytics.js
|
||||
!reading-access.js
|
||||
!public/
|
||||
|
||||
@@ -9,6 +9,9 @@ DATA_PATH=./data
|
||||
SCRAPER_DATA_PATH=./data/scraper
|
||||
SENTINEL_DATA_PATH=./data/sentinel
|
||||
SCRAPER_TOKEN_PATH=./.secrets/scraper_token
|
||||
# Built-in backups of DATA_PATH (docs/backup.md): an existing folder, ideally on another
|
||||
# disk/NAS, writable by uid 1000. Unset = backups off (./backup is then mounted unused).
|
||||
# BACKUP_PATH=./backup
|
||||
TZ=Europe/Berlin
|
||||
COOKIE_SECURE=0
|
||||
# Optional server-enforced reading windows. Day keys: 0=Sunday through 6=Saturday.
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
/data/
|
||||
# the manga library: demo content is generated by `node tools/make-dummy.js`, a real one is yours alone
|
||||
/library/
|
||||
# built-in backups (BACKUP_PATH default)
|
||||
/backup/
|
||||
|
||||
# --- Node ---
|
||||
node_modules/
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ RUN apk add --no-cache tzdata openssl
|
||||
WORKDIR /app
|
||||
ARG VCS_REF=development
|
||||
LABEL org.opencontainers.image.title="MangaCircus" org.opencontainers.image.revision=$VCS_REF
|
||||
COPY package.json server.js library.js reading-direction.js chapter-identity.js reading-analytics.js reading-access.js catalog.js ./
|
||||
COPY package.json server.js library.js reading-direction.js chapter-identity.js reading-analytics.js reading-access.js catalog.js backup.js ./
|
||||
COPY public ./public
|
||||
COPY greatest ./greatest
|
||||
COPY discovery ./discovery
|
||||
|
||||
@@ -74,6 +74,7 @@ see administration; personal settings do not poll scraper services. See
|
||||
| `SCRAPER_TOKEN` | empty | Shared daemon token; required when it listens beyond loopback |
|
||||
| `SCRAPER_TOKEN_FILE` | empty | Alternative secret file (used in Compose) |
|
||||
| `SENTINEL_URL` | `http://127.0.0.1:8092` | Private observation service URL |
|
||||
| `BACKUP` | – | Existing folder for built-in backups of `DATA`; unset = off. Compose: set `BACKUP_PATH`. Schedule and retention under Settings → Service health ([backup](docs/backup.md)) |
|
||||
|
||||
Serve it behind a reverse proxy with HTTPS if it is reachable from the internet. All pages use relative URLs, so a sub-path like `/manga/` works.
|
||||
|
||||
@@ -113,6 +114,7 @@ whole-series completeness. See the scraper service and library format contracts
|
||||
| [docs/architecture.md](docs/architecture.md) | Components, data model, API, security, the reader's layout engine, deployment, design decisions |
|
||||
| [docs/local-deployment.md](docs/local-deployment.md) | Current local Compose deployment, Tailscale access, persistent state, checks, restart, and recovery |
|
||||
| [docs/settings-navigation.md](docs/settings-navigation.md) | User/Admin settings sections, scraper tabs, routes and UI checks |
|
||||
| [docs/backup.md](docs/backup.md) | Built-in scheduled backup of `data/`, what it contains, retention and the restore procedure |
|
||||
| [docs/reading-insights-concept.md](docs/reading-insights-concept.md) | Active-time presentation, private summaries, deletion/export controls, and the future journal design |
|
||||
| [docs/scraper-guide.md](docs/scraper-guide.md) | Rule book (agent-readable) for scrapers and other tools that write into the library |
|
||||
| [scraper/AGENT_RUNBOOK.md](scraper/AGENT_RUNBOOK.md) | Conservative scraper workflow and host-specific procedures for automated agents |
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
'use strict';
|
||||
/*
|
||||
* Built-in backup of the reader's DATA tree into BACKUP (see docs/backup.md).
|
||||
*
|
||||
* One run: pause the scraper queue through its control API, wait (bounded) until no
|
||||
* job is active, write a gzip'd tar of DATA into BACKUP, resume the queue only if this
|
||||
* run paused it, then apply retention. No container stops. db.json comes from the
|
||||
* reader's in-memory state; scraper/queue.json is stored with `paused: true` so a
|
||||
* restored copy never starts downloading by itself. Runtime/lock/pid/temp files are
|
||||
* left out. Retention only ever deletes `circus-YYYYMMDD-HHMMSS.tar.gz` files directly
|
||||
* inside BACKUP. No dependencies: the tar writer/reader below is plain ustar + PAX.
|
||||
*/
|
||||
const fs = require('fs'), fsp = fs.promises, path = require('path'), zlib = require('zlib');
|
||||
const { Readable } = require('stream'), { pipeline } = require('stream/promises');
|
||||
|
||||
const ARCHIVE = /^circus-(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})\.tar\.gz$/;
|
||||
const PART = /^circus-\d{8}-\d{6}\.tar\.gz\.part$/;
|
||||
const EXCLUDE = [/^runtime.*\.json$/, /\.lock$/, /\.pid$/, /\.tmp$/];
|
||||
const DEFAULT_SETTINGS = { enabled: false, time: '03:30', retention: { mode: 'count', value: 14 } };
|
||||
const RETENTION_LIMITS = { count: [1, 365], days: [1, 3650] };
|
||||
const CATCH_UP_MS = 24 * 3600e3;
|
||||
|
||||
class BackupError extends Error { constructor(code, msg) { super(msg); this.code = code; } }
|
||||
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
const pad = n => String(n).padStart(2, '0');
|
||||
const stampOf = d => `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`;
|
||||
const archiveTime = name => { const m = ARCHIVE.exec(name); return m ? new Date(+m[1], m[2] - 1, +m[3], +m[4], +m[5], +m[6]).getTime() : NaN; };
|
||||
const message = e => String((e && e.message) || e).slice(0, 500);
|
||||
|
||||
function validateSettings(input, current = DEFAULT_SETTINGS) {
|
||||
const next = { ...current, retention: { ...current.retention } };
|
||||
if (input.enabled !== undefined) {
|
||||
if (typeof input.enabled !== 'boolean') throw new BackupError(400, 'enabled must be true or false');
|
||||
next.enabled = input.enabled;
|
||||
}
|
||||
if (input.time !== undefined) {
|
||||
if (!/^([01]\d|2[0-3]):[0-5]\d$/.test(String(input.time))) throw new BackupError(400, 'The backup time must be HH:MM');
|
||||
next.time = String(input.time);
|
||||
}
|
||||
if (input.retention !== undefined) {
|
||||
const mode = input.retention && input.retention.mode, value = Number(input.retention && input.retention.value);
|
||||
const limits = RETENTION_LIMITS[mode];
|
||||
if (!limits) throw new BackupError(400, 'Retention is "count" or "days"');
|
||||
if (!Number.isInteger(value) || value < limits[0] || value > limits[1]) throw new BackupError(400, `Keep ${limits[0]}–${limits[1]} ${mode === 'count' ? 'backups' : 'days'}`);
|
||||
next.retention = { mode, value };
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
// Next local occurrence of HH:MM strictly after `now` (process TZ).
|
||||
function nextOccurrence(time, now) {
|
||||
const [hh, mm] = time.split(':').map(Number), d = new Date(now);
|
||||
const at = new Date(d.getFullYear(), d.getMonth(), d.getDate(), hh, mm, 0, 0);
|
||||
if (at.getTime() <= now) at.setDate(at.getDate() + 1);
|
||||
return at.getTime();
|
||||
}
|
||||
|
||||
// ---------- tar (ustar + PAX path records), enough for `tar -xzf` everywhere ----------
|
||||
function octal(buf, offset, length, value) { buf.write(value.toString(8).padStart(length - 1, '0') + '\0', offset, length, 'ascii'); }
|
||||
function rawHeader(name, { size = 0, mode = 0o644, uid = 0, gid = 0, mtime = 0, type = '0' }) {
|
||||
const b = Buffer.alloc(512);
|
||||
b.write(name, 0, 100, 'utf8');
|
||||
octal(b, 100, 8, mode & 0o7777); octal(b, 108, 8, uid & 0o7777777); octal(b, 116, 8, gid & 0o7777777);
|
||||
octal(b, 124, 12, size); octal(b, 136, 12, Math.max(0, Math.floor(mtime / 1000)));
|
||||
b.write(' ', 148, 8, 'ascii');
|
||||
b.write(type, 156, 1, 'ascii');
|
||||
b.write('ustar\0' + '00', 257, 8, 'ascii');
|
||||
let sum = 0; for (const byte of b) sum += byte;
|
||||
b.write(sum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii');
|
||||
return b;
|
||||
}
|
||||
const padding = size => Buffer.alloc((512 - (size % 512)) % 512);
|
||||
function paxRecord(key, value) {
|
||||
const body = ` ${key}=${value}\n`, bytes = Buffer.byteLength(body);
|
||||
let length = bytes + String(bytes).length;
|
||||
if (String(length).length !== String(bytes).length) length = bytes + String(length).length;
|
||||
return `${length}${body}`;
|
||||
}
|
||||
function tarEntry(name, data, meta) {
|
||||
const parts = [];
|
||||
// Names longer than the ustar field or non-ASCII go into a PAX header first.
|
||||
if (Buffer.byteLength(name) > 100 || /[^\x20-\x7e]/.test(name)) {
|
||||
const pax = Buffer.from(paxRecord('path', name), 'utf8');
|
||||
parts.push(rawHeader('PaxHeader/' + name.replace(/[^\x20-\x7e]/g, '_').slice(-80), { size: pax.length, type: 'x', mtime: meta.mtime }), pax, padding(pax.length));
|
||||
name = name.replace(/[^\x20-\x7e]/g, '_').slice(-100);
|
||||
}
|
||||
parts.push(rawHeader(name, { ...meta, size: data ? data.length : 0 }));
|
||||
if (data && data.length) parts.push(data, padding(data.length));
|
||||
return parts;
|
||||
}
|
||||
|
||||
// Streaming reader for verification and tests: [{ name, type, size, mode, mtime, data? }].
|
||||
async function readArchive(file, { contents = false } = {}) {
|
||||
const entries = []; let pending = Buffer.alloc(0), current = null, paxPath = null, ended = false;
|
||||
const consume = chunk => {
|
||||
pending = pending.length ? Buffer.concat([pending, chunk]) : chunk;
|
||||
for (;;) {
|
||||
if (ended) return;
|
||||
if (current) {
|
||||
if (pending.length < current.need) return;
|
||||
const body = pending.subarray(0, current.size);
|
||||
if (current.entry.type === 'x') paxPath = /(?:^|\n)\d+ path=([^\n]*)\n/.exec(body.toString('utf8'))?.[1] ?? null;
|
||||
else { if (contents) current.entry.data = Buffer.from(body); entries.push(current.entry); }
|
||||
pending = pending.subarray(current.need); current = null; continue;
|
||||
}
|
||||
if (pending.length < 512) return;
|
||||
const h = pending.subarray(0, 512); pending = pending.subarray(512);
|
||||
if (h.every(byte => byte === 0)) { ended = true; return; }
|
||||
let sum = 0; for (let i = 0; i < 512; i++) sum += i >= 148 && i < 156 ? 32 : h[i];
|
||||
if (sum !== parseInt(h.toString('ascii', 148, 156), 8)) throw new Error('Corrupt tar header');
|
||||
const field = (o, l) => h.toString('utf8', o, o + l).replace(/\0.*$/s, '');
|
||||
const size = parseInt(field(124, 12), 8) || 0, type = field(156, 1) || '0';
|
||||
const prefix = field(345, 155), name = prefix ? prefix + '/' + field(0, 100) : field(0, 100);
|
||||
const entry = { name: type === 'x' ? name : (paxPath ?? name), type, size, mode: parseInt(field(100, 8), 8), mtime: parseInt(field(136, 12), 8) * 1000 };
|
||||
if (type !== 'x') paxPath = null;
|
||||
current = { entry, size, need: size + padding(size).length };
|
||||
}
|
||||
};
|
||||
await pipeline(fs.createReadStream(file), zlib.createGunzip(), async function* (source) { for await (const chunk of source) consume(chunk); });
|
||||
if (!ended) throw new Error('Archive ends without the tar end marker');
|
||||
return entries;
|
||||
}
|
||||
|
||||
class BackupService {
|
||||
/**
|
||||
* dataRoot DATA (read) backupRoot BACKUP (rw) or '' = feature off
|
||||
* stateFile settings + history snapshotDb () => db.json text from memory
|
||||
* scraper { state(), control(action) } or null
|
||||
*/
|
||||
constructor({ dataRoot, backupRoot, stateFile, snapshotDb, scraper = null, now = Date.now, log = console,
|
||||
idleTimeoutMs = 15 * 60e3, pollMs = 2000, resumeRetryMs = 5000 }) {
|
||||
this.dataRoot = path.resolve(dataRoot);
|
||||
this.backupRoot = backupRoot ? path.resolve(backupRoot) : '';
|
||||
this.stateFile = stateFile || path.join(this.dataRoot, 'backup.json');
|
||||
this.snapshotDb = snapshotDb; this.scraper = scraper; this.now = now; this.log = log;
|
||||
this.idleTimeoutMs = idleTimeoutMs; this.pollMs = pollMs; this.resumeRetryMs = resumeRetryMs;
|
||||
this.state = { version: 1, settings: { ...DEFAULT_SETTINGS }, nextAt: null, last: null, history: [], pending: null };
|
||||
this.running = null; this.archives = []; this.targetError = null; this.timer = null;
|
||||
}
|
||||
|
||||
load() {
|
||||
try { Object.assign(this.state, JSON.parse(fs.readFileSync(this.stateFile, 'utf8'))); }
|
||||
catch (e) { if (e.code !== 'ENOENT') this.log.warn(`[backup] ignoring unreadable ${this.stateFile}: ${e.message}`); }
|
||||
this.state.settings = validateSettings(this.state.settings || {}, DEFAULT_SETTINGS);
|
||||
this.state.history = Array.isArray(this.state.history) ? this.state.history.slice(-10) : [];
|
||||
return this;
|
||||
}
|
||||
|
||||
persist() {
|
||||
fs.writeFileSync(this.stateFile + '.tmp', JSON.stringify(this.state, null, 2) + '\n');
|
||||
fs.renameSync(this.stateFile + '.tmp', this.stateFile);
|
||||
}
|
||||
|
||||
// A run interrupted by a restart may have left the queue paused on its behalf.
|
||||
async recoverInterrupted() {
|
||||
const pending = this.state.pending;
|
||||
if (!pending) return;
|
||||
this.state.pending = null;
|
||||
const result = { at: pending.startedAt, trigger: pending.trigger, error: 'Interrupted by a reader restart', warnings: [] };
|
||||
if (pending.pausedByUs) {
|
||||
try { await this.resume(); result.warnings.push('Queue resumed after the interrupted backup'); }
|
||||
catch (e) { result.warnings.push(`Queue left paused: ${message(e)}`); }
|
||||
}
|
||||
this.record(result);
|
||||
}
|
||||
|
||||
async refreshTarget() {
|
||||
if (!this.backupRoot) { this.targetError = null; this.archives = []; return; }
|
||||
try {
|
||||
const st = await fsp.stat(this.backupRoot);
|
||||
if (!st.isDirectory()) throw new Error('not a directory');
|
||||
await fsp.access(this.backupRoot, fs.constants.W_OK);
|
||||
const names = (await fsp.readdir(this.backupRoot)).filter(name => ARCHIVE.test(name)).sort();
|
||||
this.archives = await Promise.all(names.map(async name => ({ name, size: (await fsp.stat(path.join(this.backupRoot, name))).size, at: archiveTime(name) })));
|
||||
this.targetError = null;
|
||||
} catch (e) { this.targetError = `Backup folder ${this.backupRoot} is not usable: ${e.code || message(e)}`; this.archives = []; }
|
||||
}
|
||||
|
||||
publicState() {
|
||||
const s = this.state;
|
||||
return {
|
||||
configured: !!this.backupRoot, path: this.backupRoot || null, targetError: this.targetError,
|
||||
settings: s.settings, nextAt: s.settings.enabled && this.backupRoot ? s.nextAt : null,
|
||||
running: this.running && { ...this.running }, last: s.last, history: s.history.slice().reverse(),
|
||||
archives: { count: this.archives.length, bytes: this.archives.reduce((n, a) => n + a.size, 0),
|
||||
newest: this.archives.length ? this.archives[this.archives.length - 1].name : null },
|
||||
};
|
||||
}
|
||||
|
||||
updateSettings(input) {
|
||||
this.state.settings = validateSettings(input || {}, this.state.settings);
|
||||
this.state.nextAt = this.state.settings.enabled ? nextOccurrence(this.state.settings.time, this.now()) : null;
|
||||
this.persist();
|
||||
return this.publicState();
|
||||
}
|
||||
|
||||
start(intervalMs = 30000) {
|
||||
this.load();
|
||||
this.refreshTarget().then(() => this.recoverInterrupted()).catch(e => this.log.error('[backup]', message(e)));
|
||||
this.timer = setInterval(() => this.tick(), intervalMs);
|
||||
this.timer.unref?.();
|
||||
return this;
|
||||
}
|
||||
|
||||
stop() { clearInterval(this.timer); }
|
||||
|
||||
// Scheduled runs: catch up a missed slot from the last day, never older ones
|
||||
// (e.g. right after restoring an old backup.json).
|
||||
async tick() {
|
||||
const s = this.state.settings, now = this.now();
|
||||
if (!s.enabled || !this.backupRoot || this.running) return null;
|
||||
if (!this.state.nextAt) { this.state.nextAt = nextOccurrence(s.time, now); this.persist(); return null; }
|
||||
if (now < this.state.nextAt) return null;
|
||||
const due = now - this.state.nextAt < CATCH_UP_MS;
|
||||
this.state.nextAt = nextOccurrence(s.time, now); this.persist();
|
||||
return due ? this.run('schedule').catch(() => null) : null;
|
||||
}
|
||||
|
||||
/** Starts a run and resolves with its result; rejects at once when no run is possible. */
|
||||
run(trigger = 'manual') {
|
||||
if (!this.backupRoot) return Promise.reject(new BackupError(409, 'Backups are off: set BACKUP to a mounted folder'));
|
||||
if (this.running) return Promise.reject(new BackupError(409, 'A backup is already running'));
|
||||
this.running = { startedAt: this.now(), trigger, phase: 'starting' };
|
||||
return this.perform(trigger).finally(() => { this.running = null; });
|
||||
}
|
||||
|
||||
async perform(trigger) {
|
||||
const started = this.now(), result = { at: started, trigger, warnings: [] };
|
||||
let part = null, pausedByUs = false;
|
||||
try {
|
||||
await this.refreshTarget();
|
||||
if (this.targetError) throw new Error(this.targetError);
|
||||
await this.removeStaleParts();
|
||||
// 1. Pause the queue (unless already paused) and wait for the active chapter.
|
||||
let scraperState = null;
|
||||
if (this.scraper) {
|
||||
try { scraperState = await this.scraper.state(); }
|
||||
catch (e) { result.warnings.push(`Scraper unreachable, queue not paused: ${message(e)}`); }
|
||||
}
|
||||
if (scraperState) {
|
||||
result.queueWasPaused = !!scraperState.paused;
|
||||
if (!scraperState.paused) {
|
||||
this.state.pending = { startedAt: started, trigger, pausedByUs: true }; this.persist();
|
||||
await this.scraper.control('pause'); pausedByUs = true;
|
||||
}
|
||||
this.running.phase = 'waiting for the scraper to finish its current step';
|
||||
await this.waitIdle(scraperState);
|
||||
}
|
||||
// 2. Snapshot DATA into BACKUP/<name>.part, then rename.
|
||||
this.running.phase = 'writing archive';
|
||||
let stamp = started, name, file;
|
||||
do { name = `circus-${stampOf(new Date(stamp))}.tar.gz`; file = path.join(this.backupRoot, name); stamp += 1000; }
|
||||
while (fs.existsSync(file)); // two runs within one second must not overwrite each other
|
||||
part = file + '.part';
|
||||
const stats = await this.writeArchive(part);
|
||||
this.running.phase = 'verifying archive';
|
||||
const entries = await readArchive(part);
|
||||
const files = entries.filter(e => e.type === '0').length;
|
||||
if (files !== stats.files) throw new Error(`Verification read ${files} files, expected ${stats.files}`);
|
||||
await fsp.rename(part, file); part = null;
|
||||
Object.assign(result, { file: name, size: (await fsp.stat(file)).size, files, dataBytes: stats.bytes });
|
||||
} catch (e) {
|
||||
result.error = message(e);
|
||||
if (part) await fsp.rm(part, { force: true }).catch(() => {});
|
||||
} finally {
|
||||
// 3. Resume only what this run paused.
|
||||
if (pausedByUs) {
|
||||
this.running.phase = 'resuming queue';
|
||||
try { await this.resume(); result.queueResumed = true; }
|
||||
catch (e) { result.warnings.push(`Queue could not be resumed, resume it in Settings → Scraper: ${message(e)}`); }
|
||||
}
|
||||
if (this.state.pending) this.state.pending = null;
|
||||
}
|
||||
// 4. Retention only after a successful run, so failures never thin out old copies.
|
||||
if (!result.error) {
|
||||
this.running.phase = 'applying retention';
|
||||
try { result.deleted = await this.applyRetention(result.file); }
|
||||
catch (e) { result.warnings.push(`Retention failed: ${message(e)}`); }
|
||||
}
|
||||
result.durationMs = this.now() - started;
|
||||
await this.refreshTarget();
|
||||
this.record(result);
|
||||
if (result.error) this.log.error(`[backup] failed: ${result.error}`);
|
||||
else this.log.log(`[backup] ${result.file} ${result.size} bytes in ${result.durationMs} ms`);
|
||||
return result;
|
||||
}
|
||||
|
||||
record(result) {
|
||||
this.state.last = result;
|
||||
this.state.history = [...this.state.history, result].slice(-10);
|
||||
this.persist();
|
||||
}
|
||||
|
||||
async waitIdle(initial) {
|
||||
const deadline = this.now() + this.idleTimeoutMs;
|
||||
let state = initial, lastError = null;
|
||||
for (;;) {
|
||||
if (state && !(state.active || []).length && state.status !== 'pausing') return;
|
||||
if (this.now() >= deadline) {
|
||||
throw new Error(`The scraper did not become idle within ${Math.round(this.idleTimeoutMs / 60000)} min${lastError ? ` (${lastError})` : ''}; no backup written`);
|
||||
}
|
||||
await delay(this.pollMs);
|
||||
try { state = await this.scraper.state(); lastError = null; }
|
||||
catch (e) { state = null; lastError = message(e); }
|
||||
}
|
||||
}
|
||||
|
||||
async resume() {
|
||||
let error;
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try { await this.scraper.control('resume'); return; }
|
||||
catch (e) { error = e; if (attempt < 2) await delay(this.resumeRetryMs); }
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Files of DATA in a stable order, skipping excluded names, symlinks and a nested BACKUP.
|
||||
async collect(dir = this.dataRoot, rel = '') {
|
||||
const out = [];
|
||||
const entries = (await fsp.readdir(dir, { withFileTypes: true })).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
||||
for (const entry of entries) {
|
||||
const full = path.join(dir, entry.name), relative = rel ? rel + '/' + entry.name : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
if (full === this.backupRoot) continue;
|
||||
out.push({ full, relative, dir: true });
|
||||
out.push(...await this.collect(full, relative));
|
||||
} else if (entry.isFile() && !EXCLUDE.some(re => re.test(entry.name))) out.push({ full, relative, dir: false });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Contents of one file as it goes into the archive.
|
||||
async contentOf(item) {
|
||||
if (item.relative === 'db.json' && this.snapshotDb) return Buffer.from(this.snapshotDb());
|
||||
if (item.relative === path.relative(this.dataRoot, this.stateFile).split(path.sep).join('/')) {
|
||||
const { pending, ...rest } = this.state; void pending;
|
||||
return Buffer.from(JSON.stringify(rest, null, 2) + '\n');
|
||||
}
|
||||
const data = await fsp.readFile(item.full);
|
||||
if (item.relative === 'scraper/queue.json') {
|
||||
const queue = JSON.parse(data.toString('utf8'));
|
||||
queue.paused = true; queue.status = 'paused';
|
||||
return Buffer.from(JSON.stringify(queue, null, 2) + '\n');
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
async writeArchive(part) {
|
||||
const items = await this.collect(), stats = { files: 0, bytes: 0 };
|
||||
// The in-memory db is archived even before its first debounced write.
|
||||
if (this.snapshotDb && !items.some(item => item.relative === 'db.json')) items.unshift({ full: path.join(this.dataRoot, 'db.json'), relative: 'db.json', dir: false, memory: true });
|
||||
const manifest = { kind: 'mangacircus-data-backup', version: 1, created: new Date(this.now()).toISOString(),
|
||||
excluded: 'runtime*.json, *.lock, *.pid, *.tmp', queuePaused: true };
|
||||
const self = this;
|
||||
async function* tar() {
|
||||
yield* tarEntry('.circus-backup.json', Buffer.from(JSON.stringify(manifest, null, 2) + '\n'), { mtime: self.now() });
|
||||
for (const item of items) {
|
||||
let st;
|
||||
try { st = item.memory ? { mode: 0o644, uid: process.getuid?.() ?? 0, gid: process.getgid?.() ?? 0, mtimeMs: self.now() } : await fsp.stat(item.full); }
|
||||
catch (e) { if (e.code === 'ENOENT') continue; throw e; }
|
||||
const meta = { mode: st.mode, uid: st.uid, gid: st.gid, mtime: st.mtimeMs };
|
||||
if (item.dir) { yield* tarEntry(item.relative + '/', null, { ...meta, type: '5' }); continue; }
|
||||
let data;
|
||||
try { data = await self.contentOf(item); } catch (e) { if (e.code === 'ENOENT') continue; throw new Error(`${item.relative}: ${message(e)}`); }
|
||||
stats.files++; stats.bytes += data.length;
|
||||
yield* tarEntry(item.relative, data, meta);
|
||||
}
|
||||
yield Buffer.alloc(1024);
|
||||
}
|
||||
await pipeline(Readable.from(tar()), zlib.createGzip(), fs.createWriteStream(part, { flags: 'wx' }));
|
||||
const handle = await fsp.open(part, 'r+'); // fsync before the rename makes it visible
|
||||
try { await handle.sync(); } finally { await handle.close(); }
|
||||
stats.files++; // the manifest
|
||||
return stats;
|
||||
}
|
||||
|
||||
async removeStaleParts() {
|
||||
for (const name of await fsp.readdir(this.backupRoot)) if (PART.test(name)) await fsp.rm(path.join(this.backupRoot, name), { force: true });
|
||||
}
|
||||
|
||||
// Deletes only our own archive names directly inside BACKUP; never the newest one.
|
||||
async applyRetention(keepName) {
|
||||
const { mode, value } = this.state.settings.retention;
|
||||
const names = (await fsp.readdir(this.backupRoot)).filter(name => ARCHIVE.test(name)).sort();
|
||||
let doomed;
|
||||
if (mode === 'count') doomed = names.slice(0, Math.max(0, names.length - value));
|
||||
else { const cutoff = this.now() - value * 864e5; doomed = names.filter(name => archiveTime(name) < cutoff); }
|
||||
doomed = doomed.filter(name => name !== keepName && name !== names[names.length - 1]);
|
||||
for (const name of doomed) {
|
||||
const target = path.join(this.backupRoot, name);
|
||||
if (path.dirname(target) !== this.backupRoot) continue;
|
||||
await fsp.rm(target);
|
||||
}
|
||||
return doomed.length;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { BackupService, BackupError, readArchive, validateSettings, nextOccurrence, DEFAULT_SETTINGS, ARCHIVE };
|
||||
@@ -36,10 +36,17 @@ services:
|
||||
source: ${DATA_PATH:-./data}
|
||||
target: /data
|
||||
bind: {create_host_path: false}
|
||||
# Built-in backups (docs/backup.md). The folder must exist and be writable by
|
||||
# uid 1000; backups stay off unless BACKUP_PATH is set.
|
||||
- type: bind
|
||||
source: ${BACKUP_PATH:-./backup}
|
||||
target: /backup
|
||||
bind: {create_host_path: false}
|
||||
environment:
|
||||
PORT: "8080"
|
||||
DATA: /data
|
||||
LIBRARY: /library
|
||||
BACKUP: ${BACKUP_PATH:+/backup}
|
||||
TZ: ${TZ:-Europe/Berlin}
|
||||
SIGNUP: closed
|
||||
SCAN_INTERVAL: "10"
|
||||
|
||||
+11
-2
@@ -18,6 +18,7 @@ Related documents:
|
||||
- [peering.md](peering.md): optional private server-to-server catalog and transfer.
|
||||
- [discovery.md](discovery.md): reader-owned, opt-in deterministic discovery feed and its limits.
|
||||
- [source-modules.md](source-modules.md): adding and maintaining first-party hosters inside the single scraper process.
|
||||
- [backup.md](backup.md): built-in scheduled backup of `data/` and the restore procedure.
|
||||
|
||||
## 1. Principles
|
||||
|
||||
@@ -67,6 +68,7 @@ survives restarts, so restarting does not itself force another complete walk.
|
||||
|------|------:|----------------|
|
||||
| `server.js` | ~350 | HTTP server and router, auth and sessions, JSON storage, API handlers, scan scheduling, notifications and webhooks |
|
||||
| `library.js` | ~170 | Folder scanner, name parsing, image-size reader, scan cache |
|
||||
| `backup.js` | ~330 | Built-in `DATA` backup: queue pause/resume, tar.gz writer and verifier, schedule, retention |
|
||||
| `public/common.js` | ~85 | Browser helpers: `h()` element builder, `api()` fetch wrapper, top navigation with the notification bell |
|
||||
| `public/index.html` + `library.js` | ~60 | Library overview (cover grid, filter, sort) |
|
||||
| `public/manga.html` + `manga.js` | ~110 | Manga page: info, follow, chapter list with book separators, versions and votes, read/unread toggle |
|
||||
@@ -157,7 +159,11 @@ This keeps ids human-readable and derivable without a database. The price: renam
|
||||
- `data/library-index.json` holds a disposable, versioned index snapshot and shallow
|
||||
directory stamps. The library folders remain authoritative; a missing or invalid
|
||||
snapshot triggers a full scan. Rollback-compatible `scan-cache.json` is retained.
|
||||
- **Backups:** copy `data/db.json`. The atomic writes make a copy at any time consistent.
|
||||
- `data/backup.json` holds the built-in backup schedule, retention and recent results.
|
||||
- **Backups:** the reader's built-in backup ([backup.md](backup.md)) archives the whole
|
||||
`data/` tree into `BACKUP` on a schedule, with `db.json` taken from memory and the
|
||||
queue paused around the copy. A plain copy of `data/db.json` also works: atomic
|
||||
writes make it consistent at any time.
|
||||
|
||||
## 6. Server
|
||||
|
||||
@@ -199,6 +205,7 @@ All requests and responses are JSON. POSTs require `Content-Type: application/js
|
||||
| `POST /api/admin/rescan` `{full}` | admin | Scan now, and wait for the result |
|
||||
| `POST /api/admin/user` `{name, password, admin}` | admin | Create a user or reset a password |
|
||||
| `POST /api/admin/user/delete` `{name}` | admin | Delete a user and their data |
|
||||
| `GET /api/admin/backup` · `POST /api/admin/backup` `{command:"settings"\|"run"}` | admin | Built-in backup status, schedule/retention, "back up now" ([backup.md](backup.md)) |
|
||||
|
||||
### 6.3 Authentication and security
|
||||
|
||||
@@ -342,7 +349,8 @@ docker compose up -d --build
|
||||
- **Volumes:**
|
||||
- `/library`: read-only in the reader and writable in the scraper. Files must be readable—and scraper targets writable—by uid 1000.
|
||||
- `/data`: reader state must be writable; the scraper also needs its writable `/data/scraper` state.
|
||||
- **Configuration:** process wiring uses environment variables (`PORT`, `LIBRARY`, `DATA`, `SIGNUP`, `SCAN_INTERVAL`, `FULL_SCAN_INTERVAL`, `COOKIE_SECURE`, `SCRAPER_URL`, and `SCRAPER_TOKEN`). Queue pacing and automatic-update schedule are persisted in `data/scraper/queue.json` and edited through the authenticated admin UI.
|
||||
- `/backup`: reader only, writable target of the built-in backup (`BACKUP_PATH`; off when unset).
|
||||
- **Configuration:** process wiring uses environment variables (`PORT`, `LIBRARY`, `DATA`, `BACKUP`, `SIGNUP`, `SCAN_INTERVAL`, `FULL_SCAN_INTERVAL`, `COOKIE_SECURE`, `SCRAPER_URL`, and `SCRAPER_TOKEN`). Queue pacing and automatic-update schedule are persisted in `data/scraper/queue.json` and edited through the authenticated admin UI.
|
||||
- **Without Docker:** `npm run start:all` with Node ≥ 18, or supervise `npm start` and `npm run start:scraper` independently. On the current Windows machine that supervision is already owned by `\MangaCircus Supervisor`.
|
||||
- **Behind a reverse proxy:** terminate TLS, set `COOKIE_SECURE=1`. A sub-path works.
|
||||
- **Upgrades:** replace the code and restart. Reader state has no explicit migration;
|
||||
@@ -366,6 +374,7 @@ docker compose up -d --build
|
||||
|----------|--------------|-----|
|
||||
| Index the folder tree in place | Import into a DB / managed storage | Stays readable by any viewer and writable by any tool. Nothing to keep in sync |
|
||||
| No dependencies, no framework, no build | Express, React, bundler | Minimal code, maximum longevity, runs anywhere Node runs |
|
||||
| Built-in backup in the reader, queue paused, no container stop | Host job that stops the stack; backup in Sentinel | Only the reader sees all of `/data` and the in-memory `db.json`; the scraper's pause API gives a consistent point without downtime |
|
||||
| JSON file for user data | SQLite (`node:sqlite` is still experimental in Node 22/24), Postgres | Zero dependencies, trivial backup, enough for the target size. Easy to swap: all access goes through the `db` object |
|
||||
| Page roles as file-name tags | Page lists in JSON | Visible in any file browser, survives copying, no second source of truth. Unknown tags are harmless |
|
||||
| Image sizes read on the server | Measure in the browser after load | Layout before load: exact scroll restore, zoom anchoring, auto-detected double pages, no layout jumps |
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
# Built-in backup
|
||||
|
||||
The reader can back up its own `DATA` folder on a schedule set in
|
||||
**Settings → Admin → Service health → Backups**, without stopping any container.
|
||||
The library is not part of it (it is plain files; back it up with the storage that
|
||||
holds it). Implementation: [`backup.js`](../backup.js); tests: `test/backup.test.js`.
|
||||
|
||||
## Turning it on
|
||||
|
||||
| Where | Setting |
|
||||
|---|---|
|
||||
| Native | `BACKUP=/path/to/backups` for `server.js`. Unset = feature off |
|
||||
| Compose | `BACKUP_PATH=/host/folder` in `.env`. The reader mounts it rw at `/backup` and gets `BACKUP=/backup`; without `BACKUP_PATH` the variable stays empty and backups are off |
|
||||
|
||||
The folder must already exist and be writable by uid 1000 (`create_host_path: false`:
|
||||
a missing folder makes `docker compose up` fail instead of silently creating an empty
|
||||
one on the wrong disk). Use another disk or a NAS share, never a folder inside `DATA`.
|
||||
Without `BACKUP_PATH`, Compose still mounts the unused default `./backup`, so that
|
||||
folder must exist too (`node tools/prepare-compose.js` creates it). Upgrading an
|
||||
existing deployment therefore needs either `BACKUP_PATH` in `.env` or an empty
|
||||
`backup/` beside `docker-compose.yml`.
|
||||
|
||||
Archives contain password hashes, sessions and the peering private key: keep the
|
||||
target private.
|
||||
|
||||
### Why the reader owns it
|
||||
|
||||
The reader is the only service that mounts all of `/data` (the scraper sees only
|
||||
`/data/scraper`, Sentinel only `/data/sentinel`), it holds `db.json` in memory and owns
|
||||
its atomic writer, it already talks to the scraper's private control API with the
|
||||
shared token, and the admin settings UI lives there. Running it in Sentinel would need
|
||||
a full read-write `/data` mount there and break its "writes only `data/sentinel/`"
|
||||
contract (see [architecture.md §3](architecture.md#3-components)).
|
||||
|
||||
## Settings and status
|
||||
|
||||
Admin only, under Service health (`GET/POST /api/admin/backup`, also included in
|
||||
`GET /api/admin/health` as `backup`):
|
||||
|
||||
- **Back up daily at HH:MM** (container time zone `TZ`) and an enable switch.
|
||||
- **Keep the newest** N backups (`count`, 1–365) or N days (`days`, 1–3650).
|
||||
- **Back up now** starts a run immediately; the page polls every 2 s while it runs.
|
||||
- Status: target folder, number/size of archives, the running phase, last backup
|
||||
(time, trigger, size, duration, file), its error and warnings, next scheduled run.
|
||||
|
||||
Settings and the last 10 results persist atomically in `data/backup.json`. A slot
|
||||
missed while the reader was down runs once at the next start if it is less than a
|
||||
day old; older slots (for example after restoring an old `backup.json`) are skipped.
|
||||
|
||||
```text
|
||||
POST /api/admin/backup {command:"settings", settings:{enabled, time:"HH:MM", retention:{mode:"count"|"days", value}}}
|
||||
POST /api/admin/backup {command:"run"} -> 409 when off or already running
|
||||
```
|
||||
|
||||
## What one run does
|
||||
|
||||
1. Ask the scraper for its state (`GET /api/state`). If the queue runs, pause it
|
||||
(`POST /api/control {action:"pause"}`) and remember that this run paused it.
|
||||
2. Wait until no job is active (the worker stops at its next page/chapter boundary),
|
||||
polling every 2 s for at most 15 minutes. On timeout the run fails, writes nothing
|
||||
and resumes the queue if it paused it. An unreachable scraper does not block the
|
||||
backup; that is recorded as a warning.
|
||||
3. Write `BACKUP/circus-YYYYMMDD-HHMMSS.tar.gz.part`: a gzip'd tar of `DATA` with
|
||||
paths relative to `DATA`, fsync it, read it back to verify, rename it to `.tar.gz`.
|
||||
- `db.json` is serialized from the reader's in-memory state at that moment.
|
||||
- `scraper/queue.json` is stored with `paused: true` (status `paused`), so a
|
||||
restored copy never starts downloading by itself. The live file is not touched.
|
||||
- `scan-cache.json`, `library-index.json`, `catalog.json`, peering, Sentinel state,
|
||||
logs and everything else under `DATA` is copied as found; all of them are
|
||||
written by atomic rename, so each file is complete.
|
||||
- Left out: `runtime*.json`, `*.lock` (`daemon.lock`, `supervisor.lock`), `*.pid`,
|
||||
`*.tmp`, symlinks, and the `BACKUP` folder itself if it lies inside `DATA`.
|
||||
- `.circus-backup.json` at the archive root records when and how it was made.
|
||||
4. Resume the queue only if this run paused it (three attempts; a failure is shown
|
||||
as a warning: resume it under Settings → Scraper).
|
||||
5. After a successful run only, apply retention. It deletes only files named
|
||||
`circus-YYYYMMDD-HHMMSS.tar.gz` directly inside `BACKUP` (age from the name),
|
||||
never the newest one, never other files or subfolders. Stale
|
||||
`circus-*.tar.gz.part` files from an interrupted run are removed at the next run.
|
||||
|
||||
If the reader restarts during a run, `backup.json` still carries the run's
|
||||
"queue paused by backup" marker; the next start resumes the queue and records the
|
||||
run as interrupted. The marker is never part of an archive.
|
||||
|
||||
Measured on a copy of a production data folder (54 MB, 1000 files): 5.8 MB archive in
|
||||
about 5 s including verification. Pauses last until the current chapter is done
|
||||
plus those seconds.
|
||||
|
||||
## Restore
|
||||
|
||||
Restoring replaces the user data with the state of the archive. Reading progress
|
||||
and queue changes made after that backup are lost.
|
||||
|
||||
1. Stop the stack (or at least reader, scraper and Sentinel):
|
||||
`docker compose stop reader scraper sentinel`.
|
||||
2. Move the current data folder aside instead of deleting it:
|
||||
`mv /srv/mangacircus/data /srv/mangacircus/data.before-restore`.
|
||||
3. Unpack the archive into an empty data folder owned by uid 1000:
|
||||
```bash
|
||||
mkdir /srv/mangacircus/data
|
||||
tar -xzf /path/to/circus-YYYYMMDD-HHMMSS.tar.gz -C /srv/mangacircus/data
|
||||
chown -R 1000:1000 /srv/mangacircus/data
|
||||
```
|
||||
(`--same-owner` is the default for root; the archive keeps the original uid/gid.)
|
||||
4. Start the reader first: `docker compose up -d --no-build reader`. Check `/health`,
|
||||
log in, check progress and library. The reader rebuilds a missing or stale library
|
||||
index by itself.
|
||||
5. Start `scraper sentinel`. The restored queue is paused: check it under
|
||||
Settings → Scraper and resume it when it looks right.
|
||||
|
||||
The scraper token is not in `DATA`; Compose keeps it under `.secrets/` (or the
|
||||
deployment's `SCRAPER_TOKEN_PATH`). Restore tests should unpack into a separate
|
||||
folder and run an isolated stack (no second writer against the live library), as
|
||||
a restore-test script does for the host-level archives.
|
||||
@@ -59,6 +59,10 @@ download times by manga/source, and a collapsed recent-check log. Successful
|
||||
no-change nightly checks update this summary and leave the queue. Sources owns
|
||||
the per-hoster request and chapter delay settings.
|
||||
|
||||
Service health also holds the **Backups** card: built-in backup status (last run,
|
||||
size, duration, error), **Back up now**, and the daily schedule and retention form,
|
||||
whose draft survives the page's polling. See [backup.md](backup.md).
|
||||
|
||||
Reading-time collection controls, export and deletion live under Reading time &
|
||||
privacy; reader layout controls stay under Reading. Rescan controls and user
|
||||
administration are separate. Server & library describes existing maintenance,
|
||||
|
||||
+54
-2
@@ -4,10 +4,13 @@
|
||||
try { me = await api('api/me'); } catch (e) { return showError(e); }
|
||||
topNav(me);
|
||||
if (!me.admin) return showError(new Error('Administrator access required.'));
|
||||
const main = settingsShell(me, 'admin/health').content;
|
||||
const shell = settingsShell(me, 'admin/health').content;
|
||||
const main = h('div'), backup = backupCard();
|
||||
shell.append(main, backup.card);
|
||||
async function load() {
|
||||
try {
|
||||
const [health, scraper] = await Promise.all([api('api/admin/health'), api('api/admin/scraper')]);
|
||||
backup.draw(health.backup);
|
||||
main.textContent = '';
|
||||
main.append(h('section', { class: 'card' },
|
||||
h('p', {}, `Reader: ${health.reader.ok ? 'healthy' : 'scan error'} · Sentinel: ${health.sentinel.status} · Scraper: ${scraper.status || 'unavailable'}`),
|
||||
@@ -28,7 +31,56 @@
|
||||
main.append(h('details', { class: 'card', open: incidents.some(i => i.state === 'open') }, h('summary', {}, 'Incident & recovery history'), incidents.slice(-50).map(i =>
|
||||
h('p', {}, `${i.at} · ${i.state} · ${i.message || i.failure?.message || ''}${i.resolution ? ' · ' + i.resolution : ''}`))));
|
||||
} catch (e) { main.textContent = ''; main.append(h('p', { class: 'error', role: 'alert' }, e.message)); }
|
||||
setTimeout(load, document.hidden ? 30000 : 10000);
|
||||
setTimeout(load, document.hidden ? 30000 : backup.busy() ? 2000 : 10000);
|
||||
}
|
||||
load();
|
||||
|
||||
// Built-in backup: status redraws on every poll; the schedule form keeps its draft.
|
||||
function backupCard() {
|
||||
const mb = n => (n / 1048576).toFixed(n < 10485760 ? 1 : 0) + ' MB';
|
||||
const secs = ms => ms < 60000 ? Math.round(ms / 1000) + ' s' : Math.round(ms / 60000) + ' min';
|
||||
const status = h('div'), msg = h('p', { class: 'small', role: 'status' });
|
||||
const enabled = h('input', { type: 'checkbox' }), time = h('input', { type: 'time', required: true });
|
||||
const mode = h('select', {}, h('option', { value: 'count' }, 'backups'), h('option', { value: 'days' }, 'days'));
|
||||
const keep = h('input', { type: 'number', min: '1', step: '1', style: 'width:6em' });
|
||||
let state = null, filled = false;
|
||||
const fill = s => { enabled.checked = s.enabled; time.value = s.time; mode.value = s.retention.mode; keep.value = s.retention.value; filled = true; };
|
||||
const command = async (body, button) => {
|
||||
button.disabled = true; msg.textContent = ''; msg.className = 'small';
|
||||
try { const next = await api('api/admin/backup', body); if (body.command === 'settings') { fill(next.settings); msg.textContent = 'Saved.'; } draw(next); }
|
||||
catch (e) { msg.textContent = e.message; msg.className = 'small error'; }
|
||||
finally { button.disabled = false; }
|
||||
};
|
||||
const runButton = h('button', { class: 'btn primary', onclick: e => command({ command: 'run' }, e.target) }, 'Back up now');
|
||||
const saveButton = h('button', { class: 'btn', onclick: e => command({ command: 'settings', settings: {
|
||||
enabled: enabled.checked, time: time.value, retention: { mode: mode.value, value: Number(keep.value) } } }, e.target) }, 'Save schedule');
|
||||
const form = h('div', { class: 'field' }, h('span', {}, 'Schedule'),
|
||||
h('div', { class: 'row' }, h('label', { class: 'row' }, enabled, 'Back up daily at'), time),
|
||||
h('div', { class: 'row', style: 'margin-top:8px' }, h('span', {}, 'Keep the newest'), keep, mode),
|
||||
h('div', { class: 'row', style: 'margin-top:8px' }, saveButton, runButton), msg);
|
||||
const card = h('section', { class: 'card', id: 'backup' }, h('h2', {}, 'Backups'), status, form);
|
||||
function draw(next) {
|
||||
if (!next) return;
|
||||
state = next;
|
||||
if (!filled) fill(next.settings);
|
||||
const last = next.last, lines = [];
|
||||
if (!next.configured) lines.push(h('p', { class: 'muted' }, 'Backups are off: mount a folder and set BACKUP_PATH (Compose) or BACKUP (native). See docs/backup.md.'));
|
||||
else {
|
||||
lines.push(h('p', {}, `Target: ${next.path} · ${next.archives.count} archive${next.archives.count === 1 ? '' : 's'}, ${mb(next.archives.bytes)}`));
|
||||
if (next.targetError) lines.push(h('p', { class: 'error' }, next.targetError));
|
||||
}
|
||||
if (next.running) lines.push(h('p', { class: 'tag' }, `Running (${next.running.trigger}): ${next.running.phase}…`));
|
||||
if (last) {
|
||||
lines.push(h('p', {}, `Last backup: ${new Date(last.at).toLocaleString()} (${last.trigger})`,
|
||||
last.error ? '' : ` · ${mb(last.size)} · ${secs(last.durationMs || 0)} · ${last.file}`));
|
||||
if (last.error) lines.push(h('p', { class: 'error', role: 'alert' }, `Failed: ${last.error}`));
|
||||
for (const warning of last.warnings || []) lines.push(h('p', { class: 'small error' }, warning));
|
||||
} else if (next.configured) lines.push(h('p', { class: 'muted' }, 'No backup yet.'));
|
||||
lines.push(h('p', { class: 'muted small' }, next.nextAt ? `Next scheduled backup: ${new Date(next.nextAt).toLocaleString()}` : 'No scheduled backup.'));
|
||||
status.textContent = ''; status.append(...lines);
|
||||
runButton.disabled = !next.configured || !!next.running;
|
||||
form.hidden = !next.configured && !next.settings.enabled;
|
||||
}
|
||||
return { card, draw, busy: () => !!state?.running };
|
||||
}
|
||||
})();
|
||||
@@ -10,6 +10,7 @@
|
||||
* COOKIE_SECURE=1 mark the session cookie Secure (when served over HTTPS)
|
||||
* SCRAPER_URL=http://127.0.0.1:8091 scraper daemon API
|
||||
* SCRAPER_TOKEN=... shared daemon token (optional on loopback)
|
||||
* BACKUP=/backup folder for built-in DATA backups (unset = backups off)
|
||||
*/
|
||||
const http = require('http'), fs = require('fs'), path = require('path'), crypto = require('crypto');
|
||||
const { scanLibrary, scanChangedLibrary, libraryStamps, snapshotIndex, restoreIndex } = require('./library');
|
||||
@@ -26,6 +27,7 @@ const {
|
||||
applyPulse, applyVisit, chapterInsight, eligiblePages, emptyRecord, libraryInsight, mangaInsight, publicSummary,
|
||||
} = require('./reading-analytics');
|
||||
const { parseSchedule, publicSchedule, loadQuestionBank, createReadingAccess } = require('./reading-access');
|
||||
const { BackupService, BackupError } = require('./backup');
|
||||
|
||||
const PORT = +process.env.PORT || 8080;
|
||||
const LIB = path.resolve(process.env.LIBRARY || 'library');
|
||||
@@ -65,6 +67,9 @@ function save() { saveTimer = saveTimer || setTimeout(flush, 500); }
|
||||
function flush() { clearTimeout(saveTimer); saveTimer = null; writeAtomic(DB_FILE, JSON.stringify(db)); }
|
||||
for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => { flush(); process.exit(0); });
|
||||
db.readingAccess ||= {};
|
||||
// The reader owns backups: it alone mounts all of DATA and holds db.json in memory.
|
||||
const backup = new BackupService({ dataRoot: DATA, backupRoot: process.env.BACKUP || '', snapshotDb: () => JSON.stringify(db),
|
||||
scraper: { state: () => scraperRequest('/api/state'), control: action => scraperRequest('/api/control', { action }) } }).start();
|
||||
const readingSchedule = parseSchedule(process.env.READING_WINDOWS_JSON || '', process.env.READING_TIMEZONE || process.env.TZ || 'Europe/Berlin');
|
||||
const readingAccess = createReadingAccess({ schedule: readingSchedule,
|
||||
bank: loadQuestionBank(process.env.READING_QUIZ_CATALOG_PATH || ''),
|
||||
@@ -377,7 +382,19 @@ const routes = {
|
||||
admin(u);
|
||||
const sentinel = await fetch(SENTINEL_URL + '/api/state', { headers: SCRAPER_TOKEN ? { authorization: `Bearer ${SCRAPER_TOKEN}` } : {}, signal: AbortSignal.timeout(4000) })
|
||||
.then(async r => { if (!r.ok) throw new Error(); return r.json(); }).catch(() => ({ status: 'unreachable' }));
|
||||
return { reader: { ok: !scanState.error, uptime: Math.round(process.uptime()), scan: scanState }, sentinel };
|
||||
return { reader: { ok: !scanState.error, uptime: Math.round(process.uptime()), scan: scanState }, sentinel, backup: backup.publicState() };
|
||||
},
|
||||
'GET /api/admin/backup': ({ u }) => { admin(u); return backup.publicState(); },
|
||||
'POST /api/admin/backup': ({ u, b }) => {
|
||||
admin(u);
|
||||
try {
|
||||
if (b.command === 'settings') return backup.updateSettings(b.settings);
|
||||
if (b.command !== 'run') bad(400, 'Unknown backup command');
|
||||
if (!backup.backupRoot) bad(409, 'Backups are off: set BACKUP to a mounted folder');
|
||||
if (backup.running) bad(409, 'A backup is already running');
|
||||
backup.run('manual').catch(() => {}); // the result lands in last/history
|
||||
return backup.publicState();
|
||||
} catch (e) { if (e instanceof BackupError) bad(e.code, e.message); throw e; }
|
||||
},
|
||||
'GET /api/auth': () => ({ firstRun: !Object.keys(db.users).length, signup: OPEN_SIGNUP }),
|
||||
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
'use strict';
|
||||
const test = require('node:test'), assert = require('node:assert/strict');
|
||||
const fs = require('fs'), os = require('os'), path = require('path'), net = require('net');
|
||||
const { spawn, spawnSync } = require('child_process');
|
||||
const { BackupService, readArchive, validateSettings, nextOccurrence } = require('../backup');
|
||||
const root = path.resolve(__dirname, '..');
|
||||
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
const quiet = { log() {}, warn() {}, error() {} };
|
||||
const LONG = 'collection-covers/' + 'a-very-long-folder-name-that-keeps-going-'.repeat(3) + '/cover-with-a-long-name.png';
|
||||
|
||||
function fixture() {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-backup-'));
|
||||
const data = path.join(dir, 'data'), target = path.join(dir, 'backup');
|
||||
const write = (rel, text) => { fs.mkdirSync(path.dirname(path.join(data, rel)), { recursive: true }); fs.writeFileSync(path.join(data, rel), text); };
|
||||
write('db.json', '{"stale":true}');
|
||||
write('scan-cache.json', '{"cache":1}');
|
||||
write('library-index.json', '{"version":1}');
|
||||
write('catalog.json', '{"collections":[]}');
|
||||
write('scraper/queue.json', JSON.stringify({ version: 3, paused: false, status: 'idle', queue: [{ id: 'a' }] }));
|
||||
write('scraper/events.log', '{"level":"info"}\n');
|
||||
write('sentinel/state.json', '{}');
|
||||
write(LONG, 'png-bytes');
|
||||
// Must not be archived.
|
||||
write('runtime-reader.json', '{}'); write('scraper/runtime.json', '{}'); write('sentinel/runtime.json', '{}');
|
||||
write('scraper/daemon.lock', '1'); write('scraper/supervisor.lock', '1'); write('reader.pid', '1'); write('db.json.tmp', '{}');
|
||||
fs.mkdirSync(target);
|
||||
return { dir, data, target, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) };
|
||||
}
|
||||
|
||||
function fakeScraper({ paused = false, busyPolls = 0, failResume = false } = {}) {
|
||||
const calls = [];
|
||||
let polls = 0;
|
||||
return {
|
||||
calls,
|
||||
get paused() { return paused; },
|
||||
async state() { polls++; return { paused, status: paused && polls <= busyPolls ? 'pausing' : paused ? 'paused' : 'downloading', active: polls <= busyPolls ? [{ source: 'x' }] : [] }; },
|
||||
async control(action) {
|
||||
calls.push(action);
|
||||
if (action === 'resume' && failResume) throw new Error('scraper gone');
|
||||
paused = action === 'pause';
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const service = (f, options = {}) => new BackupService({ dataRoot: f.data, backupRoot: f.target, log: quiet, pollMs: 5, resumeRetryMs: 1,
|
||||
snapshotDb: () => JSON.stringify({ users: { admin: {} }, live: true }), ...options }).load();
|
||||
|
||||
test('settings validation and schedule computation', () => {
|
||||
assert.deepEqual(validateSettings({ enabled: true, time: '04:15', retention: { mode: 'days', value: 30 } }),
|
||||
{ enabled: true, time: '04:15', retention: { mode: 'days', value: 30 } });
|
||||
assert.throws(() => validateSettings({ time: '24:00' }), /HH:MM/);
|
||||
assert.throws(() => validateSettings({ retention: { mode: 'count', value: 0 } }), /Keep/);
|
||||
assert.throws(() => validateSettings({ retention: { mode: 'weeks', value: 2 } }), /count/);
|
||||
assert.throws(() => validateSettings({ enabled: 'yes' }), /enabled/);
|
||||
const now = new Date(2026, 9, 22, 3, 0).getTime();
|
||||
assert.equal(nextOccurrence('04:00', now), new Date(2026, 9, 22, 4, 0).getTime());
|
||||
assert.equal(nextOccurrence('03:00', now), new Date(2026, 9, 23, 3, 0).getTime());
|
||||
});
|
||||
|
||||
test('a backup pauses a running queue, waits for idle, archives a consistent DATA snapshot and resumes', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const scraper = fakeScraper({ busyPolls: 3 });
|
||||
const backup = service(f, { scraper });
|
||||
const result = await backup.run('manual');
|
||||
assert.equal(result.error, undefined, result.error);
|
||||
assert.deepEqual(scraper.calls, ['pause', 'resume']);
|
||||
assert.equal(result.queueWasPaused, false); assert.equal(result.queueResumed, true);
|
||||
assert.match(result.file, /^circus-\d{8}-\d{6}\.tar\.gz$/);
|
||||
assert.deepEqual(fs.readdirSync(f.target), [result.file]);
|
||||
const entries = await readArchive(path.join(f.target, result.file), { contents: true });
|
||||
const byName = new Map(entries.map(e => [e.name, e]));
|
||||
const text = name => byName.get(name).data.toString('utf8');
|
||||
// db.json comes from memory, not from the stale file on disk.
|
||||
assert.deepEqual(JSON.parse(text('db.json')), { users: { admin: {} }, live: true });
|
||||
// The archived queue is paused; the live one is untouched by the snapshot.
|
||||
const queue = JSON.parse(text('scraper/queue.json'));
|
||||
assert.equal(queue.paused, true); assert.deepEqual(queue.queue, [{ id: 'a' }]);
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(f.data, 'scraper/queue.json'), 'utf8')).paused, false);
|
||||
for (const name of ['scan-cache.json', 'library-index.json', 'catalog.json', 'scraper/events.log', 'sentinel/state.json', LONG, '.circus-backup.json'])
|
||||
assert.ok(byName.has(name), `${name} missing`);
|
||||
assert.equal(text(LONG), 'png-bytes');
|
||||
for (const name of ['runtime-reader.json', 'scraper/runtime.json', 'sentinel/runtime.json', 'scraper/daemon.lock', 'scraper/supervisor.lock', 'reader.pid', 'db.json.tmp'])
|
||||
assert.ok(!byName.has(name), `${name} must be excluded`);
|
||||
assert.ok(byName.get('scraper/').type === '5');
|
||||
// The persisted state records the result but never a pending marker inside the archive.
|
||||
assert.equal(JSON.parse(text('backup.json')).pending, undefined);
|
||||
assert.equal(backup.publicState().last.file, result.file);
|
||||
assert.equal(backup.publicState().archives.count, 1);
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('an already paused queue stays paused and is not resumed', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const scraper = fakeScraper({ paused: true });
|
||||
const result = await service(f, { scraper }).run('schedule');
|
||||
assert.equal(result.error, undefined);
|
||||
assert.deepEqual(scraper.calls, []);
|
||||
assert.equal(result.queueWasPaused, true); assert.equal(scraper.paused, true);
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('a scraper that never becomes idle aborts the backup, leaves no archive and resumes the queue', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const scraper = fakeScraper({ busyPolls: Infinity });
|
||||
const result = await service(f, { scraper, idleTimeoutMs: 40 }).run('manual');
|
||||
assert.match(result.error, /did not become idle/);
|
||||
assert.deepEqual(scraper.calls, ['pause', 'resume']);
|
||||
assert.deepEqual(fs.readdirSync(f.target), []);
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('an unreachable scraper does not block the backup and is reported', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const scraper = { async state() { throw new Error('connect ECONNREFUSED'); }, async control() { throw new Error('unused'); } };
|
||||
const result = await service(f, { scraper }).run('manual');
|
||||
assert.equal(result.error, undefined);
|
||||
assert.match(result.warnings[0], /unreachable/);
|
||||
const queue = (await readArchive(path.join(f.target, result.file), { contents: true })).find(e => e.name === 'scraper/queue.json');
|
||||
assert.equal(JSON.parse(queue.data).paused, true);
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('a failed resume is surfaced as a warning', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const result = await service(f, { scraper: fakeScraper({ failResume: true }) }).run('manual');
|
||||
assert.equal(result.error, undefined);
|
||||
assert.match(result.warnings.join(' '), /could not be resumed/);
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('retention keeps the newest archives and never touches other files', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const old = ['circus-20260101-030000.tar.gz', 'circus-20260102-030000.tar.gz', 'circus-20260103-030000.tar.gz'];
|
||||
const foreign = ['circus-20250101-0300.tar.zst', 'notes.txt', 'circus-20250101-030000.tar.gz.bak'];
|
||||
for (const name of [...old, ...foreign]) fs.writeFileSync(path.join(f.target, name), 'x');
|
||||
fs.mkdirSync(path.join(f.target, 'circus-20250101-030000.tar.gz-dir'));
|
||||
fs.writeFileSync(path.join(f.target, 'circus-20250101-030000.tar.gz.part'), 'stale');
|
||||
const backup = service(f);
|
||||
backup.updateSettings({ retention: { mode: 'count', value: 2 } });
|
||||
const result = await backup.run('manual');
|
||||
assert.equal(result.error, undefined); assert.equal(result.deleted, 2);
|
||||
assert.deepEqual(fs.readdirSync(f.target).sort(), [...foreign, 'circus-20250101-030000.tar.gz-dir', old[2], result.file].sort());
|
||||
// Days mode works from the timestamp in the name.
|
||||
backup.updateSettings({ retention: { mode: 'days', value: 1 } });
|
||||
const second = await backup.run('manual');
|
||||
assert.ok(!fs.existsSync(path.join(f.target, old[2])));
|
||||
assert.ok(fs.existsSync(path.join(f.target, second.file)));
|
||||
for (const name of foreign) assert.ok(fs.existsSync(path.join(f.target, name)));
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('backups are off without BACKUP, and a missing folder is reported, never created', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const off = service(f, { backupRoot: '' });
|
||||
await assert.rejects(off.run('manual'), /Backups are off/);
|
||||
assert.equal(off.publicState().configured, false);
|
||||
const missing = path.join(f.dir, 'not-mounted');
|
||||
const result = await service(f, { backupRoot: missing }).run('manual');
|
||||
assert.match(result.error, /not usable/);
|
||||
assert.ok(!fs.existsSync(missing));
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('only one run at a time', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const backup = service(f, { scraper: fakeScraper({ busyPolls: 5 }) });
|
||||
const first = backup.run('manual');
|
||||
await assert.rejects(backup.run('manual'), /already running/);
|
||||
assert.equal((await first).error, undefined);
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('the schedule runs once per slot, catches up a recent miss and skips stale slots', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
let now = new Date(2026, 9, 22, 2, 0).getTime();
|
||||
const backup = service(f, { now: () => now });
|
||||
backup.updateSettings({ enabled: true, time: '03:30' });
|
||||
assert.equal(backup.state.nextAt, new Date(2026, 9, 22, 3, 30).getTime());
|
||||
assert.equal(await backup.tick(), null);
|
||||
now = new Date(2026, 9, 22, 9, 0).getTime(); // reader was down at 03:30
|
||||
const result = await backup.tick();
|
||||
assert.equal(result.trigger, 'schedule'); assert.equal(result.error, undefined);
|
||||
assert.equal(backup.state.nextAt, new Date(2026, 9, 23, 3, 30).getTime());
|
||||
assert.equal(await backup.tick(), null);
|
||||
now = new Date(2026, 9, 30, 9, 0).getTime(); // e.g. an old backup.json was restored
|
||||
assert.equal(await backup.tick(), null);
|
||||
assert.equal(backup.state.nextAt, new Date(2026, 9, 31, 3, 30).getTime());
|
||||
assert.equal(fs.readdirSync(f.target).length, 1);
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('a run interrupted by a restart resumes the queue it paused', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
fs.writeFileSync(path.join(f.data, 'backup.json'), JSON.stringify({ version: 1, pending: { startedAt: 1, trigger: 'schedule', pausedByUs: true } }));
|
||||
const scraper = fakeScraper({ paused: true });
|
||||
const backup = service(f, { scraper });
|
||||
await backup.recoverInterrupted();
|
||||
assert.deepEqual(scraper.calls, ['resume']);
|
||||
assert.match(backup.state.last.error, /Interrupted/);
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(f.data, 'backup.json'), 'utf8')).pending, null);
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
test('system tar extracts the archive', { skip: spawnSync('tar', ['--version']).error ? 'no tar binary' : false }, async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const result = await service(f).run('manual');
|
||||
const out = path.join(f.dir, 'restored'); fs.mkdirSync(out);
|
||||
const tar = spawnSync('tar', ['-xzf', path.join(f.target, result.file), '-C', out], { encoding: 'utf8' });
|
||||
assert.equal(tar.status, 0, tar.stderr);
|
||||
assert.equal(fs.readFileSync(path.join(out, LONG), 'utf8'), 'png-bytes');
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(out, 'scraper/queue.json'), 'utf8')).paused, true);
|
||||
assert.ok(!fs.existsSync(path.join(out, 'scraper/daemon.lock')));
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
async function port() {
|
||||
const server = net.createServer();
|
||||
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
||||
const result = server.address().port; await new Promise(resolve => server.close(resolve)); return result;
|
||||
}
|
||||
|
||||
test('reader and scraper: admin backup API pauses and resumes the real queue', { timeout: 30000 }, async () => {
|
||||
const f = fixture(), children = [], output = [];
|
||||
try {
|
||||
const library = path.join(f.dir, 'library'); fs.mkdirSync(library);
|
||||
fs.writeFileSync(path.join(f.data, 'scraper', 'queue.json'), JSON.stringify({ paused: false, queue: [], settings: { autoUpdateEnabled: false } }));
|
||||
for (const name of ['runtime.json', 'daemon.lock', 'supervisor.lock']) fs.rmSync(path.join(f.data, 'scraper', name), { force: true });
|
||||
for (const name of ['db.json', 'catalog.json', 'library-index.json', 'scan-cache.json']) fs.rmSync(path.join(f.data, name));
|
||||
const [rp, sp] = await Promise.all([port(), port()]);
|
||||
const base = `http://127.0.0.1:${rp}`, token = 'backup-test-only';
|
||||
const env = { ...process.env, DATA: f.data, LIBRARY: library, SCRAPER_DATA: path.join(f.data, 'scraper'), BACKUP: f.target,
|
||||
PORT: String(rp), SCRAPER_PORT: String(sp), SCRAPER_HOST: '127.0.0.1', SCRAPER_URL: `http://127.0.0.1:${sp}`, SENTINEL_URL: 'http://127.0.0.1:9',
|
||||
SCRAPER_TOKEN: token, SCRAPER_TOKEN_FILE: '', SCAN_INTERVAL: '0' };
|
||||
for (const script of ['server.js', 'scraper/daemon.js']) {
|
||||
const child = spawn(process.execPath, [script], { cwd: root, env, windowsHide: true }); children.push(child);
|
||||
for (const stream of [child.stdout, child.stderr]) stream.on('data', chunk => output.push(chunk.toString()));
|
||||
}
|
||||
for (const url of [base, `http://127.0.0.1:${sp}`]) {
|
||||
let ready = false;
|
||||
for (let n = 0; n < 80 && !ready; n++) { ready = await fetch(url + '/health').then(r => r.ok).catch(() => false); if (!ready) await delay(100); }
|
||||
assert.ok(ready, output.join(''));
|
||||
}
|
||||
const signup = await fetch(base + '/api/signup', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'admin', password: 'local-test-password' }) });
|
||||
const cookie = signup.headers.get('set-cookie').split(';')[0];
|
||||
const request = (endpoint, body) => fetch(base + endpoint, { headers: { cookie, 'content-type': 'application/json' }, ...(body ? { method: 'POST', body: JSON.stringify(body) } : {}) });
|
||||
assert.equal((await request('/api/admin/backup', { command: 'settings', settings: { time: '25:00' } })).status, 400);
|
||||
const saved = await (await request('/api/admin/backup', { command: 'settings', settings: { enabled: true, time: '04:00', retention: { mode: 'count', value: 3 } } })).json();
|
||||
assert.equal(saved.settings.retention.value, 3); assert.ok(saved.nextAt > Date.now());
|
||||
assert.equal((await request('/api/admin/backup', { command: 'run' })).status, 200);
|
||||
let state;
|
||||
for (let n = 0; n < 100; n++) { state = await (await request('/api/admin/backup')).json(); if (state.last && !state.running) break; await delay(100); }
|
||||
assert.equal(state.last.error, undefined, state.last.error);
|
||||
assert.equal(state.last.queueWasPaused, false); assert.equal(state.last.queueResumed, true);
|
||||
const scraperState = await (await request('/api/admin/scraper')).json();
|
||||
assert.equal(scraperState.paused, false);
|
||||
const entries = await readArchive(path.join(f.target, state.last.file), { contents: true });
|
||||
const db = JSON.parse(entries.find(e => e.name === 'db.json').data);
|
||||
assert.ok(db.users.admin);
|
||||
assert.equal(JSON.parse(entries.find(e => e.name === 'scraper/queue.json').data).paused, true);
|
||||
assert.ok((await (await request('/api/admin/health')).json()).backup.last.file);
|
||||
} finally {
|
||||
await Promise.all(children.map(child => new Promise(resolve => { if (child.exitCode !== null) return resolve(); child.once('exit', resolve); child.kill(); })));
|
||||
f.cleanup();
|
||||
}
|
||||
});
|
||||
@@ -2,7 +2,7 @@
|
||||
'use strict';
|
||||
const fs = require('fs'), path = require('path'), crypto = require('crypto');
|
||||
const root = path.resolve(__dirname, '..');
|
||||
for (const relative of ['data', 'data/scraper', 'data/sentinel', 'library', '.secrets']) fs.mkdirSync(path.join(root, relative), { recursive: true });
|
||||
for (const relative of ['data', 'data/scraper', 'data/sentinel', 'library', 'backup', '.secrets']) fs.mkdirSync(path.join(root, relative), { recursive: true });
|
||||
const file = path.join(root, '.secrets', 'scraper_token');
|
||||
if (!fs.existsSync(file)) fs.writeFileSync(file, crypto.randomBytes(32).toString('hex') + '\n', { flag: 'wx', mode: 0o600 });
|
||||
if (!fs.existsSync(path.join(root, '.env'))) fs.copyFileSync(path.join(root, '.env.example'), path.join(root, '.env'));
|
||||
|
||||
+12
-4
@@ -6,10 +6,10 @@ const assert = require('assert/strict'), { spawn } = require('child_process');
|
||||
const { Canvas } = require('./png');
|
||||
const root = path.resolve(__dirname, '..'), dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mc-compose-'));
|
||||
const project = 'mc-qa-' + crypto.randomBytes(3).toString('hex'), port = process.env.QA_PORT || '8180';
|
||||
const library = path.join(dir, 'library'), data = path.join(dir, 'data'), tokenFile = path.join(dir, 'token');
|
||||
const library = path.join(dir, 'library'), data = path.join(dir, 'data'), backupDir = path.join(dir, 'backup'), tokenFile = path.join(dir, 'token');
|
||||
const directionQa = process.argv.includes('--direction-ui');
|
||||
const peerGatewayQa = process.argv.includes('--peer-gateway');
|
||||
fs.mkdirSync(path.join(data, 'scraper'), { recursive: true }); fs.mkdirSync(path.join(data, 'sentinel'));
|
||||
fs.mkdirSync(path.join(data, 'scraper'), { recursive: true }); fs.mkdirSync(path.join(data, 'sentinel')); fs.mkdirSync(backupDir);
|
||||
if (peerGatewayQa) {
|
||||
const peerData = path.join(data, 'peering');
|
||||
const peerUsage = path.join(data, 'peering-usage');
|
||||
@@ -48,7 +48,7 @@ for (const [index, name] of ['Moon Garden', 'Before the Rain', 'Iron Harbor'].en
|
||||
status: index ? 'completed' : 'ongoing', tags: ['Fantasy', 'Demo'] }));
|
||||
}
|
||||
const env = { ...process.env, LIBRARY_PATH: library, DATA_PATH: data, SCRAPER_DATA_PATH: path.join(data, 'scraper'), SENTINEL_DATA_PATH: path.join(data, 'sentinel'),
|
||||
SCRAPER_TOKEN_PATH: tokenFile, READER_PORT: port, READER_BIND: '127.0.0.1', COOKIE_SECURE: '0', APP_IMAGE: process.env.APP_IMAGE || 'mangacircus:local',
|
||||
SCRAPER_TOKEN_PATH: tokenFile, BACKUP_PATH: backupDir, READER_PORT: port, READER_BIND: '127.0.0.1', COOKIE_SECURE: '0', APP_IMAGE: process.env.APP_IMAGE || 'mangacircus:local',
|
||||
READER_IMAGE: process.env.APP_IMAGE || 'mangacircus:local',
|
||||
...(readingGateQa ? { READING_WINDOWS_JSON: '{"0":[],"1":[],"2":[],"3":[],"4":[],"5":[],"6":[]}',
|
||||
READING_TIMEZONE: 'Europe/Berlin', READING_QUIZ_UNLOCK_MINUTES: '60', READING_QUIZ_CATALOG_PATH: '/data/reading-quiz.json' }
|
||||
@@ -93,6 +93,14 @@ let keep = false;
|
||||
let inventory;
|
||||
await until(async () => { inventory = await (await api('/api/library')).json(); return inventory.mangas?.length === 3; }, 'Fixture inventory missing');
|
||||
const byTitle = title => inventory.mangas.find(m => m.title === title).id;
|
||||
// Built-in backup through the real control API: archive written, paused queue left paused.
|
||||
const queueBefore = JSON.parse(fs.readFileSync(path.join(data, 'scraper', 'queue.json'), 'utf8'));
|
||||
assert.equal((await api('/api/admin/backup', { command: 'run' })).status, 200);
|
||||
let backupState;
|
||||
await until(async () => { backupState = await (await api('/api/admin/backup')).json(); return backupState.last && !backupState.running; }, 'Backup did not finish');
|
||||
assert.equal(backupState.last.error, undefined, backupState.last.error);
|
||||
assert.ok(fs.existsSync(path.join(backupDir, backupState.last.file)), 'Backup archive missing on the host');
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(data, 'scraper', 'queue.json'), 'utf8')).paused, queueBefore.paused);
|
||||
if (readingGateQa) {
|
||||
const manga = byTitle('Moon Garden'), other = byTitle('Before the Rain');
|
||||
const chapter = (await (await api('/api/manga?m=' + encodeURIComponent(manga))).json()).chapters[0].id;
|
||||
@@ -243,7 +251,7 @@ let keep = false;
|
||||
keep = process.argv.includes('--keep');
|
||||
if (keep) {
|
||||
const envFile = path.join(dir, 'qa.env');
|
||||
fs.writeFileSync(envFile, ['LIBRARY_PATH', 'DATA_PATH', 'SCRAPER_DATA_PATH', 'SENTINEL_DATA_PATH', 'SCRAPER_TOKEN_PATH', 'READER_PORT', 'READER_BIND', 'COOKIE_SECURE', 'APP_IMAGE',
|
||||
fs.writeFileSync(envFile, ['LIBRARY_PATH', 'DATA_PATH', 'SCRAPER_DATA_PATH', 'SENTINEL_DATA_PATH', 'SCRAPER_TOKEN_PATH', 'BACKUP_PATH', 'READER_PORT', 'READER_BIND', 'COOKIE_SECURE', 'APP_IMAGE',
|
||||
...(readingGateQa ? ['READING_WINDOWS_JSON', 'READING_TIMEZONE', 'READING_QUIZ_UNLOCK_MINUTES', 'READING_QUIZ_CATALOG_PATH'] : [])]
|
||||
.map(k => `${k}=${k === 'READING_WINDOWS_JSON' ? `'${env[k]}'` : env[k].replaceAll('\\', '/')}`).join('\n') + '\n');
|
||||
console.log(JSON.stringify({ project, url, envFile, fixture: dir }));
|
||||
|
||||
Reference in new issue
Block a user