65 lines
4.2 KiB
JavaScript
65 lines
4.2 KiB
JavaScript
'use strict';
|
|
const fs = require('fs'), path = require('path'), { spawn } = require('child_process');
|
|
const SPECS = {
|
|
reader: { script: 'server.js', port: () => process.env.PORT || 8080, file: () => path.join(process.env.DATA || 'data', 'runtime-reader.json') },
|
|
scraper: { script: 'scraper/daemon.js', port: () => process.env.SCRAPER_PORT || 8091, file: () => path.join(process.env.SCRAPER_DATA || path.join(process.env.DATA || 'data', 'scraper'), 'runtime.json') },
|
|
sentinel: { script: 'sentinel/daemon.js', port: () => process.env.SENTINEL_PORT || 8092, file: () => path.join(process.env.SENTINEL_DATA || path.join(process.env.DATA || 'data', 'sentinel'), 'runtime.json') },
|
|
};
|
|
class Supervisor {
|
|
constructor(name, options = {}) {
|
|
this.name = name; this.spec = SPECS[name]; if (!this.spec) throw new Error('Unknown service');
|
|
this.root = options.root || path.resolve(__dirname, '..'); this.child = null; this.stopped = false; this.timer = null;
|
|
this.healthFailures = 0; this.restarting = false; this.startedAt = 0;
|
|
this.spawn = options.spawn || spawn; this.retryBaseMs = options.retryBaseMs ?? 5000;
|
|
this.file = options.file || path.resolve(this.root, this.spec.file()); fs.mkdirSync(path.dirname(this.file), { recursive: true });
|
|
this.state = { failures: [], status: 'starting' };
|
|
if (fs.existsSync(this.file)) this.state = JSON.parse(fs.readFileSync(this.file, 'utf8'));
|
|
}
|
|
save() { fs.writeFileSync(this.file + '.tmp', JSON.stringify(this.state, null, 2) + '\n'); fs.renameSync(this.file + '.tmp', this.file); }
|
|
log(event, extra = {}) { console.log(JSON.stringify({ service: 'supervisor', target: this.name, event, ...extra })); }
|
|
start() {
|
|
if (this.stopped) return;
|
|
this.state.failures = this.state.failures.filter(at => Date.now() - at < 3600000);
|
|
if (this.state.failures.length >= 3) { this.state.status = 'parked'; this.save(); this.log('repair-required', { reason: 'Three failures within one hour' }); return; }
|
|
this.child = this.spawn(process.execPath, [this.spec.script], { cwd: this.root, env: process.env, stdio: 'inherit', windowsHide: true });
|
|
this.state.status = 'running'; this.state.pid = this.child.pid; this.startedAt = Date.now(); this.healthFailures = 0; this.restarting = false; this.save();
|
|
this.log('started', { pid: this.child.pid });
|
|
const child = this.child;
|
|
child.on('error', e => this.log('spawn-error', { message: e.message }));
|
|
child.once('close', (code, signal) => {
|
|
if (this.child !== child) return;
|
|
this.child = null; clearTimeout(this.killTimer);
|
|
if (this.stopped) return;
|
|
clearTimeout(this.timer);
|
|
this.state.failures.push(Date.now()); this.state.status = 'retry_wait'; this.state.lastExit = { at: new Date().toISOString(), code, signal }; this.save();
|
|
this.log('exited', { code, signal });
|
|
this.timer = setTimeout(() => this.start(), Math.min(60000, this.retryBaseMs * 2 ** (this.state.failures.length - 1)));
|
|
});
|
|
this.timer = setTimeout(() => this.check(), 30000);
|
|
}
|
|
async check() {
|
|
if (this.stopped || !this.child) return;
|
|
const observedChild = this.child;
|
|
const healthy = await fetch(`http://127.0.0.1:${this.spec.port()}/health`, { signal: AbortSignal.timeout(5000) }).then(async r => r.ok && (await r.json()).ok !== false).catch(() => false);
|
|
if (this.stopped || this.child !== observedChild) return;
|
|
this.healthFailures = healthy ? 0 : this.healthFailures + 1;
|
|
if (this.healthFailures >= 3 && Date.now() - this.startedAt > 90000 && !this.restarting) {
|
|
this.restarting = true; this.log('liveness-recovery');
|
|
const child = this.child; child.kill('SIGTERM');
|
|
this.killTimer = setTimeout(() => { if (this.child === child) child.kill('SIGKILL'); }, 65000);
|
|
}
|
|
if (!this.stopped) this.timer = setTimeout(() => this.check(), 30000);
|
|
}
|
|
async close() {
|
|
this.stopped = true; clearTimeout(this.timer); clearTimeout(this.killTimer);
|
|
if (!this.child) return;
|
|
const child = this.child;
|
|
await new Promise(resolve => {
|
|
const timer = setTimeout(() => child.kill('SIGKILL'), 65000);
|
|
child.once('exit', () => { clearTimeout(timer); resolve(); }); child.kill('SIGTERM');
|
|
});
|
|
this.state.status = 'stopped'; this.save();
|
|
}
|
|
}
|
|
module.exports = { Supervisor, SPECS };
|