feat: show why units failed, with their last journal lines
Failed jobs keep their kind: units that broke or timed out get a failure panel entry with the reason systemd reports (exit status, signal, timeout, start limit, status text) and their last journal lines read through libsystemd. Units that failed only through a dependency are dimmed and left out of the panel, and units that crash after starting count as failed. Restructure the repository around this: - move the library and binary into bootvine/ and the scripted demos into their own unpublished scripted-demos crate, with their VHS tapes - move the crane builds and NixOS module into nix/ - group each sample VM with its check under vm-tests/, adding text-console, scripted-initrd, encrypted-volume and failures machines and a vm-failures check - add docs/guide.md walking through the screen with every recording, keep one recording in the README, and re-render all recordings
This commit is contained in:
1 parent
f2544a964a
commit
7e95cbd8e9
96 files changed
+2027
-462
No files matched your search
@@ -0,0 +1,49 @@
|
||||
{ machine }:
|
||||
{
|
||||
name = "bootvine-failures";
|
||||
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
holdUntilReleased =
|
||||
marker: "${pkgs.bash}/bin/bash -c 'until [ -e /run/${marker} ]; do sleep 0.1; done'";
|
||||
in
|
||||
{
|
||||
imports = [ machine ];
|
||||
bootvineSample.startFailuresByHand = true;
|
||||
testing.initrdBackdoor = true;
|
||||
|
||||
boot.initrd.systemd = {
|
||||
storePaths = [ "${pkgs.bash}/bin/bash" ];
|
||||
services.initrd-failures-hold = {
|
||||
description = "Initrd held for the check";
|
||||
wantedBy = [ "initrd.target" ];
|
||||
before = [ "initrd.target" ];
|
||||
unitConfig.DefaultDependencies = false;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = holdUntilReleased "release-initrd-failures-hold";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.boot-failures-hold = {
|
||||
description = "Boot held for the check";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = holdUntilReleased "release-boot-failures-hold";
|
||||
};
|
||||
};
|
||||
|
||||
virtualisation = {
|
||||
memorySize = 2048;
|
||||
qemu.options = [
|
||||
"-vga none"
|
||||
"-device virtio-vga,xres=1280,yres=800"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
def start_failing_unit(unit):
|
||||
machine.succeed(f"systemctl start --no-block {unit}")
|
||||
|
||||
|
||||
def unit_result(unit):
|
||||
return machine.succeed(f"systemctl show -p Result --value {unit}").strip()
|
||||
|
||||
|
||||
def failure_panel():
|
||||
lines = screen().splitlines()
|
||||
start = next(index for index, line in enumerate(lines) if " Failures " in line)
|
||||
return "\n".join(lines[start:])
|
||||
|
||||
|
||||
machine.start()
|
||||
|
||||
wait_for_screen("Initrd", "Initrd held for the check")
|
||||
start_failing_unit("initrd-nic-firmware-load")
|
||||
wait_for_screen("Failures", "initrd-nic-firmware-load exited with status 1", "rtl8168h-2.fw not found")
|
||||
assert_bootvine_in_front()
|
||||
assert_text_visible(font_has_512_glyphs=True)
|
||||
machine.succeed("touch /run/release-initrd-failures-hold")
|
||||
machine.switch_root()
|
||||
|
||||
wait_for_screen("Stage 2", "Boot held for the check")
|
||||
wait_for_screen("initrd-nic-firmware-load exited with status 1")
|
||||
|
||||
start_failing_unit("failure-report")
|
||||
wait_for_screen("app-config-check exited with status 2", "missing key 'listen'")
|
||||
retry(lambda _: unit_result("failure-report") != "", timeout_seconds=10)
|
||||
assert "× failure-report" not in failure_panel(), f"a dependency failure reached the panel:\n{screen()}"
|
||||
assert "failure-report" in screen(), f"the dependency failure left the tree:\n{screen()}"
|
||||
|
||||
start_failing_unit("worker-pool")
|
||||
wait_for_screen("worker-pool", "SIGSEGV", "starting worker pool with 4 threads")
|
||||
|
||||
start_failing_unit("license-server-wait")
|
||||
wait_for_screen("license-server-wait timed out", "waiting for license server at 10.0.0.9:27000")
|
||||
|
||||
start_failing_unit("metrics-agent")
|
||||
wait_for_screen("metrics-agent exited with status 203")
|
||||
|
||||
start_failing_unit("http-listener")
|
||||
wait_for_screen("http-listener restarted too often", "address already in use")
|
||||
|
||||
start_failing_unit("database-client")
|
||||
wait_for_screen("database-client exited with status 1: Cannot reach database", "connection refused")
|
||||
|
||||
start_failing_unit("upstream-uplink")
|
||||
wait_for_screen("upstream-uplink exited with status 1", "lost connection to upstream, giving up")
|
||||
|
||||
wait_for_screen("× 9 failed", "connection refused")
|
||||
assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}"
|
||||
print(screen())
|
||||
machine.screenshot("failures")
|
||||
|
||||
machine.succeed("touch /run/release-boot-failures-hold")
|
||||
retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60)
|
||||
print(bootvine_journal())
|
||||
@@ -0,0 +1,148 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
startedByHand = config.bootvineSample.startFailuresByHand;
|
||||
wantedAtBoot = target: lib.optional (!startedByHand) target;
|
||||
|
||||
failingService =
|
||||
{
|
||||
description,
|
||||
script,
|
||||
serviceConfig ? { },
|
||||
unitConfig ? { },
|
||||
extraAttributes ? { },
|
||||
}:
|
||||
{
|
||||
inherit description unitConfig;
|
||||
wantedBy = wantedAtBoot "multi-user.target";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = pkgs.writeShellScript "failing-service" script;
|
||||
}
|
||||
// serviceConfig;
|
||||
}
|
||||
// extraAttributes;
|
||||
|
||||
initrdFirmwareScript = pkgs.writeShellScript "initrd-nic-firmware-load" ''
|
||||
echo "loading firmware for enp1s0 (r8169)"
|
||||
echo "firmware: rtl_nic/rtl8168h-2.fw not found" >&2
|
||||
exit 1
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.bootvineSample.startFailuresByHand = lib.mkEnableOption ''
|
||||
leaving the failing sample units out of the boot targets, so a check can
|
||||
start them one at a time
|
||||
'';
|
||||
|
||||
config = {
|
||||
boot.initrd.systemd = {
|
||||
storePaths = [
|
||||
initrdFirmwareScript
|
||||
"${pkgs.bash}/bin/bash"
|
||||
];
|
||||
services.initrd-nic-firmware-load = {
|
||||
description = "Network card firmware";
|
||||
wantedBy = wantedAtBoot "initrd.target";
|
||||
before = [ "initrd.target" ];
|
||||
unitConfig.DefaultDependencies = false;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${pkgs.bash}/bin/bash ${initrdFirmwareScript}";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services = {
|
||||
app-config-check = failingService {
|
||||
description = "Application configuration check";
|
||||
script = ''
|
||||
echo "reading /etc/app/app.toml"
|
||||
echo "config: /etc/app/app.toml: missing key 'listen'" >&2
|
||||
exit 2
|
||||
'';
|
||||
};
|
||||
|
||||
worker-pool = failingService {
|
||||
description = "Worker pool";
|
||||
script = ''
|
||||
echo "starting worker pool with 4 threads"
|
||||
kill -SEGV $$
|
||||
'';
|
||||
};
|
||||
|
||||
license-server-wait = failingService {
|
||||
description = "License server check";
|
||||
script = ''
|
||||
echo "waiting for license server at 10.0.0.9:27000"
|
||||
sleep infinity
|
||||
'';
|
||||
serviceConfig.TimeoutStartSec = 4;
|
||||
};
|
||||
|
||||
metrics-agent = {
|
||||
description = "Metrics agent";
|
||||
wantedBy = wantedAtBoot "multi-user.target";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "/usr/bin/metrics-agent --listen :9100";
|
||||
};
|
||||
};
|
||||
|
||||
http-listener = failingService {
|
||||
description = "HTTP listener";
|
||||
script = ''
|
||||
echo "bind 0.0.0.0:8080: address already in use" >&2
|
||||
exit 1
|
||||
'';
|
||||
serviceConfig = {
|
||||
Restart = "on-failure";
|
||||
RestartSec = "200ms";
|
||||
};
|
||||
unitConfig = {
|
||||
StartLimitBurst = 3;
|
||||
StartLimitIntervalSec = 30;
|
||||
};
|
||||
};
|
||||
|
||||
database-client = failingService {
|
||||
description = "Database client";
|
||||
script = ''
|
||||
systemd-notify --status="Cannot reach database at db.internal:5432"
|
||||
echo "connect db.internal:5432: connection refused" >&2
|
||||
exit 1
|
||||
'';
|
||||
serviceConfig = {
|
||||
Type = "notify";
|
||||
NotifyAccess = "all";
|
||||
};
|
||||
};
|
||||
|
||||
upstream-uplink = failingService {
|
||||
description = "Upstream uplink";
|
||||
script = ''
|
||||
echo "connected to upstream"
|
||||
sleep 3
|
||||
echo "lost connection to upstream, giving up" >&2
|
||||
exit 1
|
||||
'';
|
||||
serviceConfig.Type = "simple";
|
||||
};
|
||||
|
||||
failure-report = failingService {
|
||||
description = "Failure report";
|
||||
script = ''
|
||||
echo "all checks passed"
|
||||
'';
|
||||
extraAttributes = {
|
||||
requires = [ "app-config-check.service" ];
|
||||
after = [ "app-config-check.service" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user