feat: show why units failed, with their last journal lines

Failed jobs keep their kind: units that broke or timed out get a failure
panel entry with the reason systemd reports (exit status, signal, timeout,
start limit, status text) and their last journal lines read through
libsystemd. Units that failed only through a dependency are dimmed and left
out of the panel, and units that crash after starting count as failed.

Restructure the repository around this:
- move the library and binary into bootvine/ and the scripted demos into
  their own unpublished scripted-demos crate, with their VHS tapes
- move the crane builds and NixOS module into nix/
- group each sample VM with its check under vm-tests/, adding text-console,
  scripted-initrd, encrypted-volume and failures machines and a vm-failures
  check
- add docs/guide.md walking through the screen with every recording, keep
  one recording in the README, and re-render all recordings
This commit is contained in:
Willem Vanhulle committed 2026-10-30 14:59:10 +01:00
1 parent f2544a964a
commit 7e95cbd8e9
96 files changed
+2027 -462

No files matched your search

+49
View File
@@ -0,0 +1,49 @@
{ machine }:
{
name = "bootvine-failures";
nodes.machine =
{ pkgs, ... }:
let
holdUntilReleased =
marker: "${pkgs.bash}/bin/bash -c 'until [ -e /run/${marker} ]; do sleep 0.1; done'";
in
{
imports = [ machine ];
bootvineSample.startFailuresByHand = true;
testing.initrdBackdoor = true;
boot.initrd.systemd = {
storePaths = [ "${pkgs.bash}/bin/bash" ];
services.initrd-failures-hold = {
description = "Initrd held for the check";
wantedBy = [ "initrd.target" ];
before = [ "initrd.target" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
ExecStart = holdUntilReleased "release-initrd-failures-hold";
};
};
};
systemd.services.boot-failures-hold = {
description = "Boot held for the check";
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = holdUntilReleased "release-boot-failures-hold";
};
};
virtualisation = {
memorySize = 2048;
qemu.options = [
"-vga none"
"-device virtio-vga,xres=1280,yres=800"
];
};
};
testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py;
}
+59
View File
@@ -0,0 +1,59 @@
def start_failing_unit(unit):
machine.succeed(f"systemctl start --no-block {unit}")
def unit_result(unit):
return machine.succeed(f"systemctl show -p Result --value {unit}").strip()
def failure_panel():
lines = screen().splitlines()
start = next(index for index, line in enumerate(lines) if " Failures " in line)
return "\n".join(lines[start:])
machine.start()
wait_for_screen("Initrd", "Initrd held for the check")
start_failing_unit("initrd-nic-firmware-load")
wait_for_screen("Failures", "initrd-nic-firmware-load exited with status 1", "rtl8168h-2.fw not found")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
machine.succeed("touch /run/release-initrd-failures-hold")
machine.switch_root()
wait_for_screen("Stage 2", "Boot held for the check")
wait_for_screen("initrd-nic-firmware-load exited with status 1")
start_failing_unit("failure-report")
wait_for_screen("app-config-check exited with status 2", "missing key 'listen'")
retry(lambda _: unit_result("failure-report") != "", timeout_seconds=10)
assert "× failure-report" not in failure_panel(), f"a dependency failure reached the panel:\n{screen()}"
assert "failure-report" in screen(), f"the dependency failure left the tree:\n{screen()}"
start_failing_unit("worker-pool")
wait_for_screen("worker-pool", "SIGSEGV", "starting worker pool with 4 threads")
start_failing_unit("license-server-wait")
wait_for_screen("license-server-wait timed out", "waiting for license server at 10.0.0.9:27000")
start_failing_unit("metrics-agent")
wait_for_screen("metrics-agent exited with status 203")
start_failing_unit("http-listener")
wait_for_screen("http-listener restarted too often", "address already in use")
start_failing_unit("database-client")
wait_for_screen("database-client exited with status 1: Cannot reach database", "connection refused")
start_failing_unit("upstream-uplink")
wait_for_screen("upstream-uplink exited with status 1", "lost connection to upstream, giving up")
wait_for_screen("× 9 failed", "connection refused")
assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}"
print(screen())
machine.screenshot("failures")
machine.succeed("touch /run/release-boot-failures-hold")
retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60)
print(bootvine_journal())
+148
View File
@@ -0,0 +1,148 @@
{
config,
lib,
pkgs,
...
}:
let
startedByHand = config.bootvineSample.startFailuresByHand;
wantedAtBoot = target: lib.optional (!startedByHand) target;
failingService =
{
description,
script,
serviceConfig ? { },
unitConfig ? { },
extraAttributes ? { },
}:
{
inherit description unitConfig;
wantedBy = wantedAtBoot "multi-user.target";
serviceConfig = {
Type = "oneshot";
ExecStart = pkgs.writeShellScript "failing-service" script;
}
// serviceConfig;
}
// extraAttributes;
initrdFirmwareScript = pkgs.writeShellScript "initrd-nic-firmware-load" ''
echo "loading firmware for enp1s0 (r8169)"
echo "firmware: rtl_nic/rtl8168h-2.fw not found" >&2
exit 1
'';
in
{
options.bootvineSample.startFailuresByHand = lib.mkEnableOption ''
leaving the failing sample units out of the boot targets, so a check can
start them one at a time
'';
config = {
boot.initrd.systemd = {
storePaths = [
initrdFirmwareScript
"${pkgs.bash}/bin/bash"
];
services.initrd-nic-firmware-load = {
description = "Network card firmware";
wantedBy = wantedAtBoot "initrd.target";
before = [ "initrd.target" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.bash}/bin/bash ${initrdFirmwareScript}";
};
};
};
systemd.services = {
app-config-check = failingService {
description = "Application configuration check";
script = ''
echo "reading /etc/app/app.toml"
echo "config: /etc/app/app.toml: missing key 'listen'" >&2
exit 2
'';
};
worker-pool = failingService {
description = "Worker pool";
script = ''
echo "starting worker pool with 4 threads"
kill -SEGV $$
'';
};
license-server-wait = failingService {
description = "License server check";
script = ''
echo "waiting for license server at 10.0.0.9:27000"
sleep infinity
'';
serviceConfig.TimeoutStartSec = 4;
};
metrics-agent = {
description = "Metrics agent";
wantedBy = wantedAtBoot "multi-user.target";
serviceConfig = {
Type = "oneshot";
ExecStart = "/usr/bin/metrics-agent --listen :9100";
};
};
http-listener = failingService {
description = "HTTP listener";
script = ''
echo "bind 0.0.0.0:8080: address already in use" >&2
exit 1
'';
serviceConfig = {
Restart = "on-failure";
RestartSec = "200ms";
};
unitConfig = {
StartLimitBurst = 3;
StartLimitIntervalSec = 30;
};
};
database-client = failingService {
description = "Database client";
script = ''
systemd-notify --status="Cannot reach database at db.internal:5432"
echo "connect db.internal:5432: connection refused" >&2
exit 1
'';
serviceConfig = {
Type = "notify";
NotifyAccess = "all";
};
};
upstream-uplink = failingService {
description = "Upstream uplink";
script = ''
echo "connected to upstream"
sleep 3
echo "lost connection to upstream, giving up" >&2
exit 1
'';
serviceConfig.Type = "simple";
};
failure-report = failingService {
description = "Failure report";
script = ''
echo "all checks passed"
'';
extraAttributes = {
requires = [ "app-config-check.service" ];
after = [ "app-config-check.service" ];
};
};
};
};
}