feat: show why units failed, with their last journal lines

Failed jobs keep their kind: units that broke or timed out get a failure
panel entry with the reason systemd reports (exit status, signal, timeout,
start limit, status text) and their last journal lines read through
libsystemd. Units that failed only through a dependency are dimmed and left
out of the panel, and units that crash after starting count as failed.

Restructure the repository around this:
- move the library and binary into bootvine/ and the scripted demos into
  their own unpublished scripted-demos crate, with their VHS tapes
- move the crane builds and NixOS module into nix/
- group each sample VM with its check under vm-tests/, adding text-console,
  scripted-initrd, encrypted-volume and failures machines and a vm-failures
  check
- add docs/guide.md walking through the screen with every recording, keep
  one recording in the README, and re-render all recordings
This commit is contained in:
Willem Vanhulle committed 2026-10-30 14:59:10 +01:00
1 parent f2544a964a
commit 7e95cbd8e9
96 files changed
+2027 -462

No files matched your search

+51
View File
@@ -0,0 +1,51 @@
{ lib, bootvineModule }:
let
machineModule = machineName: {
imports = [
bootvineModule
./shared/common.nix
./${machineName}/machine.nix
];
networking.hostName = "bootvine-${machineName}";
};
machineNames = [
"slow-services"
"text-console"
"scripted-initrd"
"encrypted-volume"
"failures"
];
machines = lib.genAttrs machineNames machineModule;
in
{
inherit machines;
sampleUnits = import ./slow-services/sample-units.nix;
nixosSystemFor =
system: machineName:
lib.nixosSystem {
inherit system;
modules = [
machines.${machineName}
(import ./shared/interactive-vm.nix machineName)
];
};
checksFor = pkgs: {
vm-boot-and-shutdown = pkgs.testers.runNixOSTest (
import ./slow-services/check.nix { machine = machines.slow-services; }
);
vm-luks-passphrase = pkgs.testers.runNixOSTest (
import ./encrypted-volume/check.nix {
machine = machines.encrypted-volume;
inherit pkgs;
}
);
vm-failures = pkgs.testers.runNixOSTest (
import ./failures/check.nix { machine = machines.failures; }
);
};
}
+24
View File
@@ -0,0 +1,24 @@
{ machine, pkgs }:
let
secretDisk = import ./secret-disk.nix { inherit pkgs; };
in
{
name = "bootvine-luks-passphrase";
nodes.machine = {
imports = [ machine ];
testing.initrdBackdoor = true;
virtualisation.qemu.options = [
"-vga none"
"-device virtio-vga,xres=1280,yres=800"
];
virtualisation.qemu.drives = [ secretDisk.qemuDrive ];
};
testScript = ''
PASSPHRASE = "${secretDisk.passphrase}"
''
+ builtins.readFile ../shared/console.py
+ builtins.readFile ./check.py;
}
+31
View File
@@ -0,0 +1,31 @@
def answered_prompts():
return bootvine_journal().count("answered a password prompt")
def pending_prompt():
listing = machine.succeed("ls /run/systemd/ask-password").split()
return next((name for name in listing if name.startswith("ask.")), "")
machine.start()
retry(lambda _: pending_prompt() != "", timeout_seconds=60)
wait_for_screen("passphrase")
assert_text_visible(font_has_512_glyphs=True)
first_prompt = pending_prompt()
machine.send_chars("not the passphrase\n")
retry(lambda _: answered_prompts() == 1, timeout_seconds=30)
retry(lambda _: pending_prompt() not in ("", first_prompt), timeout_seconds=30)
wait_for_screen("passphrase")
assert_text_visible(font_has_512_glyphs=True)
machine.screenshot("passphrase")
machine.send_chars(f"{PASSPHRASE}\n")
machine.wait_for_unit("systemd-cryptsetup@secret.service")
machine.switch_root()
machine.wait_for_unit("multi-user.target")
machine.succeed("test -e /dev/mapper/secret")
journal = bootvine_journal()
print(journal)
assert " in Initrd " in journal, "never ran in the initrd"
assert journal.count("answered a password prompt") == 2, "did not answer both attempts itself"
+11
View File
@@ -0,0 +1,11 @@
{ lib, pkgs, ... }:
let
secretDisk = import ./secret-disk.nix { inherit pkgs; };
in
{
boot.initrd.luks.devices = lib.mkVMOverride {
secret.device = "/dev/disk/by-uuid/${secretDisk.uuid}";
};
virtualisation.vmVariant.virtualisation.qemu.drives = [ secretDisk.qemuDrive ];
}
+22
View File
@@ -0,0 +1,22 @@
{ pkgs }:
let
passphrase = "sample passphrase";
uuid = "9d1f2a8e-0b7c-4c55-9a8e-6b2f8f3c1d00";
image = pkgs.runCommand "bootvine-luks-disk" { nativeBuildInputs = [ pkgs.cryptsetup ]; } ''
truncate -s 32M disk.img
echo -n '${passphrase}' | cryptsetup luksFormat -q --disable-locks --type luks2 \
--pbkdf pbkdf2 --pbkdf-force-iterations 1000 --uuid ${uuid} disk.img -
mv disk.img $out
'';
in
{
inherit passphrase uuid;
qemuDrive = {
name = "secret";
file = "${image}";
driveExtraOpts = {
format = "raw";
snapshot = "on";
};
};
}
+49
View File
@@ -0,0 +1,49 @@
{ machine }:
{
name = "bootvine-failures";
nodes.machine =
{ pkgs, ... }:
let
holdUntilReleased =
marker: "${pkgs.bash}/bin/bash -c 'until [ -e /run/${marker} ]; do sleep 0.1; done'";
in
{
imports = [ machine ];
bootvineSample.startFailuresByHand = true;
testing.initrdBackdoor = true;
boot.initrd.systemd = {
storePaths = [ "${pkgs.bash}/bin/bash" ];
services.initrd-failures-hold = {
description = "Initrd held for the check";
wantedBy = [ "initrd.target" ];
before = [ "initrd.target" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
ExecStart = holdUntilReleased "release-initrd-failures-hold";
};
};
};
systemd.services.boot-failures-hold = {
description = "Boot held for the check";
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = holdUntilReleased "release-boot-failures-hold";
};
};
virtualisation = {
memorySize = 2048;
qemu.options = [
"-vga none"
"-device virtio-vga,xres=1280,yres=800"
];
};
};
testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py;
}
+59
View File
@@ -0,0 +1,59 @@
def start_failing_unit(unit):
machine.succeed(f"systemctl start --no-block {unit}")
def unit_result(unit):
return machine.succeed(f"systemctl show -p Result --value {unit}").strip()
def failure_panel():
lines = screen().splitlines()
start = next(index for index, line in enumerate(lines) if " Failures " in line)
return "\n".join(lines[start:])
machine.start()
wait_for_screen("Initrd", "Initrd held for the check")
start_failing_unit("initrd-nic-firmware-load")
wait_for_screen("Failures", "initrd-nic-firmware-load exited with status 1", "rtl8168h-2.fw not found")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
machine.succeed("touch /run/release-initrd-failures-hold")
machine.switch_root()
wait_for_screen("Stage 2", "Boot held for the check")
wait_for_screen("initrd-nic-firmware-load exited with status 1")
start_failing_unit("failure-report")
wait_for_screen("app-config-check exited with status 2", "missing key 'listen'")
retry(lambda _: unit_result("failure-report") != "", timeout_seconds=10)
assert "× failure-report" not in failure_panel(), f"a dependency failure reached the panel:\n{screen()}"
assert "failure-report" in screen(), f"the dependency failure left the tree:\n{screen()}"
start_failing_unit("worker-pool")
wait_for_screen("worker-pool", "SIGSEGV", "starting worker pool with 4 threads")
start_failing_unit("license-server-wait")
wait_for_screen("license-server-wait timed out", "waiting for license server at 10.0.0.9:27000")
start_failing_unit("metrics-agent")
wait_for_screen("metrics-agent exited with status 203")
start_failing_unit("http-listener")
wait_for_screen("http-listener restarted too often", "address already in use")
start_failing_unit("database-client")
wait_for_screen("database-client exited with status 1: Cannot reach database", "connection refused")
start_failing_unit("upstream-uplink")
wait_for_screen("upstream-uplink exited with status 1", "lost connection to upstream, giving up")
wait_for_screen("× 9 failed", "connection refused")
assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}"
print(screen())
machine.screenshot("failures")
machine.succeed("touch /run/release-boot-failures-hold")
retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60)
print(bootvine_journal())
+148
View File
@@ -0,0 +1,148 @@
{
config,
lib,
pkgs,
...
}:
let
startedByHand = config.bootvineSample.startFailuresByHand;
wantedAtBoot = target: lib.optional (!startedByHand) target;
failingService =
{
description,
script,
serviceConfig ? { },
unitConfig ? { },
extraAttributes ? { },
}:
{
inherit description unitConfig;
wantedBy = wantedAtBoot "multi-user.target";
serviceConfig = {
Type = "oneshot";
ExecStart = pkgs.writeShellScript "failing-service" script;
}
// serviceConfig;
}
// extraAttributes;
initrdFirmwareScript = pkgs.writeShellScript "initrd-nic-firmware-load" ''
echo "loading firmware for enp1s0 (r8169)"
echo "firmware: rtl_nic/rtl8168h-2.fw not found" >&2
exit 1
'';
in
{
options.bootvineSample.startFailuresByHand = lib.mkEnableOption ''
leaving the failing sample units out of the boot targets, so a check can
start them one at a time
'';
config = {
boot.initrd.systemd = {
storePaths = [
initrdFirmwareScript
"${pkgs.bash}/bin/bash"
];
services.initrd-nic-firmware-load = {
description = "Network card firmware";
wantedBy = wantedAtBoot "initrd.target";
before = [ "initrd.target" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.bash}/bin/bash ${initrdFirmwareScript}";
};
};
};
systemd.services = {
app-config-check = failingService {
description = "Application configuration check";
script = ''
echo "reading /etc/app/app.toml"
echo "config: /etc/app/app.toml: missing key 'listen'" >&2
exit 2
'';
};
worker-pool = failingService {
description = "Worker pool";
script = ''
echo "starting worker pool with 4 threads"
kill -SEGV $$
'';
};
license-server-wait = failingService {
description = "License server check";
script = ''
echo "waiting for license server at 10.0.0.9:27000"
sleep infinity
'';
serviceConfig.TimeoutStartSec = 4;
};
metrics-agent = {
description = "Metrics agent";
wantedBy = wantedAtBoot "multi-user.target";
serviceConfig = {
Type = "oneshot";
ExecStart = "/usr/bin/metrics-agent --listen :9100";
};
};
http-listener = failingService {
description = "HTTP listener";
script = ''
echo "bind 0.0.0.0:8080: address already in use" >&2
exit 1
'';
serviceConfig = {
Restart = "on-failure";
RestartSec = "200ms";
};
unitConfig = {
StartLimitBurst = 3;
StartLimitIntervalSec = 30;
};
};
database-client = failingService {
description = "Database client";
script = ''
systemd-notify --status="Cannot reach database at db.internal:5432"
echo "connect db.internal:5432: connection refused" >&2
exit 1
'';
serviceConfig = {
Type = "notify";
NotifyAccess = "all";
};
};
upstream-uplink = failingService {
description = "Upstream uplink";
script = ''
echo "connected to upstream"
sleep 3
echo "lost connection to upstream, giving up" >&2
exit 1
'';
serviceConfig.Type = "simple";
};
failure-report = failingService {
description = "Failure report";
script = ''
echo "all checks passed"
'';
extraAttributes = {
requires = [ "app-config-check.service" ];
after = [ "app-config-check.service" ];
};
};
};
};
}
+6
View File
@@ -0,0 +1,6 @@
{ lib, ... }:
{
imports = [ ../slow-services/machine.nix ];
boot.initrd.systemd.enable = lib.mkForce false;
}
+45
View File
@@ -0,0 +1,45 @@
{ lib, pkgs, ... }:
{
system.stateVersion = "26.05";
boot.consoleLogLevel = lib.mkForce 3;
services.bootvine = {
enable = true;
settings.colors = {
tree.current-stage = "light-yellow";
tree.running-marker = "light-yellow";
timeline.running = "light-yellow";
footer.running = "light-yellow";
password.frame = "light-yellow";
};
};
console = {
colors = [
"1d2021"
"fb4934"
"b8bb26"
"fabd2f"
"83a598"
"d3869b"
"8ec07c"
"d5c4a1"
"665c54"
"fb4934"
"b8bb26"
"fabd2f"
"83a598"
"d3869b"
"8ec07c"
"fbf1c7"
];
earlySetup = true;
font = "${pkgs.terminus_font}/share/consolefonts/ter-v20n.psf.gz";
};
users.users.sample-user = {
isNormalUser = true;
password = "sample-user";
};
}
+70
View File
@@ -0,0 +1,70 @@
import base64
BOOTVINE_TERMINAL = 12
def read_terminal():
attributed, characters = machine.succeed(
f"base64 -w0 /dev/vcsa{BOOTVINE_TERMINAL}; echo; "
f"base64 -w0 /dev/vcsu{BOOTVINE_TERMINAL}"
).split()
attributed = base64.b64decode(attributed)
columns = attributed[1]
attributes = attributed[5::2]
text = base64.b64decode(characters).decode("utf-32-le")
return columns, text, attributes
def as_rows(text, columns):
return "\n".join(text[start : start + columns] for start in range(0, len(text), columns))
def screen():
columns, text, _ = read_terminal()
return as_rows(text, columns)
def colours(attribute, font_has_512_glyphs):
if font_has_512_glyphs:
return (attribute >> 1) & 0x0F, (attribute >> 5) & 0x07
return attribute & 0x0F, (attribute >> 4) & 0x07
def assert_text_visible(font_has_512_glyphs=False):
columns, text, attributes = read_terminal()
def is_hidden(character, attribute):
foreground, background = colours(attribute, font_has_512_glyphs)
return character != " " and foreground == background
hidden = "".join(
"#" if is_hidden(character, attribute) else " "
for character, attribute in zip(text, attributes)
)
assert "#" not in hidden, (
f"text drawn in its background colour, marked #:\n{as_rows(hidden, columns)}\n{as_rows(text, columns)}"
)
def wait_for_screen(*fragments):
def shows_all(last_try):
text = screen()
if last_try:
print(text)
print(bootvine_journal())
return all(fragment in text for fragment in fragments)
retry(shows_all, timeout_seconds=60)
def foreground_terminal():
return machine.succeed("cat /sys/class/tty/tty0/active").strip()
def assert_bootvine_in_front():
active = foreground_terminal()
assert active == f"tty{BOOTVINE_TERMINAL}", f"bootvine is not in front: {active}"
def bootvine_journal():
return machine.succeed("journalctl -b -u bootvine -o cat")
+20
View File
@@ -0,0 +1,20 @@
machineName: {
boot.loader.grub.enable = false;
services.getty.autologinUser = "root";
fileSystems."/" = {
device = "/dev/disk/by-label/nixos";
fsType = "ext4";
};
virtualisation.vmVariant.virtualisation = {
memorySize = 2048;
cores = 2;
graphics = true;
diskImage = "\${XDG_CACHE_HOME:-$HOME/.cache}/bootvine-${machineName}.qcow2";
qemu.options = [
"-vga none"
"-device virtio-vga,xres=1920,yres=1080"
];
};
}
+39
View File
@@ -0,0 +1,39 @@
{ machine }:
{
name = "bootvine-boot-and-shutdown";
nodes.machine =
{ lib, ... }:
let
waitFor = flag: "/bin/sh -c 'until [ -e /run/${flag} ]; do sleep 0.1; done'";
in
{
imports = [ machine ];
testing.initrdBackdoor = true;
boot.initrd.systemd.services.initrd-hardware-probe.serviceConfig.ExecStart = lib.mkForce (
waitFor "release-initrd-hardware-probe"
);
systemd = {
settings.Manager.ShowStatus = lib.mkForce true;
services.sample-database.serviceConfig.ExecStop = lib.mkForce (waitFor "release-sample-database");
services.backdoor.before = [ "sample-database.service" ];
services.late-sensor-probe = {
wantedBy = [ "graphical.target" ];
after = [ "greetd.service" ];
serviceConfig = {
Type = "oneshot";
ExecStart = waitFor "release-late-sensor-probe";
};
};
};
virtualisation = {
memorySize = 2048;
qemu.options = [
"-vga none"
"-device virtio-vga,xres=1280,yres=800"
];
};
};
testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py;
}
+58
View File
@@ -0,0 +1,58 @@
def console_level():
return machine.succeed("cut -f1 /proc/sys/kernel/printk").strip()
def unit_state(unit):
return machine.succeed(f"systemctl is-active {unit} || true").strip()
machine.start()
wait_for_screen("Initrd", "Slow hardware probe")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
assert console_level() == "1", f"the kernel console was not quieted: {console_level()}"
machine.succeed("touch /run/release-initrd-hardware-probe")
machine.switch_root()
retry(lambda _: unit_state("sample-database") == "activating", timeout_seconds=60)
wait_for_screen("Stage 2", "sample-database")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
assert console_level() == "1", f"stage 2 sysctl reopened the kernel console: {console_level()}"
machine.wait_for_unit("greetd.service")
retry(lambda _: unit_state("bootvine") == "inactive", timeout_seconds=60)
assert unit_state("late-sensor-probe") == "activating", "the boot finished before the display manager took over"
assert foreground_terminal() != f"tty{BOOTVINE_TERMINAL}", "bootvine took the console back from the display manager"
machine.succeed("touch /run/release-late-sensor-probe")
machine.wait_for_unit("graphical.target")
journal = bootvine_journal()
print(journal)
watched = [line for line in journal.splitlines() if line.startswith("bootvine: watching")]
assert any(" in Initrd " in line for line in watched), "never ran in the initrd"
assert any(" in Stage 2 " in line for line in watched), "never reconnected after the root switch"
pids = {line.split(" as pid ")[1].split()[0] for line in watched}
assert len(pids) == 1, f"the initrd process did not survive the root switch: {pids}"
assert "handing the console over" in journal, "never handed the console over"
machine.succeed('test "$(systemctl show -p Result --value bootvine)" = success')
assert console_level() != "1", f"kernel console level was not restored: {console_level()}"
show_status = machine.succeed("systemctl show -p ShowStatus --value").strip()
assert show_status == "yes", f"systemd status output was not restored: {show_status}"
foreground = foreground_terminal()
assert foreground != f"tty{BOOTVINE_TERMINAL}", "bootvine kept the console after handing it over"
machine.succeed("systemctl restart bootvine")
retry(lambda _: unit_state("bootvine") == "inactive", timeout_seconds=10)
assert "boot already finished" in bootvine_journal(), "a restart after boot did not stand down"
assert foreground_terminal() == foreground, "a restart after boot took the console"
machine.succeed("systemctl poweroff --no-block")
retry(lambda _: unit_state("sample-database") == "deactivating", timeout_seconds=60)
wait_for_screen("Shutdown", "sample-database")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
machine.succeed("touch /run/release-sample-database")
machine.wait_for_shutdown()
+42
View File
@@ -0,0 +1,42 @@
{ lib, pkgs, ... }:
let
sampleUnits = import ./sample-units.nix;
sampleService = spec: {
inherit (spec) description;
after = map (dependency: "${dependency}.service") (spec.after or [ ]);
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = [
"${pkgs.coreutils}/bin/sleep ${spec.seconds}"
]
++ lib.optional (spec.fails or false) "${pkgs.coreutils}/bin/false";
}
// lib.optionalAttrs (spec ? stopSeconds) {
RemainAfterExit = true;
ExecStop = "${pkgs.coreutils}/bin/sleep ${spec.stopSeconds}";
};
};
in
{
boot.initrd.systemd.services.initrd-hardware-probe = {
description = "Slow hardware probe";
wantedBy = [ "initrd.target" ];
before = [ "initrd.target" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
ExecStart = "/bin/sleep ${sampleUnits.initrdProbeSeconds}";
};
};
systemd.services = lib.mapAttrs (_: sampleService) sampleUnits.services // {
greetd.after = [ "multi-user.target" ];
};
services.greetd = {
enable = true;
settings.default_session.command = "${pkgs.greetd}/bin/agreety --cmd ${pkgs.bashInteractive}/bin/bash";
};
}
@@ -0,0 +1,54 @@
{
sampleUnits,
lib,
coreutils,
runCommand,
writeText,
}:
let
toUnitFile = lib.generators.toINI { listsAsDuplicateKeys = true; };
serviceFile =
name: spec:
writeText "${name}.service" (toUnitFile {
Unit = {
Description = spec.description;
After = map (dependency: "${dependency}.service") (spec.after or [ ]);
};
Service = {
Type = "oneshot";
ExecStart = [
"${coreutils}/bin/sleep ${spec.seconds}"
]
++ lib.optional (spec.fails or false) "${coreutils}/bin/false";
}
// lib.optionalAttrs (spec ? stopSeconds) {
RemainAfterExit = true;
ExecStop = "${coreutils}/bin/sleep ${spec.stopSeconds}";
};
});
serviceNames = map (name: "${name}.service") (lib.attrNames sampleUnits.services);
targetFile = writeText "${sampleUnits.target}.target" (toUnitFile {
Unit = {
Description = "bootvine demo services";
Wants = serviceNames;
After = serviceNames;
};
});
unitFiles =
lib.mapAttrsToList (name: spec: {
name = "${name}.service";
path = serviceFile name spec;
}) sampleUnits.services
++ [
{
name = "${sampleUnits.target}.target";
path = targetFile;
}
];
in
runCommand "bootvine-sample-unit-files" { } (
lib.concatMapStringsSep "\n" (unit: "install -Dm644 ${unit.path} $out/${unit.name}") unitFiles
)
+52
View File
@@ -0,0 +1,52 @@
{
target = "bootvine-sample-services";
initrdProbeSeconds = "6";
services = {
scratch-disk-prepare = {
description = "Scratch disk preparation";
seconds = "2";
stopSeconds = "2";
};
sample-database = {
description = "Sample database";
seconds = "3";
stopSeconds = "3";
after = [ "scratch-disk-prepare" ];
};
memory-cache = {
description = "In-memory cache";
seconds = "1.5";
stopSeconds = "1";
};
web-frontend = {
description = "Web frontend";
seconds = "2";
stopSeconds = "2";
after = [
"sample-database"
"memory-cache"
];
};
remote-content-sync = {
description = "Remote content sync";
seconds = "4";
};
flaky-health-check = {
description = "Flaky health check";
seconds = "1";
fails = true;
};
slow-warm-up = {
description = "Slow warm-up job";
seconds = "8";
};
startup-report = {
description = "Startup report";
seconds = "1";
after = [
"web-frontend"
"remote-content-sync"
];
};
};
}
+6
View File
@@ -0,0 +1,6 @@
{ lib, ... }:
{
imports = [ ../slow-services/machine.nix ];
services.greetd.enable = lib.mkForce false;
}