feat: show why units failed, with their last journal lines

Failed jobs keep their kind: units that broke or timed out get a failure
panel entry with the reason systemd reports (exit status, signal, timeout,
start limit, status text) and their last journal lines read through
libsystemd. Units that failed only through a dependency are dimmed and left
out of the panel, and units that crash after starting count as failed.

Restructure the repository around this:
- move the library and binary into bootvine/ and the scripted demos into
  their own unpublished scripted-demos crate, with their VHS tapes
- move the crane builds and NixOS module into nix/
- group each sample VM with its check under vm-tests/, adding text-console,
  scripted-initrd, encrypted-volume and failures machines and a vm-failures
  check
- add docs/guide.md walking through the screen with every recording, keep
  one recording in the README, and re-render all recordings
This commit is contained in:
Willem Vanhulle committed 2026-10-30 14:59:10 +01:00
1 parent f2544a964a
commit 7e95cbd8e9
96 files changed
+2027 -462

No files matched your search

+59
View File
@@ -0,0 +1,59 @@
def start_failing_unit(unit):
machine.succeed(f"systemctl start --no-block {unit}")
def unit_result(unit):
return machine.succeed(f"systemctl show -p Result --value {unit}").strip()
def failure_panel():
lines = screen().splitlines()
start = next(index for index, line in enumerate(lines) if " Failures " in line)
return "\n".join(lines[start:])
machine.start()
wait_for_screen("Initrd", "Initrd held for the check")
start_failing_unit("initrd-nic-firmware-load")
wait_for_screen("Failures", "initrd-nic-firmware-load exited with status 1", "rtl8168h-2.fw not found")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
machine.succeed("touch /run/release-initrd-failures-hold")
machine.switch_root()
wait_for_screen("Stage 2", "Boot held for the check")
wait_for_screen("initrd-nic-firmware-load exited with status 1")
start_failing_unit("failure-report")
wait_for_screen("app-config-check exited with status 2", "missing key 'listen'")
retry(lambda _: unit_result("failure-report") != "", timeout_seconds=10)
assert "× failure-report" not in failure_panel(), f"a dependency failure reached the panel:\n{screen()}"
assert "failure-report" in screen(), f"the dependency failure left the tree:\n{screen()}"
start_failing_unit("worker-pool")
wait_for_screen("worker-pool", "SIGSEGV", "starting worker pool with 4 threads")
start_failing_unit("license-server-wait")
wait_for_screen("license-server-wait timed out", "waiting for license server at 10.0.0.9:27000")
start_failing_unit("metrics-agent")
wait_for_screen("metrics-agent exited with status 203")
start_failing_unit("http-listener")
wait_for_screen("http-listener restarted too often", "address already in use")
start_failing_unit("database-client")
wait_for_screen("database-client exited with status 1: Cannot reach database", "connection refused")
start_failing_unit("upstream-uplink")
wait_for_screen("upstream-uplink exited with status 1", "lost connection to upstream, giving up")
wait_for_screen("× 9 failed", "connection refused")
assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}"
print(screen())
machine.screenshot("failures")
machine.succeed("touch /run/release-boot-failures-hold")
retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60)
print(bootvine_journal())