Files
bootvine/vm-tests/failures/check.py
T
Willem Vanhulle 7e95cbd8e9 feat: show why units failed, with their last journal lines
Failed jobs keep their kind: units that broke or timed out get a failure
panel entry with the reason systemd reports (exit status, signal, timeout,
start limit, status text) and their last journal lines read through
libsystemd. Units that failed only through a dependency are dimmed and left
out of the panel, and units that crash after starting count as failed.

Restructure the repository around this:
- move the library and binary into bootvine/ and the scripted demos into
  their own unpublished scripted-demos crate, with their VHS tapes
- move the crane builds and NixOS module into nix/
- group each sample VM with its check under vm-tests/, adding text-console,
  scripted-initrd, encrypted-volume and failures machines and a vm-failures
  check
- add docs/guide.md walking through the screen with every recording, keep
  one recording in the README, and re-render all recordings
2026-10-30 14:59:10 +01:00

60 lines
2.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
def start_failing_unit(unit):
machine.succeed(f"systemctl start --no-block {unit}")
def unit_result(unit):
return machine.succeed(f"systemctl show -p Result --value {unit}").strip()
def failure_panel():
lines = screen().splitlines()
start = next(index for index, line in enumerate(lines) if " Failures " in line)
return "\n".join(lines[start:])
machine.start()
wait_for_screen("Initrd", "Initrd held for the check")
start_failing_unit("initrd-nic-firmware-load")
wait_for_screen("Failures", "initrd-nic-firmware-load exited with status 1", "rtl8168h-2.fw not found")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
machine.succeed("touch /run/release-initrd-failures-hold")
machine.switch_root()
wait_for_screen("Stage 2", "Boot held for the check")
wait_for_screen("initrd-nic-firmware-load exited with status 1")
start_failing_unit("failure-report")
wait_for_screen("app-config-check exited with status 2", "missing key 'listen'")
retry(lambda _: unit_result("failure-report") != "", timeout_seconds=10)
assert "× failure-report" not in failure_panel(), f"a dependency failure reached the panel:\n{screen()}"
assert "failure-report" in screen(), f"the dependency failure left the tree:\n{screen()}"
start_failing_unit("worker-pool")
wait_for_screen("worker-pool", "SIGSEGV", "starting worker pool with 4 threads")
start_failing_unit("license-server-wait")
wait_for_screen("license-server-wait timed out", "waiting for license server at 10.0.0.9:27000")
start_failing_unit("metrics-agent")
wait_for_screen("metrics-agent exited with status 203")
start_failing_unit("http-listener")
wait_for_screen("http-listener restarted too often", "address already in use")
start_failing_unit("database-client")
wait_for_screen("database-client exited with status 1: Cannot reach database", "connection refused")
start_failing_unit("upstream-uplink")
wait_for_screen("upstream-uplink exited with status 1", "lost connection to upstream, giving up")
wait_for_screen("× 9 failed", "connection refused")
assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}"
print(screen())
machine.screenshot("failures")
machine.succeed("touch /run/release-boot-failures-hold")
retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60)
print(bootvine_journal())