feat: show why units failed, with their last journal lines
Failed jobs keep their kind: units that broke or timed out get a failure panel entry with the reason systemd reports (exit status, signal, timeout, start limit, status text) and their last journal lines read through libsystemd. Units that failed only through a dependency are dimmed and left out of the panel, and units that crash after starting count as failed. Restructure the repository around this: - move the library and binary into bootvine/ and the scripted demos into their own unpublished scripted-demos crate, with their VHS tapes - move the crane builds and NixOS module into nix/ - group each sample VM with its check under vm-tests/, adding text-console, scripted-initrd, encrypted-volume and failures machines and a vm-failures check - add docs/guide.md walking through the screen with every recording, keep one recording in the README, and re-render all recordings
This commit is contained in:
1 parent
f2544a964a
commit
7e95cbd8e9
96 files changed
+2027
-462
No files matched your search
@@ -0,0 +1,51 @@
|
||||
{ lib, bootvineModule }:
|
||||
let
|
||||
machineModule = machineName: {
|
||||
imports = [
|
||||
bootvineModule
|
||||
./shared/common.nix
|
||||
./${machineName}/machine.nix
|
||||
];
|
||||
networking.hostName = "bootvine-${machineName}";
|
||||
};
|
||||
|
||||
machineNames = [
|
||||
"slow-services"
|
||||
"text-console"
|
||||
"scripted-initrd"
|
||||
"encrypted-volume"
|
||||
"failures"
|
||||
];
|
||||
|
||||
machines = lib.genAttrs machineNames machineModule;
|
||||
in
|
||||
{
|
||||
inherit machines;
|
||||
|
||||
sampleUnits = import ./slow-services/sample-units.nix;
|
||||
|
||||
nixosSystemFor =
|
||||
system: machineName:
|
||||
lib.nixosSystem {
|
||||
inherit system;
|
||||
modules = [
|
||||
machines.${machineName}
|
||||
(import ./shared/interactive-vm.nix machineName)
|
||||
];
|
||||
};
|
||||
|
||||
checksFor = pkgs: {
|
||||
vm-boot-and-shutdown = pkgs.testers.runNixOSTest (
|
||||
import ./slow-services/check.nix { machine = machines.slow-services; }
|
||||
);
|
||||
vm-luks-passphrase = pkgs.testers.runNixOSTest (
|
||||
import ./encrypted-volume/check.nix {
|
||||
machine = machines.encrypted-volume;
|
||||
inherit pkgs;
|
||||
}
|
||||
);
|
||||
vm-failures = pkgs.testers.runNixOSTest (
|
||||
import ./failures/check.nix { machine = machines.failures; }
|
||||
);
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{ machine, pkgs }:
|
||||
let
|
||||
secretDisk = import ./secret-disk.nix { inherit pkgs; };
|
||||
in
|
||||
{
|
||||
name = "bootvine-luks-passphrase";
|
||||
nodes.machine = {
|
||||
imports = [ machine ];
|
||||
|
||||
testing.initrdBackdoor = true;
|
||||
|
||||
virtualisation.qemu.options = [
|
||||
"-vga none"
|
||||
"-device virtio-vga,xres=1280,yres=800"
|
||||
];
|
||||
virtualisation.qemu.drives = [ secretDisk.qemuDrive ];
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
PASSPHRASE = "${secretDisk.passphrase}"
|
||||
''
|
||||
+ builtins.readFile ../shared/console.py
|
||||
+ builtins.readFile ./check.py;
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
def answered_prompts():
|
||||
return bootvine_journal().count("answered a password prompt")
|
||||
|
||||
|
||||
def pending_prompt():
|
||||
listing = machine.succeed("ls /run/systemd/ask-password").split()
|
||||
return next((name for name in listing if name.startswith("ask.")), "")
|
||||
|
||||
|
||||
machine.start()
|
||||
|
||||
retry(lambda _: pending_prompt() != "", timeout_seconds=60)
|
||||
wait_for_screen("passphrase")
|
||||
assert_text_visible(font_has_512_glyphs=True)
|
||||
first_prompt = pending_prompt()
|
||||
machine.send_chars("not the passphrase\n")
|
||||
retry(lambda _: answered_prompts() == 1, timeout_seconds=30)
|
||||
retry(lambda _: pending_prompt() not in ("", first_prompt), timeout_seconds=30)
|
||||
wait_for_screen("passphrase")
|
||||
assert_text_visible(font_has_512_glyphs=True)
|
||||
machine.screenshot("passphrase")
|
||||
machine.send_chars(f"{PASSPHRASE}\n")
|
||||
machine.wait_for_unit("systemd-cryptsetup@secret.service")
|
||||
machine.switch_root()
|
||||
machine.wait_for_unit("multi-user.target")
|
||||
|
||||
machine.succeed("test -e /dev/mapper/secret")
|
||||
journal = bootvine_journal()
|
||||
print(journal)
|
||||
assert " in Initrd " in journal, "never ran in the initrd"
|
||||
assert journal.count("answered a password prompt") == 2, "did not answer both attempts itself"
|
||||
@@ -0,0 +1,11 @@
|
||||
{ lib, pkgs, ... }:
|
||||
let
|
||||
secretDisk = import ./secret-disk.nix { inherit pkgs; };
|
||||
in
|
||||
{
|
||||
boot.initrd.luks.devices = lib.mkVMOverride {
|
||||
secret.device = "/dev/disk/by-uuid/${secretDisk.uuid}";
|
||||
};
|
||||
|
||||
virtualisation.vmVariant.virtualisation.qemu.drives = [ secretDisk.qemuDrive ];
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{ pkgs }:
|
||||
let
|
||||
passphrase = "sample passphrase";
|
||||
uuid = "9d1f2a8e-0b7c-4c55-9a8e-6b2f8f3c1d00";
|
||||
image = pkgs.runCommand "bootvine-luks-disk" { nativeBuildInputs = [ pkgs.cryptsetup ]; } ''
|
||||
truncate -s 32M disk.img
|
||||
echo -n '${passphrase}' | cryptsetup luksFormat -q --disable-locks --type luks2 \
|
||||
--pbkdf pbkdf2 --pbkdf-force-iterations 1000 --uuid ${uuid} disk.img -
|
||||
mv disk.img $out
|
||||
'';
|
||||
in
|
||||
{
|
||||
inherit passphrase uuid;
|
||||
qemuDrive = {
|
||||
name = "secret";
|
||||
file = "${image}";
|
||||
driveExtraOpts = {
|
||||
format = "raw";
|
||||
snapshot = "on";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
{ machine }:
|
||||
{
|
||||
name = "bootvine-failures";
|
||||
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
holdUntilReleased =
|
||||
marker: "${pkgs.bash}/bin/bash -c 'until [ -e /run/${marker} ]; do sleep 0.1; done'";
|
||||
in
|
||||
{
|
||||
imports = [ machine ];
|
||||
bootvineSample.startFailuresByHand = true;
|
||||
testing.initrdBackdoor = true;
|
||||
|
||||
boot.initrd.systemd = {
|
||||
storePaths = [ "${pkgs.bash}/bin/bash" ];
|
||||
services.initrd-failures-hold = {
|
||||
description = "Initrd held for the check";
|
||||
wantedBy = [ "initrd.target" ];
|
||||
before = [ "initrd.target" ];
|
||||
unitConfig.DefaultDependencies = false;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = holdUntilReleased "release-initrd-failures-hold";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.boot-failures-hold = {
|
||||
description = "Boot held for the check";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = holdUntilReleased "release-boot-failures-hold";
|
||||
};
|
||||
};
|
||||
|
||||
virtualisation = {
|
||||
memorySize = 2048;
|
||||
qemu.options = [
|
||||
"-vga none"
|
||||
"-device virtio-vga,xres=1280,yres=800"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
def start_failing_unit(unit):
|
||||
machine.succeed(f"systemctl start --no-block {unit}")
|
||||
|
||||
|
||||
def unit_result(unit):
|
||||
return machine.succeed(f"systemctl show -p Result --value {unit}").strip()
|
||||
|
||||
|
||||
def failure_panel():
|
||||
lines = screen().splitlines()
|
||||
start = next(index for index, line in enumerate(lines) if " Failures " in line)
|
||||
return "\n".join(lines[start:])
|
||||
|
||||
|
||||
machine.start()
|
||||
|
||||
wait_for_screen("Initrd", "Initrd held for the check")
|
||||
start_failing_unit("initrd-nic-firmware-load")
|
||||
wait_for_screen("Failures", "initrd-nic-firmware-load exited with status 1", "rtl8168h-2.fw not found")
|
||||
assert_bootvine_in_front()
|
||||
assert_text_visible(font_has_512_glyphs=True)
|
||||
machine.succeed("touch /run/release-initrd-failures-hold")
|
||||
machine.switch_root()
|
||||
|
||||
wait_for_screen("Stage 2", "Boot held for the check")
|
||||
wait_for_screen("initrd-nic-firmware-load exited with status 1")
|
||||
|
||||
start_failing_unit("failure-report")
|
||||
wait_for_screen("app-config-check exited with status 2", "missing key 'listen'")
|
||||
retry(lambda _: unit_result("failure-report") != "", timeout_seconds=10)
|
||||
assert "× failure-report" not in failure_panel(), f"a dependency failure reached the panel:\n{screen()}"
|
||||
assert "failure-report" in screen(), f"the dependency failure left the tree:\n{screen()}"
|
||||
|
||||
start_failing_unit("worker-pool")
|
||||
wait_for_screen("worker-pool", "SIGSEGV", "starting worker pool with 4 threads")
|
||||
|
||||
start_failing_unit("license-server-wait")
|
||||
wait_for_screen("license-server-wait timed out", "waiting for license server at 10.0.0.9:27000")
|
||||
|
||||
start_failing_unit("metrics-agent")
|
||||
wait_for_screen("metrics-agent exited with status 203")
|
||||
|
||||
start_failing_unit("http-listener")
|
||||
wait_for_screen("http-listener restarted too often", "address already in use")
|
||||
|
||||
start_failing_unit("database-client")
|
||||
wait_for_screen("database-client exited with status 1: Cannot reach database", "connection refused")
|
||||
|
||||
start_failing_unit("upstream-uplink")
|
||||
wait_for_screen("upstream-uplink exited with status 1", "lost connection to upstream, giving up")
|
||||
|
||||
wait_for_screen("× 9 failed", "connection refused")
|
||||
assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}"
|
||||
print(screen())
|
||||
machine.screenshot("failures")
|
||||
|
||||
machine.succeed("touch /run/release-boot-failures-hold")
|
||||
retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60)
|
||||
print(bootvine_journal())
|
||||
@@ -0,0 +1,148 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
startedByHand = config.bootvineSample.startFailuresByHand;
|
||||
wantedAtBoot = target: lib.optional (!startedByHand) target;
|
||||
|
||||
failingService =
|
||||
{
|
||||
description,
|
||||
script,
|
||||
serviceConfig ? { },
|
||||
unitConfig ? { },
|
||||
extraAttributes ? { },
|
||||
}:
|
||||
{
|
||||
inherit description unitConfig;
|
||||
wantedBy = wantedAtBoot "multi-user.target";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = pkgs.writeShellScript "failing-service" script;
|
||||
}
|
||||
// serviceConfig;
|
||||
}
|
||||
// extraAttributes;
|
||||
|
||||
initrdFirmwareScript = pkgs.writeShellScript "initrd-nic-firmware-load" ''
|
||||
echo "loading firmware for enp1s0 (r8169)"
|
||||
echo "firmware: rtl_nic/rtl8168h-2.fw not found" >&2
|
||||
exit 1
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.bootvineSample.startFailuresByHand = lib.mkEnableOption ''
|
||||
leaving the failing sample units out of the boot targets, so a check can
|
||||
start them one at a time
|
||||
'';
|
||||
|
||||
config = {
|
||||
boot.initrd.systemd = {
|
||||
storePaths = [
|
||||
initrdFirmwareScript
|
||||
"${pkgs.bash}/bin/bash"
|
||||
];
|
||||
services.initrd-nic-firmware-load = {
|
||||
description = "Network card firmware";
|
||||
wantedBy = wantedAtBoot "initrd.target";
|
||||
before = [ "initrd.target" ];
|
||||
unitConfig.DefaultDependencies = false;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "${pkgs.bash}/bin/bash ${initrdFirmwareScript}";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services = {
|
||||
app-config-check = failingService {
|
||||
description = "Application configuration check";
|
||||
script = ''
|
||||
echo "reading /etc/app/app.toml"
|
||||
echo "config: /etc/app/app.toml: missing key 'listen'" >&2
|
||||
exit 2
|
||||
'';
|
||||
};
|
||||
|
||||
worker-pool = failingService {
|
||||
description = "Worker pool";
|
||||
script = ''
|
||||
echo "starting worker pool with 4 threads"
|
||||
kill -SEGV $$
|
||||
'';
|
||||
};
|
||||
|
||||
license-server-wait = failingService {
|
||||
description = "License server check";
|
||||
script = ''
|
||||
echo "waiting for license server at 10.0.0.9:27000"
|
||||
sleep infinity
|
||||
'';
|
||||
serviceConfig.TimeoutStartSec = 4;
|
||||
};
|
||||
|
||||
metrics-agent = {
|
||||
description = "Metrics agent";
|
||||
wantedBy = wantedAtBoot "multi-user.target";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "/usr/bin/metrics-agent --listen :9100";
|
||||
};
|
||||
};
|
||||
|
||||
http-listener = failingService {
|
||||
description = "HTTP listener";
|
||||
script = ''
|
||||
echo "bind 0.0.0.0:8080: address already in use" >&2
|
||||
exit 1
|
||||
'';
|
||||
serviceConfig = {
|
||||
Restart = "on-failure";
|
||||
RestartSec = "200ms";
|
||||
};
|
||||
unitConfig = {
|
||||
StartLimitBurst = 3;
|
||||
StartLimitIntervalSec = 30;
|
||||
};
|
||||
};
|
||||
|
||||
database-client = failingService {
|
||||
description = "Database client";
|
||||
script = ''
|
||||
systemd-notify --status="Cannot reach database at db.internal:5432"
|
||||
echo "connect db.internal:5432: connection refused" >&2
|
||||
exit 1
|
||||
'';
|
||||
serviceConfig = {
|
||||
Type = "notify";
|
||||
NotifyAccess = "all";
|
||||
};
|
||||
};
|
||||
|
||||
upstream-uplink = failingService {
|
||||
description = "Upstream uplink";
|
||||
script = ''
|
||||
echo "connected to upstream"
|
||||
sleep 3
|
||||
echo "lost connection to upstream, giving up" >&2
|
||||
exit 1
|
||||
'';
|
||||
serviceConfig.Type = "simple";
|
||||
};
|
||||
|
||||
failure-report = failingService {
|
||||
description = "Failure report";
|
||||
script = ''
|
||||
echo "all checks passed"
|
||||
'';
|
||||
extraAttributes = {
|
||||
requires = [ "app-config-check.service" ];
|
||||
after = [ "app-config-check.service" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{ lib, ... }:
|
||||
{
|
||||
imports = [ ../slow-services/machine.nix ];
|
||||
|
||||
boot.initrd.systemd.enable = lib.mkForce false;
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
{ lib, pkgs, ... }:
|
||||
{
|
||||
system.stateVersion = "26.05";
|
||||
|
||||
boot.consoleLogLevel = lib.mkForce 3;
|
||||
|
||||
services.bootvine = {
|
||||
enable = true;
|
||||
settings.colors = {
|
||||
tree.current-stage = "light-yellow";
|
||||
tree.running-marker = "light-yellow";
|
||||
timeline.running = "light-yellow";
|
||||
footer.running = "light-yellow";
|
||||
password.frame = "light-yellow";
|
||||
};
|
||||
};
|
||||
|
||||
console = {
|
||||
colors = [
|
||||
"1d2021"
|
||||
"fb4934"
|
||||
"b8bb26"
|
||||
"fabd2f"
|
||||
"83a598"
|
||||
"d3869b"
|
||||
"8ec07c"
|
||||
"d5c4a1"
|
||||
"665c54"
|
||||
"fb4934"
|
||||
"b8bb26"
|
||||
"fabd2f"
|
||||
"83a598"
|
||||
"d3869b"
|
||||
"8ec07c"
|
||||
"fbf1c7"
|
||||
];
|
||||
earlySetup = true;
|
||||
font = "${pkgs.terminus_font}/share/consolefonts/ter-v20n.psf.gz";
|
||||
};
|
||||
|
||||
users.users.sample-user = {
|
||||
isNormalUser = true;
|
||||
password = "sample-user";
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import base64
|
||||
|
||||
BOOTVINE_TERMINAL = 12
|
||||
|
||||
|
||||
def read_terminal():
|
||||
attributed, characters = machine.succeed(
|
||||
f"base64 -w0 /dev/vcsa{BOOTVINE_TERMINAL}; echo; "
|
||||
f"base64 -w0 /dev/vcsu{BOOTVINE_TERMINAL}"
|
||||
).split()
|
||||
attributed = base64.b64decode(attributed)
|
||||
columns = attributed[1]
|
||||
attributes = attributed[5::2]
|
||||
text = base64.b64decode(characters).decode("utf-32-le")
|
||||
return columns, text, attributes
|
||||
|
||||
|
||||
def as_rows(text, columns):
|
||||
return "\n".join(text[start : start + columns] for start in range(0, len(text), columns))
|
||||
|
||||
|
||||
def screen():
|
||||
columns, text, _ = read_terminal()
|
||||
return as_rows(text, columns)
|
||||
|
||||
|
||||
def colours(attribute, font_has_512_glyphs):
|
||||
if font_has_512_glyphs:
|
||||
return (attribute >> 1) & 0x0F, (attribute >> 5) & 0x07
|
||||
return attribute & 0x0F, (attribute >> 4) & 0x07
|
||||
|
||||
|
||||
def assert_text_visible(font_has_512_glyphs=False):
|
||||
columns, text, attributes = read_terminal()
|
||||
|
||||
def is_hidden(character, attribute):
|
||||
foreground, background = colours(attribute, font_has_512_glyphs)
|
||||
return character != " " and foreground == background
|
||||
|
||||
hidden = "".join(
|
||||
"#" if is_hidden(character, attribute) else " "
|
||||
for character, attribute in zip(text, attributes)
|
||||
)
|
||||
assert "#" not in hidden, (
|
||||
f"text drawn in its background colour, marked #:\n{as_rows(hidden, columns)}\n{as_rows(text, columns)}"
|
||||
)
|
||||
|
||||
|
||||
def wait_for_screen(*fragments):
|
||||
def shows_all(last_try):
|
||||
text = screen()
|
||||
if last_try:
|
||||
print(text)
|
||||
print(bootvine_journal())
|
||||
return all(fragment in text for fragment in fragments)
|
||||
|
||||
retry(shows_all, timeout_seconds=60)
|
||||
|
||||
|
||||
def foreground_terminal():
|
||||
return machine.succeed("cat /sys/class/tty/tty0/active").strip()
|
||||
|
||||
|
||||
def assert_bootvine_in_front():
|
||||
active = foreground_terminal()
|
||||
assert active == f"tty{BOOTVINE_TERMINAL}", f"bootvine is not in front: {active}"
|
||||
|
||||
|
||||
def bootvine_journal():
|
||||
return machine.succeed("journalctl -b -u bootvine -o cat")
|
||||
@@ -0,0 +1,20 @@
|
||||
machineName: {
|
||||
boot.loader.grub.enable = false;
|
||||
|
||||
services.getty.autologinUser = "root";
|
||||
fileSystems."/" = {
|
||||
device = "/dev/disk/by-label/nixos";
|
||||
fsType = "ext4";
|
||||
};
|
||||
|
||||
virtualisation.vmVariant.virtualisation = {
|
||||
memorySize = 2048;
|
||||
cores = 2;
|
||||
graphics = true;
|
||||
diskImage = "\${XDG_CACHE_HOME:-$HOME/.cache}/bootvine-${machineName}.qcow2";
|
||||
qemu.options = [
|
||||
"-vga none"
|
||||
"-device virtio-vga,xres=1920,yres=1080"
|
||||
];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{ machine }:
|
||||
{
|
||||
name = "bootvine-boot-and-shutdown";
|
||||
|
||||
nodes.machine =
|
||||
{ lib, ... }:
|
||||
let
|
||||
waitFor = flag: "/bin/sh -c 'until [ -e /run/${flag} ]; do sleep 0.1; done'";
|
||||
in
|
||||
{
|
||||
imports = [ machine ];
|
||||
testing.initrdBackdoor = true;
|
||||
boot.initrd.systemd.services.initrd-hardware-probe.serviceConfig.ExecStart = lib.mkForce (
|
||||
waitFor "release-initrd-hardware-probe"
|
||||
);
|
||||
systemd = {
|
||||
settings.Manager.ShowStatus = lib.mkForce true;
|
||||
services.sample-database.serviceConfig.ExecStop = lib.mkForce (waitFor "release-sample-database");
|
||||
services.backdoor.before = [ "sample-database.service" ];
|
||||
services.late-sensor-probe = {
|
||||
wantedBy = [ "graphical.target" ];
|
||||
after = [ "greetd.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = waitFor "release-late-sensor-probe";
|
||||
};
|
||||
};
|
||||
};
|
||||
virtualisation = {
|
||||
memorySize = 2048;
|
||||
qemu.options = [
|
||||
"-vga none"
|
||||
"-device virtio-vga,xres=1280,yres=800"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
def console_level():
|
||||
return machine.succeed("cut -f1 /proc/sys/kernel/printk").strip()
|
||||
|
||||
|
||||
def unit_state(unit):
|
||||
return machine.succeed(f"systemctl is-active {unit} || true").strip()
|
||||
|
||||
|
||||
machine.start()
|
||||
|
||||
wait_for_screen("Initrd", "Slow hardware probe")
|
||||
assert_bootvine_in_front()
|
||||
assert_text_visible(font_has_512_glyphs=True)
|
||||
assert console_level() == "1", f"the kernel console was not quieted: {console_level()}"
|
||||
machine.succeed("touch /run/release-initrd-hardware-probe")
|
||||
machine.switch_root()
|
||||
|
||||
retry(lambda _: unit_state("sample-database") == "activating", timeout_seconds=60)
|
||||
wait_for_screen("Stage 2", "sample-database")
|
||||
assert_bootvine_in_front()
|
||||
assert_text_visible(font_has_512_glyphs=True)
|
||||
assert console_level() == "1", f"stage 2 sysctl reopened the kernel console: {console_level()}"
|
||||
|
||||
machine.wait_for_unit("greetd.service")
|
||||
retry(lambda _: unit_state("bootvine") == "inactive", timeout_seconds=60)
|
||||
assert unit_state("late-sensor-probe") == "activating", "the boot finished before the display manager took over"
|
||||
assert foreground_terminal() != f"tty{BOOTVINE_TERMINAL}", "bootvine took the console back from the display manager"
|
||||
machine.succeed("touch /run/release-late-sensor-probe")
|
||||
machine.wait_for_unit("graphical.target")
|
||||
|
||||
journal = bootvine_journal()
|
||||
print(journal)
|
||||
watched = [line for line in journal.splitlines() if line.startswith("bootvine: watching")]
|
||||
assert any(" in Initrd " in line for line in watched), "never ran in the initrd"
|
||||
assert any(" in Stage 2 " in line for line in watched), "never reconnected after the root switch"
|
||||
pids = {line.split(" as pid ")[1].split()[0] for line in watched}
|
||||
assert len(pids) == 1, f"the initrd process did not survive the root switch: {pids}"
|
||||
assert "handing the console over" in journal, "never handed the console over"
|
||||
|
||||
machine.succeed('test "$(systemctl show -p Result --value bootvine)" = success')
|
||||
assert console_level() != "1", f"kernel console level was not restored: {console_level()}"
|
||||
show_status = machine.succeed("systemctl show -p ShowStatus --value").strip()
|
||||
assert show_status == "yes", f"systemd status output was not restored: {show_status}"
|
||||
|
||||
foreground = foreground_terminal()
|
||||
assert foreground != f"tty{BOOTVINE_TERMINAL}", "bootvine kept the console after handing it over"
|
||||
machine.succeed("systemctl restart bootvine")
|
||||
retry(lambda _: unit_state("bootvine") == "inactive", timeout_seconds=10)
|
||||
assert "boot already finished" in bootvine_journal(), "a restart after boot did not stand down"
|
||||
assert foreground_terminal() == foreground, "a restart after boot took the console"
|
||||
|
||||
machine.succeed("systemctl poweroff --no-block")
|
||||
retry(lambda _: unit_state("sample-database") == "deactivating", timeout_seconds=60)
|
||||
wait_for_screen("Shutdown", "sample-database")
|
||||
assert_bootvine_in_front()
|
||||
assert_text_visible(font_has_512_glyphs=True)
|
||||
machine.succeed("touch /run/release-sample-database")
|
||||
machine.wait_for_shutdown()
|
||||
@@ -0,0 +1,42 @@
|
||||
{ lib, pkgs, ... }:
|
||||
let
|
||||
sampleUnits = import ./sample-units.nix;
|
||||
|
||||
sampleService = spec: {
|
||||
inherit (spec) description;
|
||||
after = map (dependency: "${dependency}.service") (spec.after or [ ]);
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = [
|
||||
"${pkgs.coreutils}/bin/sleep ${spec.seconds}"
|
||||
]
|
||||
++ lib.optional (spec.fails or false) "${pkgs.coreutils}/bin/false";
|
||||
}
|
||||
// lib.optionalAttrs (spec ? stopSeconds) {
|
||||
RemainAfterExit = true;
|
||||
ExecStop = "${pkgs.coreutils}/bin/sleep ${spec.stopSeconds}";
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
boot.initrd.systemd.services.initrd-hardware-probe = {
|
||||
description = "Slow hardware probe";
|
||||
wantedBy = [ "initrd.target" ];
|
||||
before = [ "initrd.target" ];
|
||||
unitConfig.DefaultDependencies = false;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
ExecStart = "/bin/sleep ${sampleUnits.initrdProbeSeconds}";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services = lib.mapAttrs (_: sampleService) sampleUnits.services // {
|
||||
greetd.after = [ "multi-user.target" ];
|
||||
};
|
||||
|
||||
services.greetd = {
|
||||
enable = true;
|
||||
settings.default_session.command = "${pkgs.greetd}/bin/agreety --cmd ${pkgs.bashInteractive}/bin/bash";
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
sampleUnits,
|
||||
lib,
|
||||
coreutils,
|
||||
runCommand,
|
||||
writeText,
|
||||
}:
|
||||
let
|
||||
toUnitFile = lib.generators.toINI { listsAsDuplicateKeys = true; };
|
||||
|
||||
serviceFile =
|
||||
name: spec:
|
||||
writeText "${name}.service" (toUnitFile {
|
||||
Unit = {
|
||||
Description = spec.description;
|
||||
After = map (dependency: "${dependency}.service") (spec.after or [ ]);
|
||||
};
|
||||
Service = {
|
||||
Type = "oneshot";
|
||||
ExecStart = [
|
||||
"${coreutils}/bin/sleep ${spec.seconds}"
|
||||
]
|
||||
++ lib.optional (spec.fails or false) "${coreutils}/bin/false";
|
||||
}
|
||||
// lib.optionalAttrs (spec ? stopSeconds) {
|
||||
RemainAfterExit = true;
|
||||
ExecStop = "${coreutils}/bin/sleep ${spec.stopSeconds}";
|
||||
};
|
||||
});
|
||||
|
||||
serviceNames = map (name: "${name}.service") (lib.attrNames sampleUnits.services);
|
||||
|
||||
targetFile = writeText "${sampleUnits.target}.target" (toUnitFile {
|
||||
Unit = {
|
||||
Description = "bootvine demo services";
|
||||
Wants = serviceNames;
|
||||
After = serviceNames;
|
||||
};
|
||||
});
|
||||
unitFiles =
|
||||
lib.mapAttrsToList (name: spec: {
|
||||
name = "${name}.service";
|
||||
path = serviceFile name spec;
|
||||
}) sampleUnits.services
|
||||
++ [
|
||||
{
|
||||
name = "${sampleUnits.target}.target";
|
||||
path = targetFile;
|
||||
}
|
||||
];
|
||||
in
|
||||
runCommand "bootvine-sample-unit-files" { } (
|
||||
lib.concatMapStringsSep "\n" (unit: "install -Dm644 ${unit.path} $out/${unit.name}") unitFiles
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
target = "bootvine-sample-services";
|
||||
initrdProbeSeconds = "6";
|
||||
services = {
|
||||
scratch-disk-prepare = {
|
||||
description = "Scratch disk preparation";
|
||||
seconds = "2";
|
||||
stopSeconds = "2";
|
||||
};
|
||||
sample-database = {
|
||||
description = "Sample database";
|
||||
seconds = "3";
|
||||
stopSeconds = "3";
|
||||
after = [ "scratch-disk-prepare" ];
|
||||
};
|
||||
memory-cache = {
|
||||
description = "In-memory cache";
|
||||
seconds = "1.5";
|
||||
stopSeconds = "1";
|
||||
};
|
||||
web-frontend = {
|
||||
description = "Web frontend";
|
||||
seconds = "2";
|
||||
stopSeconds = "2";
|
||||
after = [
|
||||
"sample-database"
|
||||
"memory-cache"
|
||||
];
|
||||
};
|
||||
remote-content-sync = {
|
||||
description = "Remote content sync";
|
||||
seconds = "4";
|
||||
};
|
||||
flaky-health-check = {
|
||||
description = "Flaky health check";
|
||||
seconds = "1";
|
||||
fails = true;
|
||||
};
|
||||
slow-warm-up = {
|
||||
description = "Slow warm-up job";
|
||||
seconds = "8";
|
||||
};
|
||||
startup-report = {
|
||||
description = "Startup report";
|
||||
seconds = "1";
|
||||
after = [
|
||||
"web-frontend"
|
||||
"remote-content-sync"
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{ lib, ... }:
|
||||
{
|
||||
imports = [ ../slow-services/machine.nix ];
|
||||
|
||||
services.greetd.enable = lib.mkForce false;
|
||||
}
|
||||
Reference in new issue
Block a user