From 7e95cbd8e94d16b4969dfc0cf853be959a883664 Mon Sep 17 00:00:00 2001 From: Willem Vanhulle Date: Fri, 30 Oct 2026 13:38:59 +0100 Subject: [PATCH] feat: show why units failed, with their last journal lines Failed jobs keep their kind: units that broke or timed out get a failure panel entry with the reason systemd reports (exit status, signal, timeout, start limit, status text) and their last journal lines read through libsystemd. Units that failed only through a dependency are dimmed and left out of the panel, and units that crash after starting count as failed. Restructure the repository around this: - move the library and binary into bootvine/ and the scripted demos into their own unpublished scripted-demos crate, with their VHS tapes - move the crane builds and NixOS module into nix/ - group each sample VM with its check under vm-tests/, adding text-console, scripted-initrd, encrypted-volume and failures machines and a vm-failures check - add docs/guide.md walking through the screen with every recording, keep one recording in the README, and re-render all recordings --- Cargo.lock | 88 +++++++ Cargo.toml | 70 +----- README.md | 47 ++-- bootvine/Cargo.toml | 69 ++++++ bootvine/LICENSE | 1 + {src => bootvine/src}/boot_model.rs | 38 ++- .../src}/boot_model/boot_change.rs | 9 +- .../src}/boot_model/boot_origin.rs | 0 .../src}/boot_model/job_ingestion.rs | 133 ++++++++-- .../src}/boot_model/reached_milestones.rs | 8 +- .../src}/boot_model/unit_facts.rs | 2 +- .../src}/boot_model/waiting_tree.rs | 12 +- .../src}/boot_test_fixtures.rs | 0 {src => bootvine/src}/boot_view.rs | 72 +++++- bootvine/src/boot_view/failure_panel.rs | 227 ++++++++++++++++++ .../src}/boot_view/frame_cost_bench.rs | 2 +- .../src}/boot_view/panel_render_tests.rs | 83 ++++++- .../src}/boot_view/run_timeline.rs | 0 .../src}/boot_view/screen_chrome.rs | 0 .../src}/boot_view/screen_render_fixtures.rs | 2 +- .../src}/boot_view/stage_summary.rs | 0 .../src}/boot_view/tree_height_fitting.rs | 4 +- .../src}/boot_view/tree_render_tests.rs | 2 +- {src => bootvine/src}/boot_view/unit_tree.rs | 9 +- {src => bootvine/src}/console_session.rs | 0 .../src}/console_session/ask_password.rs | 0 .../src}/console_session/console_palette.rs | 0 .../src}/console_session/console_takeover.rs | 0 .../src}/console_session/live_desk.rs | 0 .../src}/console_session/passphrase_input.rs | 0 .../src}/console_session/stream_sources.rs | 0 .../src}/console_session/watch_loop.rs | 0 .../src}/console_session/watch_screen.rs | 2 +- {src => bootvine/src}/lib.rs | 3 +- {src => bootvine/src}/main.rs | 0 {src => bootvine/src}/screen_settings.rs | 0 .../src}/screen_settings/color_scheme.rs | 25 ++ .../src}/screen_settings/config_file.rs | 0 .../src}/screen_settings/font_glyphs.rs | 0 {src => bootvine/src}/system_install.rs | 0 .../src}/system_install/install_error.rs | 0 .../src}/system_install/install_files.rs | 0 {src => bootvine/src}/systemd_link.rs | 1 + .../src}/systemd_link/boot_feed.rs | 134 ++++++++++- bootvine/src/systemd_link/journal_tail.rs | 132 ++++++++++ .../src}/systemd_link/link_error.rs | 0 .../src}/systemd_link/monotonic_clock.rs | 0 .../src}/systemd_link/switch_root.rs | 0 .../src}/systemd_link/systemd_bus.rs | 102 +++++++- .../system-files}/dracut/.shellcheckrc | 0 .../dracut/90bootvine/module-setup.sh | 0 .../mkinitcpio/install/sd-bootvine | 0 .../systemd/bootvine-shutdown.service | 0 .../system-files}/systemd/bootvine.service | 0 docs/architecture.md | 45 +++- docs/configuration.md | 9 + docs/development.md | 41 +++- docs/guide.md | 157 ++++++++++++ docs/install.md | 4 +- flake.nix | 45 ++-- crane-builds.nix => nix/crane-builds.nix | 42 ++-- nixos-module.nix => nix/nixos-module.nix | 0 recordings/failures.png | Bin 0 -> 15468 bytes recordings/passphrase.png | Bin 9916 -> 11612 bytes scripted-demos/Cargo.toml | 17 ++ .../src/bin/scripted-boot.rs | 14 +- .../src/bin/scripted-luks-unlock.rs | 6 +- .../src/bin/scripted-shutdown.rs | 6 +- scripted-demos/src/lib.rs | 1 + .../src}/scripted_boot.rs | 59 +++-- .../tapes/scripted-boot.tape | 4 +- .../tapes/scripted-shutdown.tape | 4 +- src/demo_script.rs | 3 - tests/sample-machine/default.nix | 25 -- tests/sample-machine/machine.nix | 91 ------- tests/vm-checks/luks.nix | 49 ---- vm-tests/default.nix | 51 ++++ vm-tests/encrypted-volume/check.nix | 24 ++ .../encrypted-volume/check.py | 4 +- vm-tests/encrypted-volume/machine.nix | 11 + vm-tests/encrypted-volume/secret-disk.nix | 22 ++ vm-tests/failures/check.nix | 49 ++++ vm-tests/failures/check.py | 59 +++++ vm-tests/failures/machine.nix | 148 ++++++++++++ vm-tests/scripted-initrd/machine.nix | 6 + vm-tests/shared/common.nix | 45 ++++ .../vm-checks => vm-tests/shared}/console.py | 1 + .../shared}/interactive-vm.nix | 4 +- .../slow-services/check.nix | 18 +- .../slow-services/check.py | 16 +- vm-tests/slow-services/machine.nix | 42 ++++ .../slow-services}/sample-unit-files.nix | 0 .../slow-services}/sample-units.nix | 30 +-- vm-tests/text-console/machine.nix | 6 + xtask/src/main.rs | 11 +- xtask/src/record_screencasts.rs | 43 +++- 96 files changed, 2027 insertions(+), 462 deletions(-) create mode 100644 bootvine/Cargo.toml create mode 120000 bootvine/LICENSE rename {src => bootvine/src}/boot_model.rs (79%) rename {src => bootvine/src}/boot_model/boot_change.rs (91%) rename {src => bootvine/src}/boot_model/boot_origin.rs (100%) rename {src => bootvine/src}/boot_model/job_ingestion.rs (76%) rename {src => bootvine/src}/boot_model/reached_milestones.rs (96%) rename {src => bootvine/src}/boot_model/unit_facts.rs (98%) rename {src => bootvine/src}/boot_model/waiting_tree.rs (97%) rename {src/demo_script => bootvine/src}/boot_test_fixtures.rs (100%) rename {src => bootvine/src}/boot_view.rs (77%) create mode 100644 bootvine/src/boot_view/failure_panel.rs rename {src => bootvine/src}/boot_view/frame_cost_bench.rs (97%) rename {src => bootvine/src}/boot_view/panel_render_tests.rs (82%) rename {src => bootvine/src}/boot_view/run_timeline.rs (100%) rename {src => bootvine/src}/boot_view/screen_chrome.rs (100%) rename {src => bootvine/src}/boot_view/screen_render_fixtures.rs (98%) rename {src => bootvine/src}/boot_view/stage_summary.rs (100%) rename {src => bootvine/src}/boot_view/tree_height_fitting.rs (98%) rename {src => bootvine/src}/boot_view/tree_render_tests.rs (98%) rename {src => bootvine/src}/boot_view/unit_tree.rs (97%) rename {src => bootvine/src}/console_session.rs (100%) rename {src => bootvine/src}/console_session/ask_password.rs (100%) rename {src => bootvine/src}/console_session/console_palette.rs (100%) rename {src => bootvine/src}/console_session/console_takeover.rs (100%) rename {src => bootvine/src}/console_session/live_desk.rs (100%) rename {src => bootvine/src}/console_session/passphrase_input.rs (100%) rename {src => bootvine/src}/console_session/stream_sources.rs (100%) rename {src => bootvine/src}/console_session/watch_loop.rs (100%) rename {src => bootvine/src}/console_session/watch_screen.rs (99%) rename {src => bootvine/src}/lib.rs (78%) rename {src => bootvine/src}/main.rs (100%) rename {src => bootvine/src}/screen_settings.rs (100%) rename {src => bootvine/src}/screen_settings/color_scheme.rs (88%) rename {src => bootvine/src}/screen_settings/config_file.rs (100%) rename {src => bootvine/src}/screen_settings/font_glyphs.rs (100%) rename {src => bootvine/src}/system_install.rs (100%) rename {src => bootvine/src}/system_install/install_error.rs (100%) rename {src => bootvine/src}/system_install/install_files.rs (100%) rename {src => bootvine/src}/systemd_link.rs (85%) rename {src => bootvine/src}/systemd_link/boot_feed.rs (76%) create mode 100644 bootvine/src/systemd_link/journal_tail.rs rename {src => bootvine/src}/systemd_link/link_error.rs (100%) rename {src => bootvine/src}/systemd_link/monotonic_clock.rs (100%) rename {src => bootvine/src}/systemd_link/switch_root.rs (100%) rename {src => bootvine/src}/systemd_link/systemd_bus.rs (83%) rename {system-files => bootvine/system-files}/dracut/.shellcheckrc (100%) rename {system-files => bootvine/system-files}/dracut/90bootvine/module-setup.sh (100%) rename {system-files => bootvine/system-files}/mkinitcpio/install/sd-bootvine (100%) rename {system-files => bootvine/system-files}/systemd/bootvine-shutdown.service (100%) rename {system-files => bootvine/system-files}/systemd/bootvine.service (100%) create mode 100644 docs/guide.md rename crane-builds.nix => nix/crane-builds.nix (56%) rename nixos-module.nix => nix/nixos-module.nix (100%) create mode 100644 recordings/failures.png create mode 100644 scripted-demos/Cargo.toml rename examples/boot.rs => scripted-demos/src/bin/scripted-boot.rs (92%) rename examples/luks.rs => scripted-demos/src/bin/scripted-luks-unlock.rs (98%) rename examples/shutdown.rs => scripted-demos/src/bin/scripted-shutdown.rs (96%) create mode 100644 scripted-demos/src/lib.rs rename {src/demo_script => scripted-demos/src}/scripted_boot.rs (91%) rename recordings/boot.tape => scripted-demos/tapes/scripted-boot.tape (77%) rename recordings/shutdown.tape => scripted-demos/tapes/scripted-shutdown.tape (76%) delete mode 100644 src/demo_script.rs delete mode 100644 tests/sample-machine/default.nix delete mode 100644 tests/sample-machine/machine.nix delete mode 100644 tests/vm-checks/luks.nix create mode 100644 vm-tests/default.nix create mode 100644 vm-tests/encrypted-volume/check.nix rename tests/vm-checks/luks.py => vm-tests/encrypted-volume/check.py (91%) create mode 100644 vm-tests/encrypted-volume/machine.nix create mode 100644 vm-tests/encrypted-volume/secret-disk.nix create mode 100644 vm-tests/failures/check.nix create mode 100644 vm-tests/failures/check.py create mode 100644 vm-tests/failures/machine.nix create mode 100644 vm-tests/scripted-initrd/machine.nix create mode 100644 vm-tests/shared/common.nix rename {tests/vm-checks => vm-tests/shared}/console.py (98%) rename {tests/sample-machine => vm-tests/shared}/interactive-vm.nix (79%) rename tests/vm-checks/boot.nix => vm-tests/slow-services/check.nix (54%) rename tests/vm-checks/boot.py => vm-tests/slow-services/check.py (81%) create mode 100644 vm-tests/slow-services/machine.nix rename {tests/sample-machine => vm-tests/slow-services}/sample-unit-files.nix (100%) rename {tests/sample-machine => vm-tests/slow-services}/sample-units.nix (63%) create mode 100644 vm-tests/text-console/machine.nix diff --git a/Cargo.lock b/Cargo.lock index 3262ae3..55c2eda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,6 +158,7 @@ dependencies = [ "ser_nix", "serde", "serde_plain", + "systemd", "thiserror 2.0.21", "tokio", "toml", @@ -166,6 +167,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "build-env" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1522ac6ee801a11bf9ef3f80403f4ede6eb41291fac3dde3de09989679305f25" + [[package]] name = "bumpalo" version = "3.20.3" @@ -365,6 +372,16 @@ dependencies = [ "phf", ] +[[package]] +name = "cstr-argument" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bd9c8e659a473bce955ae5c35b116af38af11a7acb0b480e01f3ed348aeb40" +dependencies = [ + "cfg-if", + "memchr", +] + [[package]] name = "darling" version = "0.24.1" @@ -588,6 +605,33 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5190182e6915eb873ddbc16e23b711b6eb1f9c00a0d0a3a91b5f6228475225" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "futures" version = "0.3.34" @@ -880,6 +924,18 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libsystemd-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976306de183e6046819ef6505888d00996214766a3f4660a2ed5761c84a20aed" +dependencies = [ + "build-env", + "cfg-if", + "libc", + "pkg-config", +] + [[package]] name = "line-clipping" version = "0.3.8" @@ -1532,6 +1588,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scripted-demos" +version = "0.1.0" +dependencies = [ + "bootvine", + "futures", + "ratatui", + "tokio", + "zeroize", +] + [[package]] name = "semver" version = "1.0.28" @@ -1750,6 +1817,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "systemd" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e9d1976a15b86245def55d20d52b5818e1a1e81aa030b6a608d3ce57709423" +dependencies = [ + "cstr-argument", + "foreign-types", + "libc", + "libsystemd-sys", + "log", + "memchr", + "utf8-cstr", +] + [[package]] name = "termina" version = "0.3.3" @@ -1993,6 +2075,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "utf8-cstr" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55bcbb425141152b10d5693095950b51c3745d019363fc2929ffd8f61449b628" + [[package]] name = "utf8parse" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index 6dbf32e..52c9239 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,18 +1,6 @@ -[package] -categories = ["command-line-utilities", "os::linux-apis"] -description = "Boot and shutdown progress for systemd on the console, with each job nested under the units waiting on it" -edition = "2024" -include = ["/src", "/examples", "/system-files", "/README.md", "/LICENSE"] -keywords = ["systemd", "boot", "initrd", "tui", "console"] -license = "GPL-3.0-or-later" -name = "bootvine" -readme = "README.md" -repository = "https://codeberg.org/wvhulle/bootvine" -rust-version = "1.91" -version = "0.1.0" - [workspace] -members = [".", "xtask"] +members = ["bootvine", "scripted-demos", "xtask"] +resolver = "3" [workspace.lints.clippy] absolute_paths = "warn" @@ -20,60 +8,6 @@ missing_errors_doc = "allow" needless_for_each = "allow" pedantic = { priority = -1, level = "warn" } -[lints] -workspace = true - -[dependencies] -clap = { version = "4.6.7", default-features = false, features = [ - "std", - "derive", - "help", - "usage", - "error-context", -] } -crossterm = "0.29.0" -dbus = "0.9.12" -futures = { version = "0.3.34", default-features = false, features = [ - "std", - "async-await", -] } -heck = "0.5.0" -jiff = { version = "0.2.38", default-features = false, features = [ - "serde", - "std", -] } -petgraph = { version = "0.8.3", default-features = false, features = [ - "std", - "graphmap", -] } -ratatui = { version = "0.30.2", features = [ - "serde", - "unstable-rendered-line-info", -] } -rustix = { version = "1.1.5", features = [ - "event", - "fs", - "process", - "stdio", - "system", - "termios", - "time", -] } -ser_nix = "0.2.4" -serde = { version = "1.0.229", features = ["derive"] } -serde_plain = "1.0.2" -thiserror = "2.0.12" -tokio = { version = "1.54.0", default-features = false, features = [ - "rt", - "time", - "net", - "signal", -] } -toml = { version = "1.1.8", features = ["preserve_order"] } -unicode-truncate = "2.0.1" -unicode-width = "0.2.2" -zeroize = "1.9.1" - [profile.release] codegen-units = 1 lto = "fat" diff --git a/README.md b/README.md index d791983..6e8433a 100644 --- a/README.md +++ b/README.md @@ -11,21 +11,16 @@ gave me any idea of what was going on. A boot through the initrd into stage 2, with one failed unit -It follows shutdowns and reboots too: the units being stopped, nested under the -units that have to stop first. - -A poweroff: units stopping under the targets waiting on them - -Passphrase prompts, such as a -[LUKS](https://gitlab.com/cryptsetup/cryptsetup) unlock in the initrd, are -answered in place over the tree. - -A LUKS passphrase prompt floating over the initrd tree +It also follows shutdowns, answers passphrase prompts such as a +[LUKS](https://gitlab.com/cryptsetup/cryptsetup) unlock in place, and shows why +a unit failed with its last journal lines. The +[guide](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/guide.md) +walks through each part of the screen. ## Requirements - systemd ≥ 246 as PID 1 -- [libdbus](https://www.freedesktop.org/wiki/Software/dbus/) +- [libdbus](https://www.freedesktop.org/wiki/Software/dbus/) and libsystemd - a virtual terminal - a systemd initrd, to show the boot from the initrd as well as stage 2 @@ -34,33 +29,23 @@ for each distribution. ## Try it without installing -Neither of these touches your system's boot. - -Play a scripted boot, poweroff and encrypted-root boot in your terminal, with -[Rust](https://www.rust-lang.org/tools/install) installed: +With [Nix](https://nixos.org/download/), boot a sample [NixOS](https://nixos.org) machine with slow and +failing units in a [QEMU](https://www.qemu.org) window, without touching your +system's boot: ```sh -git clone https://codeberg.org/wvhulle/bootvine.git -cd bootvine -cargo run --example boot -cargo run --example shutdown -cargo run --example luks +nix run git+https://codeberg.org/wvhulle/bootvine#sample-vm-slow-services ``` -`q` quits. The encrypted root unlocks with the passphrase `demo`; `Esc` quits -while it asks. - -Or, with [Nix](https://nixos.org/download/), boot a sample [NixOS](https://nixos.org) machine with slow and -failing units in a [QEMU](https://www.qemu.org) window: - -```sh -nix run git+https://codeberg.org/wvhulle/bootvine#sample-machine-vm -``` - -It logs in as root; `poweroff` shows the shutdown. +It logs in as root; `poweroff` shows the shutdown. The +[guide](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/guide.md#trying-it) +lists the other sample machines and the scripted demos that play in a terminal. ## Documentation +- [Guide](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/guide.md) + to the screen: the tree, the timeline, failures, shutdown and passphrase + prompts. - [Install](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/install.md) on NixOS, Ubuntu and other distributions. - [Configuration](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/configuration.md) diff --git a/bootvine/Cargo.toml b/bootvine/Cargo.toml new file mode 100644 index 0000000..4745a15 --- /dev/null +++ b/bootvine/Cargo.toml @@ -0,0 +1,69 @@ +[package] +categories = ["command-line-utilities", "os::linux-apis"] +description = "Boot and shutdown progress for systemd on the console, with each job nested under the units waiting on it" +edition = "2024" +include = ["/src", "/system-files", "/LICENSE"] +keywords = ["systemd", "boot", "initrd", "tui", "console"] +license = "GPL-3.0-or-later" +name = "bootvine" +readme = "../README.md" +repository = "https://codeberg.org/wvhulle/bootvine" +rust-version = "1.91" +version = "0.1.0" + +[lints] +workspace = true + +[dependencies] +clap = { version = "4.6.7", default-features = false, features = [ + "std", + "derive", + "help", + "usage", + "error-context", +] } +crossterm = "0.29.0" +dbus = "0.9.12" +futures = { version = "0.3.34", default-features = false, features = [ + "std", + "async-await", +] } +heck = "0.5.0" +jiff = { version = "0.2.38", default-features = false, features = [ + "serde", + "std", +] } +petgraph = { version = "0.8.3", default-features = false, features = [ + "std", + "graphmap", +] } +ratatui = { version = "0.30.2", features = [ + "serde", + "unstable-rendered-line-info", +] } +rustix = { version = "1.1.5", features = [ + "event", + "fs", + "process", + "stdio", + "system", + "termios", + "time", +] } +ser_nix = "0.2.4" +serde = { version = "1.0.229", features = ["derive"] } +serde_plain = "1.0.2" +systemd = { version = "0.10.1", default-features = false, features = [ + "journal", +] } +thiserror = "2.0.12" +tokio = { version = "1.54.0", default-features = false, features = [ + "rt", + "time", + "net", + "signal", +] } +toml = { version = "1.1.8", features = ["preserve_order"] } +unicode-truncate = "2.0.1" +unicode-width = "0.2.2" +zeroize = "1.9.1" diff --git a/bootvine/LICENSE b/bootvine/LICENSE new file mode 120000 index 0000000..ea5b606 --- /dev/null +++ b/bootvine/LICENSE @@ -0,0 +1 @@ +../LICENSE \ No newline at end of file diff --git a/src/boot_model.rs b/bootvine/src/boot_model.rs similarity index 79% rename from src/boot_model.rs rename to bootvine/src/boot_model.rs index 4329907..f0628b8 100644 --- a/src/boot_model.rs +++ b/bootvine/src/boot_model.rs @@ -41,10 +41,44 @@ struct PendingJob { kind: JobKind, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FailureKind { + Broke, + TimedOut, + DependencyFailed, +} + +impl FailureKind { + #[must_use] + pub fn has_own_cause(self) -> bool { + self != Self::DependencyFailed + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FailureCause { + pub reason: String, + pub log_lines: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FailureExplanation { + Unexplained, + Explained(FailureCause), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FailedUnit { + pub unit: String, + pub label: String, + pub kind: FailureKind, + pub explanation: FailureExplanation, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Outcome { Succeeded { at: Duration }, - Failed, + Failed(FailureKind), Abandoned, } @@ -109,7 +143,7 @@ pub struct BootState { pub phase: Phase, pub origin: Duration, pub root: Root, - pub failed: Vec, + pub failed: Vec, pub finished_stages: Vec, pub finished_runs: Vec, pub handoff_unit: Option, diff --git a/src/boot_model/boot_change.rs b/bootvine/src/boot_model/boot_change.rs similarity index 91% rename from src/boot_model/boot_change.rs rename to bootvine/src/boot_model/boot_change.rs index 1c63e08..84b9f49 100644 --- a/src/boot_model/boot_change.rs +++ b/bootvine/src/boot_model/boot_change.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use super::{Activity, BootState, JobKind, JobResult, UnitFacts}; +use super::{Activity, BootState, FailureCause, JobKind, JobResult, UnitFacts}; pub enum BootChange { TrackUnit { @@ -22,6 +22,10 @@ pub enum BootChange { result: JobResult, at: Duration, }, + ExplainFailure { + unit: String, + cause: FailureCause, + }, SetRoot(String), SetHandoff(String), SetOrigin(Duration), @@ -42,6 +46,7 @@ impl BootState { BootChange::UpdateActivity { unit, activity } => self.update_activity(&unit, activity), BootChange::QueueJob { id, unit, kind, at } => self.queue_job(id, &unit, kind, at), BootChange::FinishJob { id, result, at } => self.finish_job(id, result, at), + BootChange::ExplainFailure { unit, cause } => self.explain_failure(&unit, cause), BootChange::SetRoot(root) => self.set_root(root), BootChange::SetHandoff(unit) => { if let Some(handoff) = &mut self.handoff_unit { @@ -62,7 +67,7 @@ mod tests { use super::*; use crate::{ boot_model::Phase, - demo_script::boot_test_fixtures::{MS, facts, waiting_rows}, + boot_test_fixtures::{MS, facts, waiting_rows}, }; #[test] diff --git a/src/boot_model/boot_origin.rs b/bootvine/src/boot_model/boot_origin.rs similarity index 100% rename from src/boot_model/boot_origin.rs rename to bootvine/src/boot_model/boot_origin.rs diff --git a/src/boot_model/job_ingestion.rs b/bootvine/src/boot_model/job_ingestion.rs similarity index 76% rename from src/boot_model/job_ingestion.rs rename to bootvine/src/boot_model/job_ingestion.rs index a5ef5f5..42266ac 100644 --- a/src/boot_model/job_ingestion.rs +++ b/bootvine/src/boot_model/job_ingestion.rs @@ -1,9 +1,10 @@ use std::{collections::HashMap, time::Duration}; use super::{ - ActiveState, Activity, BootState, FinishedRun, FinishedStage, HANDOFF_UNIT, HOST_MILESTONES, - JobKind, JobResult, Outcome, PendingJob, Phase, Progress, Root, SHUTDOWN_MILESTONES, Tally, - UnitFacts, UnitKind, UnitRecord, unit_facts::display_label, + ActiveState, Activity, BootState, FailedUnit, FailureCause, FailureExplanation, FailureKind, + FinishedRun, FinishedStage, HANDOFF_UNIT, HOST_MILESTONES, JobKind, JobResult, Outcome, + PendingJob, Phase, Progress, Root, SHUTDOWN_MILESTONES, Tally, UnitFacts, UnitKind, UnitRecord, + unit_facts::display_label, }; impl UnitRecord { @@ -116,11 +117,40 @@ impl BootState { } pub fn update_activity(&mut self, unit: &str, activity: Activity) { - if let Some(record) = self.units.get_mut(unit) { - record.activity = record.activity.merge(activity); + let Some(record) = self.units.get_mut(unit) else { + return; + }; + record.activity = record.activity.merge(activity); + if activity.state == Some(ActiveState::Failed) { + let label = display_label(&record.facts); + self.record_failure(unit, label, FailureKind::Broke); } } + fn record_failure(&mut self, unit: &str, label: String, kind: FailureKind) { + match self.failed.iter_mut().find(|failed| failed.unit == unit) { + Some(known) if kind != FailureKind::Broke => known.kind = kind, + Some(_) => {} + None => self.failed.push(FailedUnit { + unit: unit.to_owned(), + label, + kind, + explanation: FailureExplanation::Unexplained, + }), + } + } + + pub fn explain_failure(&mut self, unit: &str, cause: FailureCause) { + if let Some(failed) = self.failed.iter_mut().find(|failed| failed.unit == unit) { + failed.explanation = FailureExplanation::Explained(cause); + } + } + + #[must_use] + pub fn has_failed(&self, unit: &str) -> bool { + self.failed.iter().any(|failed| failed.unit == unit) + } + pub fn queue_job(&mut self, id: u32, unit: &str, kind: JobKind, at: Duration) { if self.enqueue(id, unit, kind, at) && kind == JobKind::BringsUp @@ -150,18 +180,26 @@ impl BootState { let Some(unit) = self.jobs.remove(&id) else { return; }; - let Some(record) = self.units.get_mut(&unit) else { + let Some(label) = self + .units + .get(&unit) + .map(|record| display_label(&record.facts)) + else { return; }; let outcome = match result { JobResult::Done => Outcome::Succeeded { at }, JobResult::Skipped | JobResult::Canceled => Outcome::Abandoned, - JobResult::Failed | JobResult::Timeout | JobResult::Dependency => Outcome::Failed, + JobResult::Failed => Outcome::Failed(FailureKind::Broke), + JobResult::Timeout => Outcome::Failed(FailureKind::TimedOut), + JobResult::Dependency => Outcome::Failed(FailureKind::DependencyFailed), }; - let label = display_label(&record.facts); - if outcome == Outcome::Failed && !self.failed.contains(&label) { - self.failed.push(label.clone()); + if let Outcome::Failed(kind) = outcome { + self.record_failure(&unit, label.clone(), kind); } + let Some(record) = self.units.get_mut(&unit) else { + return; + }; let job = match record.progress { Progress::Pending(job) if job.id == id => job, _ => return, @@ -176,7 +214,7 @@ impl BootState { label, started_at, finished_at: at, - failed: outcome == Outcome::Failed, + failed: matches!(outcome, Outcome::Failed(_)), }); } } @@ -193,7 +231,7 @@ impl BootState { failed_units: self .units .values() - .filter(|record| record.progress == Progress::Finished(Outcome::Failed)) + .filter(|record| matches!(record.progress, Progress::Finished(Outcome::Failed(_)))) .count(), started_at: self.origin, reached_at: at, @@ -254,7 +292,7 @@ impl BootState { Progress::Finished(Outcome::Succeeded { .. } | Outcome::Abandoned) => { tally.done += 1; } - Progress::Finished(Outcome::Failed) | Progress::Untouched => {} + Progress::Finished(Outcome::Failed(_)) | Progress::Untouched => {} } tally }) @@ -266,17 +304,82 @@ mod tests { use super::*; use crate::{ boot_model::{BootChange, RowStatus, UnitTimeouts}, - demo_script::boot_test_fixtures::{ + boot_test_fixtures::{ MS, activating, facts, nested_boot_sample as sample, queue_unit, queued_boot, running, waiting, waiting_rows, }, }; + fn failure_kinds(state: &BootState) -> Vec<(&str, FailureKind)> { + state + .failed + .iter() + .map(|failed| (failed.unit.as_str(), failed.kind)) + .collect() + } + + #[test] + fn job_results_keep_their_failure_kind() { + let mut state = sample(); + state.finish_job(4, JobResult::Timeout, 2000 * MS); + state.finish_job(3, JobResult::Dependency, 2000 * MS); + assert_eq!( + failure_kinds(&state), + [ + ("nm.service", FailureKind::TimedOut), + ("nm-wait.service", FailureKind::DependencyFailed), + ] + ); + } + + #[test] + fn a_unit_entering_the_failed_state_counts_without_a_failed_job() { + let mut state = sample(); + state.update_activity( + "nm.service", + Activity { + state: Some(ActiveState::Failed), + ..Activity::default() + }, + ); + assert_eq!(failure_kinds(&state), [("nm.service", FailureKind::Broke)]); + state.finish_job(4, JobResult::Timeout, 2000 * MS); + assert_eq!( + failure_kinds(&state), + [("nm.service", FailureKind::TimedOut)] + ); + assert_eq!(state.tally().failed, 1); + } + + #[test] + fn an_explanation_attaches_to_its_failed_unit() { + let mut state = sample(); + let cause = FailureCause { + reason: "exited with status 1".to_owned(), + log_lines: vec!["no carrier".to_owned()], + }; + state.explain_failure("nm.service", cause.clone()); + assert!(state.failed.is_empty()); + state.finish_job(4, JobResult::Failed, 2000 * MS); + state.explain_failure("nm.service", cause.clone()); + assert_eq!( + state.failed[0].explanation, + FailureExplanation::Explained(cause) + ); + } + #[test] fn failures_are_listed_and_not_counted_as_done() { let mut state = sample(); state.finish_job(4, JobResult::Failed, 2000 * MS); - assert_eq!(state.failed, ["nm"]); + assert_eq!( + state + .failed + .iter() + .map(|failed| failed.label.as_str()) + .collect::>(), + ["nm"] + ); let tally = state.tally(); assert_eq!(tally.failed, 1); assert_eq!(tally.done, 0); diff --git a/src/boot_model/reached_milestones.rs b/bootvine/src/boot_model/reached_milestones.rs similarity index 96% rename from src/boot_model/reached_milestones.rs rename to bootvine/src/boot_model/reached_milestones.rs index 48cfa16..efa0bf8 100644 --- a/src/boot_model/reached_milestones.rs +++ b/bootvine/src/boot_model/reached_milestones.rs @@ -23,7 +23,9 @@ impl BootState { Progress::Untouched => record.activity.active_enter.filter(|entered| { record.activity.state == Some(ActiveState::Active) && *entered >= self.origin }), - Progress::Pending(_) | Progress::Finished(Outcome::Failed | Outcome::Abandoned) => None, + Progress::Pending(_) | Progress::Finished(Outcome::Failed(_) | Outcome::Abandoned) => { + None + } } } @@ -81,9 +83,7 @@ mod tests { use super::*; use crate::{ boot_model::{Activity, JobResult, Phase}, - demo_script::boot_test_fixtures::{ - MS, nested_boot_sample, queued_boot, waiting, waiting_rows, - }, + boot_test_fixtures::{MS, nested_boot_sample, queued_boot, waiting, waiting_rows}, }; #[test] diff --git a/src/boot_model/unit_facts.rs b/bootvine/src/boot_model/unit_facts.rs similarity index 98% rename from src/boot_model/unit_facts.rs rename to bootvine/src/boot_model/unit_facts.rs index d986ffc..b4dc35b 100644 --- a/src/boot_model/unit_facts.rs +++ b/bootvine/src/boot_model/unit_facts.rs @@ -152,7 +152,7 @@ mod tests { use serde_plain::from_str; use super::*; - use crate::demo_script::boot_test_fixtures::{facts, waiting}; + use crate::boot_test_fixtures::{facts, waiting}; #[test] fn unknown_systemd_strings_fall_back_instead_of_failing() { diff --git a/src/boot_model/waiting_tree.rs b/bootvine/src/boot_model/waiting_tree.rs similarity index 97% rename from src/boot_model/waiting_tree.rs rename to bootvine/src/boot_model/waiting_tree.rs index 7cf6115..0cfc78a 100644 --- a/src/boot_model/waiting_tree.rs +++ b/bootvine/src/boot_model/waiting_tree.rs @@ -7,7 +7,7 @@ use petgraph::{ }; use super::{ - BootState, Outcome, Phase, Progress, UnitKind, UnitRecord, + BootState, FailureKind, Outcome, Phase, Progress, UnitKind, UnitRecord, unit_facts::{display_label, summary}, }; @@ -31,7 +31,7 @@ pub enum RowStatus { since: Duration, deadline: Option, }, - Failed, + Failed(FailureKind), Done, } @@ -87,7 +87,7 @@ impl UnitRecord { deadline: self.deadline(), }) } - Progress::Finished(Outcome::Failed) => RowStatus::Failed, + Progress::Finished(Outcome::Failed(kind)) => RowStatus::Failed(kind), Progress::Finished(_) | Progress::Untouched => RowStatus::Done, } } @@ -100,7 +100,7 @@ impl BootState { self.units .iter() .filter(|(name, record)| match record.progress { - Progress::Pending(_) | Progress::Finished(Outcome::Failed) => true, + Progress::Pending(_) | Progress::Finished(Outcome::Failed(_)) => true, Progress::Finished(Outcome::Succeeded { at }) => { at >= finished_since || self.is_milestone(name) } @@ -240,7 +240,7 @@ impl BootState { Progress::Finished(Outcome::Succeeded { at }) => { SiblingRank::RunningSince(record.activity.inactive_exit.unwrap_or(at)) } - Progress::Finished(Outcome::Failed) => SiblingRank::Failed, + Progress::Finished(Outcome::Failed(_)) => SiblingRank::Failed, Progress::Finished(Outcome::Abandoned) | Progress::Untouched => { SiblingRank::Waiting } @@ -253,7 +253,7 @@ impl BootState { #[cfg(test)] mod tests { use super::*; - use crate::demo_script::boot_test_fixtures::{ + use crate::boot_test_fixtures::{ nested_boot_sample, queued_boot, tree_outline, waiting, waiting_rows, }; diff --git a/src/demo_script/boot_test_fixtures.rs b/bootvine/src/boot_test_fixtures.rs similarity index 100% rename from src/demo_script/boot_test_fixtures.rs rename to bootvine/src/boot_test_fixtures.rs diff --git a/src/boot_view.rs b/bootvine/src/boot_view.rs similarity index 77% rename from src/boot_view.rs rename to bootvine/src/boot_view.rs index 488f292..cf2bf57 100644 --- a/src/boot_view.rs +++ b/bootvine/src/boot_view.rs @@ -1,3 +1,4 @@ +mod failure_panel; #[cfg(test)] mod frame_cost_bench; #[cfg(test)] @@ -27,6 +28,7 @@ use unicode_width::UnicodeWidthStr; pub use self::screen_chrome::PasswordPrompt; use self::{ + failure_panel::FailurePanel, run_timeline::{RunningRun, Timeline}, screen_chrome::{FOOTER_HEIGHT, Footer, Header, PasswordPopup}, stage_summary::Stages, @@ -46,6 +48,9 @@ const MIN_BAR_WIDTH: u16 = 12; const MIN_HEIGHT_FOR_TIMELINE: u16 = 12; const MIN_TIMELINE_HEIGHT: u16 = run_timeline::CHROME_HEIGHT + 4; const HEADER_HEIGHT: u16 = 1; +const MIN_FAILURE_PANEL_HEIGHT: u16 = failure_panel::RULE_HEIGHT + 1; +const RESERVED_FAILURE_PANEL_HEIGHT: u16 = failure_panel::RULE_HEIGHT + 3; +const MIN_TREE_HEIGHT_BESIDE_FAILURES: u16 = 4; const BRANCH: &str = "├─ "; const LAST_BRANCH: &str = "└─ "; const PASSING_TRUNK: &str = "│ "; @@ -75,6 +80,20 @@ pub struct BootView<'a> { } impl BootView<'_> { + fn failure_panel(&self) -> FailurePanel<'_> { + FailurePanel { + newest_first: self + .state + .failed + .iter() + .rev() + .filter(|failed| failed.kind.has_own_cause()) + .collect(), + colors: &self.colors.failures, + glyphs: self.glyphs, + } + } + fn timeline<'a>(&'a self, rows: &'a [TreeRow]) -> Timeline<'a> { let mut running: Vec = rows .iter() @@ -135,6 +154,7 @@ impl Widget for BootView<'_> { glyphs: self.glyphs, }; let timeline = self.timeline(&all_rows); + let failure_panel = self.failure_panel(); let footer = Footer { tally: self.state.tally(), colors: &self.colors.footer, @@ -148,6 +168,7 @@ impl Widget for BootView<'_> { let heights = Heights::share( free, UnitTree::height_of(pending_count, 0), + failure_panel.height(), if fits_timeline { timeline.height() } else { 0 }, ); let budget = usize::from(heights.tree.saturating_sub(1)); @@ -168,6 +189,7 @@ impl Widget for BootView<'_> { stage_area, tree_area, _, + failure_area, timeline_area, footer_area, ] = Layout::vertical([ @@ -175,6 +197,7 @@ impl Widget for BootView<'_> { Constraint::Length(stages.height()), Constraint::Length(tree_height), Constraint::Fill(1), + Constraint::Length(heights.failures), Constraint::Length(heights.timeline), Constraint::Length(FOOTER_HEIGHT), ]) @@ -183,6 +206,9 @@ impl Widget for BootView<'_> { header.render(header_area, buf); stages.render(stage_area, buf); tree.render(tree_area, buf); + if heights.failures > 0 { + failure_panel.render(failure_area, buf); + } if heights.timeline > 0 { timeline.render(timeline_area, buf); } @@ -200,19 +226,32 @@ impl Widget for BootView<'_> { struct Heights { tree: u16, + failures: u16, timeline: u16, } impl Heights { - fn share(free: u16, tree_need: u16, timeline_need: u16) -> Self { - let leftover = free.saturating_sub(tree_need); + fn share(free: u16, tree_need: u16, failures_need: u16, timeline_need: u16) -> Self { + let reserved = RESERVED_FAILURE_PANEL_HEIGHT + .min(failures_need) + .min(free.saturating_sub(MIN_TREE_HEIGHT_BESIDE_FAILURES)); + let failures = failures_need + .min(free.saturating_sub(tree_need)) + .max(reserved); + let failures = if failures >= MIN_FAILURE_PANEL_HEIGHT { + failures + } else { + 0 + }; + let leftover = free.saturating_sub(tree_need).saturating_sub(failures); let timeline = if leftover >= MIN_TIMELINE_HEIGHT { timeline_need.min(leftover) } else { 0 }; Self { - tree: free - timeline, + tree: free - failures - timeline, + failures, timeline, } } @@ -264,13 +303,32 @@ mod tests { #[test] fn the_tree_takes_the_room_and_the_timeline_only_what_is_left() { - let heights = Heights::share(30, 40, 50); + let heights = Heights::share(30, 40, 0, 50); assert_eq!((heights.tree, heights.timeline), (30, 0)); - let heights = Heights::share(30, 12, 50); + let heights = Heights::share(30, 12, 0, 50); assert_eq!((heights.tree, heights.timeline), (12, 18)); - let heights = Heights::share(30, 25, 50); + let heights = Heights::share(30, 25, 0, 50); assert_eq!((heights.tree, heights.timeline), (30, 0)); - let heights = Heights::share(30, 12, 8); + let heights = Heights::share(30, 12, 0, 8); assert_eq!((heights.tree, heights.timeline), (22, 8)); } + + #[test] + fn failures_come_before_the_timeline_and_keep_a_few_rows_beside_a_full_tree() { + let heights = Heights::share(30, 12, 6, 50); + assert_eq!( + (heights.tree, heights.failures, heights.timeline), + (12, 6, 12) + ); + let heights = Heights::share(30, 40, 9, 50); + assert_eq!( + (heights.tree, heights.failures, heights.timeline), + (26, 4, 0) + ); + let heights = Heights::share(5, 40, 9, 50); + assert_eq!( + (heights.tree, heights.failures, heights.timeline), + (5, 0, 0) + ); + } } diff --git a/bootvine/src/boot_view/failure_panel.rs b/bootvine/src/boot_view/failure_panel.rs new file mode 100644 index 0000000..8b8b9b9 --- /dev/null +++ b/bootvine/src/boot_view/failure_panel.rs @@ -0,0 +1,227 @@ +use std::iter::once; + +use ratatui::{ + buffer::Buffer, + layout::Rect, + style::Stylize, + text::{Line, Span}, + widgets::{Paragraph, Widget}, +}; +use unicode_width::UnicodeWidthStr; + +use super::{fit_width, rule}; +use crate::{ + boot_model::{FailedUnit, FailureExplanation}, + screen_settings::{color_scheme::FailureColors, font_glyphs::Glyphs}, +}; + +pub const RULE_HEIGHT: u16 = 1; +const LOG_INDENT: &str = " "; +const REASON_GAP: &str = " "; + +pub struct FailurePanel<'a> { + pub newest_first: Vec<&'a FailedUnit>, + pub colors: &'a FailureColors, + pub glyphs: &'a Glyphs, +} + +fn log_lines(failed: &FailedUnit) -> &[String] { + match &failed.explanation { + FailureExplanation::Explained(cause) => &cause.log_lines, + FailureExplanation::Unexplained => &[], + } +} + +impl FailurePanel<'_> { + pub fn height(&self) -> u16 { + if self.newest_first.is_empty() { + return 0; + } + let rows: usize = self + .newest_first + .iter() + .map(|failed| 1 + log_lines(failed).len()) + .sum(); + RULE_HEIGHT.saturating_add(u16::try_from(rows).unwrap_or(u16::MAX)) + } + + fn headline(&self, failed: &FailedUnit, width: usize) -> Line<'static> { + let marker = format!("{} ", self.glyphs.failed); + let label_room = width.saturating_sub(marker.width()); + let label = fit_width(&failed.label, label_room, self.glyphs.ellipsis); + let reason_room = label_room.saturating_sub(label.width() + REASON_GAP.width()); + let reason = match &failed.explanation { + FailureExplanation::Explained(cause) if reason_room > 0 => { + fit_width(&cause.reason, reason_room, self.glyphs.ellipsis) + } + _ => String::new(), + }; + let reason_span = + (!reason.is_empty()).then(|| format!("{REASON_GAP}{reason}").fg(self.colors.reason)); + Line::from( + [ + marker.fg(self.colors.unit), + label.fg(self.colors.unit).bold(), + ] + .into_iter() + .chain(reason_span) + .collect::>(), + ) + } + + fn block(&self, failed: &FailedUnit, width: usize) -> Vec> { + let log_room = width.saturating_sub(LOG_INDENT.width()); + once(self.headline(failed, width)) + .chain(log_lines(failed).iter().map(|line| { + format!( + "{LOG_INDENT}{}", + fit_width(line, log_room, self.glyphs.ellipsis) + ) + .fg(self.colors.log) + .into() + })) + .collect() + } + + fn lines_within(&self, rows: usize, width: usize) -> Vec> { + if self.newest_first.len() > rows { + return self.headlines_with_overflow(rows, width); + } + let mut spare_rows = rows - self.newest_first.len(); + self.newest_first + .iter() + .flat_map(|failed| { + let mut block = self.block(failed, width); + let kept_log_lines = (block.len() - 1).min(spare_rows); + spare_rows -= kept_log_lines; + block.truncate(1 + kept_log_lines); + block + }) + .collect() + } + + fn headlines_with_overflow(&self, rows: usize, width: usize) -> Vec> { + let Some(headline_rows) = rows.checked_sub(1) else { + return Vec::new(); + }; + let hidden = self.newest_first.len() - headline_rows; + self.newest_first + .iter() + .take(headline_rows) + .map(|failed| self.headline(failed, width)) + .chain(once( + Line::from(format!("{} {hidden} more failed", self.glyphs.ellipsis)) + .fg(self.colors.overflow), + )) + .collect() + } +} + +impl Widget for FailurePanel<'_> { + fn render(self, area: Rect, buf: &mut Buffer) { + let block = rule( + self.colors.rule, + Line::from(" Failures ".fg(self.colors.title)), + ); + let inner = block.inner(area); + block.render(area, buf); + Paragraph::new(self.lines_within(usize::from(inner.height), usize::from(inner.width))) + .render(inner, buf); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::boot_model::{FailureCause, FailureKind}; + + fn explained(label: &str, log_lines: &[&str]) -> FailedUnit { + FailedUnit { + unit: format!("{label}.service"), + label: label.to_owned(), + kind: FailureKind::Broke, + explanation: FailureExplanation::Explained(FailureCause { + reason: "exited with status 1".to_owned(), + log_lines: log_lines.iter().map(ToString::to_string).collect(), + }), + } + } + + fn panel_text(failures: &[FailedUnit], rows: usize) -> Vec { + FailurePanel { + newest_first: failures.iter().collect(), + colors: &FailureColors::default(), + glyphs: &Glyphs::fitting(|_| true), + } + .lines_within(rows, 60) + .iter() + .map(ToString::to_string) + .collect() + } + + #[test] + fn the_newest_failure_keeps_its_lines_and_older_ones_shrink_to_a_headline() { + let failures = [ + explained("web", &["bind: address in use", "giving up"]), + explained("db", &["disk full"]), + explained("cache", &["no memory"]), + ]; + assert_eq!( + panel_text(&failures, 5), + [ + "× web exited with status 1", + " bind: address in use", + " giving up", + "× db exited with status 1", + "× cache exited with status 1", + ] + ); + } + + #[test] + fn room_left_by_the_newest_failure_goes_to_older_ones() { + let failures = [ + explained("web", &["giving up"]), + explained("db", &["disk full", "read-only file system"]), + explained("cache", &["no memory"]), + ]; + assert_eq!( + panel_text(&failures, 6), + [ + "× web exited with status 1", + " giving up", + "× db exited with status 1", + " disk full", + " read-only file system", + "× cache exited with status 1", + ] + ); + } + + #[test] + fn failures_beyond_the_room_are_counted() { + let failures = [ + explained("web", &["giving up"]), + explained("db", &[]), + explained("cache", &[]), + explained("queue", &[]), + ]; + assert_eq!( + panel_text(&failures, 3), + [ + "× web exited with status 1", + "× db exited with status 1", + "… 2 more failed" + ] + ); + } + + #[test] + fn an_unexplained_failure_shows_only_its_name() { + let failures = [FailedUnit { + explanation: FailureExplanation::Unexplained, + ..explained("web", &[]) + }]; + assert_eq!(panel_text(&failures, 4), ["× web"]); + } +} diff --git a/src/boot_view/frame_cost_bench.rs b/bootvine/src/boot_view/frame_cost_bench.rs similarity index 97% rename from src/boot_view/frame_cost_bench.rs rename to bootvine/src/boot_view/frame_cost_bench.rs index f2d1efd..3b8d989 100644 --- a/src/boot_view/frame_cost_bench.rs +++ b/bootvine/src/boot_view/frame_cost_bench.rs @@ -9,7 +9,7 @@ use ratatui::{Terminal, backend::TestBackend}; use super::{BootProjection, BootView, project_boot, screen_render_fixtures::ryzen_origin}; use crate::{ boot_model::{BootState, JobResult, Phase, flatten_forest}, - demo_script::boot_test_fixtures::{MS, QueuedUnit, queued_boot, running, waiting}, + boot_test_fixtures::{MS, QueuedUnit, queued_boot, running, waiting}, screen_settings::{color_scheme::Colors, config_file::Timing, font_glyphs::Glyphs}, }; diff --git a/src/boot_view/panel_render_tests.rs b/bootvine/src/boot_view/panel_render_tests.rs similarity index 82% rename from src/boot_view/panel_render_tests.rs rename to bootvine/src/boot_view/panel_render_tests.rs index 98193fe..a770195 100644 --- a/src/boot_view/panel_render_tests.rs +++ b/bootvine/src/boot_view/panel_render_tests.rs @@ -5,8 +5,8 @@ use super::{ screen_render_fixtures::{boot_in_progress, render, render_prompted, screen_lines}, }; use crate::{ - boot_model::{ActiveState, Activity, BootState, HANDOFF_UNIT, JobResult, Phase}, - demo_script::boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting}, + boot_model::{ActiveState, Activity, BootState, FailureCause, HANDOFF_UNIT, JobResult, Phase}, + boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting}, screen_settings::color_scheme::Colors, }; @@ -391,3 +391,82 @@ fn the_timeline_keeps_only_the_slowest_finished_runs() { "{screen}" ); } + +fn explained_nvidia_failure() -> BootState { + let mut state = boot_in_progress(); + state.explain_failure( + "nvidia-powerd.service", + FailureCause { + reason: "exited with status 1".to_owned(), + log_lines: vec![ + "ERROR! Failed to get system power limits".to_owned(), + "giving up".to_owned(), + ], + }, + ); + state +} + +#[test] +fn failures_show_their_reason_and_journal_lines_above_the_timeline() { + let lines = render(&explained_nvidia_failure(), 12_400 * MS, 110, 30); + let screen = lines.join("\n"); + let failures = lines + .iter() + .position(|line| line.contains(" Failures ")) + .expect("failure panel"); + let timeline = lines + .iter() + .position(|line| line.contains(" Timeline ")) + .expect("timeline panel"); + assert!(failures < timeline, "{screen}"); + assert_eq!( + lines[failures + 1], + "× nvidia-powerd exited with status 1", + "{screen}" + ); + assert_eq!( + lines[failures + 2], + " ERROR! Failed to get system power limits", + "{screen}" + ); + assert_eq!(lines[failures + 3], " giving up", "{screen}"); +} + +#[test] +fn failure_lines_are_cut_to_a_narrow_screen() { + let lines = render(&explained_nvidia_failure(), 12_400 * MS, 30, 30); + let screen = lines.join("\n"); + let failures = lines + .iter() + .position(|line| line.contains(" Failures ")) + .expect("failure panel"); + assert_eq!( + lines[failures + 1], + "× nvidia-powerd exited with …", + "{screen}" + ); + assert_eq!( + lines[failures + 2], + " ERROR! Failed to get syst…", + "{screen}" + ); +} + +#[test] +fn dependency_failures_stay_out_of_the_failure_panel() { + let mut state = queued_boot( + Phase::Host, + 1000 * MS, + [ + waiting("multi-user.target", &["web.service"]), + waiting("web.service", &[]), + ], + ); + state.finish_job(2, JobResult::Dependency, 2000 * MS); + let lines = render(&state, 3000 * MS, 110, 24); + let screen = lines.join("\n"); + assert!(!screen.contains(" Failures "), "{screen}"); + assert!(screen.contains("[×] web"), "{screen}"); + assert!(screen.contains("× 1 failed"), "{screen}"); +} diff --git a/src/boot_view/run_timeline.rs b/bootvine/src/boot_view/run_timeline.rs similarity index 100% rename from src/boot_view/run_timeline.rs rename to bootvine/src/boot_view/run_timeline.rs diff --git a/src/boot_view/screen_chrome.rs b/bootvine/src/boot_view/screen_chrome.rs similarity index 100% rename from src/boot_view/screen_chrome.rs rename to bootvine/src/boot_view/screen_chrome.rs diff --git a/src/boot_view/screen_render_fixtures.rs b/bootvine/src/boot_view/screen_render_fixtures.rs similarity index 98% rename from src/boot_view/screen_render_fixtures.rs rename to bootvine/src/boot_view/screen_render_fixtures.rs index feff27a..ee322a1 100644 --- a/src/boot_view/screen_render_fixtures.rs +++ b/bootvine/src/boot_view/screen_render_fixtures.rs @@ -5,7 +5,7 @@ use ratatui::{Terminal, backend::TestBackend, buffer::Buffer}; use super::{BootView, PasswordPrompt, project_boot}; use crate::{ boot_model::{BootState, JobResult, Phase, boot_origin::BootOrigin}, - demo_script::boot_test_fixtures::{MS, queued_boot, running, waiting}, + boot_test_fixtures::{MS, queued_boot, running, waiting}, screen_settings::{color_scheme::Colors, config_file::Timing, font_glyphs::Glyphs}, }; diff --git a/src/boot_view/stage_summary.rs b/bootvine/src/boot_view/stage_summary.rs similarity index 100% rename from src/boot_view/stage_summary.rs rename to bootvine/src/boot_view/stage_summary.rs diff --git a/src/boot_view/tree_height_fitting.rs b/bootvine/src/boot_view/tree_height_fitting.rs similarity index 98% rename from src/boot_view/tree_height_fitting.rs rename to bootvine/src/boot_view/tree_height_fitting.rs index f63c22e..16fe8db 100644 --- a/src/boot_view/tree_height_fitting.rs +++ b/bootvine/src/boot_view/tree_height_fitting.rs @@ -34,7 +34,7 @@ impl TreeNode { fn leads_to_running(&self) -> bool { matches!( self.row.status, - RowStatus::Running { .. } | RowStatus::Failed + RowStatus::Running { .. } | RowStatus::Failed(_) ) || self.children.iter().any(Self::leads_to_running) } @@ -95,7 +95,7 @@ mod tests { use super::*; use crate::{ boot_model::{BootState, Phase}, - demo_script::boot_test_fixtures::{ + boot_test_fixtures::{ activating, nested_boot_sample, queued_boot, running, tree_outline, waiting, }, }; diff --git a/src/boot_view/tree_render_tests.rs b/bootvine/src/boot_view/tree_render_tests.rs similarity index 98% rename from src/boot_view/tree_render_tests.rs rename to bootvine/src/boot_view/tree_render_tests.rs index 8990f7b..88879a8 100644 --- a/src/boot_view/tree_render_tests.rs +++ b/bootvine/src/boot_view/tree_render_tests.rs @@ -5,7 +5,7 @@ use unicode_width::UnicodeWidthStr; use super::screen_render_fixtures::{boot_in_progress, chain_boot, described_boot, render}; use crate::{ boot_model::{BootState, JobResult, Phase, UnitTimeouts}, - demo_script::boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting}, + boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting}, }; #[test] diff --git a/src/boot_view/unit_tree.rs b/bootvine/src/boot_view/unit_tree.rs similarity index 97% rename from src/boot_view/unit_tree.rs rename to bootvine/src/boot_view/unit_tree.rs index 55389a3..c8f8dda 100644 --- a/src/boot_view/unit_tree.rs +++ b/bootvine/src/boot_view/unit_tree.rs @@ -166,7 +166,10 @@ impl UnitTree<'_> { fn unit_label_style(&self, row: &TreeRow) -> Style { match row.status { - RowStatus::Failed => Style::from(self.colors.failed_unit).bold(), + RowStatus::Failed(kind) if kind.has_own_cause() => { + Style::from(self.colors.failed_unit).bold() + } + RowStatus::Failed(_) => Style::from(self.colors.failed_unit).dim(), _ if row.kind == UnitKind::Target => self.colors.target.into(), RowStatus::Running { .. } => self.colors.running_unit.into(), RowStatus::Waiting => self.colors.waiting_unit.into(), @@ -187,7 +190,9 @@ impl UnitTree<'_> { fn unit_marker(&self, row: &TreeRow) -> Span<'static> { match row.status { - RowStatus::Failed => format!("[{}] ", self.glyphs.failed).fg(self.colors.failed_unit), + RowStatus::Failed(_) => { + format!("[{}] ", self.glyphs.failed).fg(self.colors.failed_unit) + } _ if row.kind == UnitKind::Target => Span::default(), RowStatus::Running { .. } => { format!("[{}] ", spinner_at(self.now)).fg(self.colors.running_marker) diff --git a/src/console_session.rs b/bootvine/src/console_session.rs similarity index 100% rename from src/console_session.rs rename to bootvine/src/console_session.rs diff --git a/src/console_session/ask_password.rs b/bootvine/src/console_session/ask_password.rs similarity index 100% rename from src/console_session/ask_password.rs rename to bootvine/src/console_session/ask_password.rs diff --git a/src/console_session/console_palette.rs b/bootvine/src/console_session/console_palette.rs similarity index 100% rename from src/console_session/console_palette.rs rename to bootvine/src/console_session/console_palette.rs diff --git a/src/console_session/console_takeover.rs b/bootvine/src/console_session/console_takeover.rs similarity index 100% rename from src/console_session/console_takeover.rs rename to bootvine/src/console_session/console_takeover.rs diff --git a/src/console_session/live_desk.rs b/bootvine/src/console_session/live_desk.rs similarity index 100% rename from src/console_session/live_desk.rs rename to bootvine/src/console_session/live_desk.rs diff --git a/src/console_session/passphrase_input.rs b/bootvine/src/console_session/passphrase_input.rs similarity index 100% rename from src/console_session/passphrase_input.rs rename to bootvine/src/console_session/passphrase_input.rs diff --git a/src/console_session/stream_sources.rs b/bootvine/src/console_session/stream_sources.rs similarity index 100% rename from src/console_session/stream_sources.rs rename to bootvine/src/console_session/stream_sources.rs diff --git a/src/console_session/watch_loop.rs b/bootvine/src/console_session/watch_loop.rs similarity index 100% rename from src/console_session/watch_loop.rs rename to bootvine/src/console_session/watch_loop.rs diff --git a/src/console_session/watch_screen.rs b/bootvine/src/console_session/watch_screen.rs similarity index 99% rename from src/console_session/watch_screen.rs rename to bootvine/src/console_session/watch_screen.rs index 270b91d..cb1c59e 100644 --- a/src/console_session/watch_screen.rs +++ b/bootvine/src/console_session/watch_screen.rs @@ -273,7 +273,7 @@ mod tests { use super::*; use crate::{ boot_model::{ActiveState, Activity, HANDOFF_UNIT}, - demo_script::boot_test_fixtures::{MS, nested_boot_sample, queue_unit, waiting}, + boot_test_fixtures::{MS, nested_boot_sample, queue_unit, waiting}, }; #[derive(Default)] diff --git a/src/lib.rs b/bootvine/src/lib.rs similarity index 78% rename from src/lib.rs rename to bootvine/src/lib.rs index 1a5c142..7ab6ec2 100644 --- a/src/lib.rs +++ b/bootvine/src/lib.rs @@ -1,7 +1,8 @@ pub mod boot_model; +#[cfg(test)] +mod boot_test_fixtures; pub mod boot_view; pub mod console_session; -pub mod demo_script; pub mod screen_settings; pub mod system_install; pub mod systemd_link; diff --git a/src/main.rs b/bootvine/src/main.rs similarity index 100% rename from src/main.rs rename to bootvine/src/main.rs diff --git a/src/screen_settings.rs b/bootvine/src/screen_settings.rs similarity index 100% rename from src/screen_settings.rs rename to bootvine/src/screen_settings.rs diff --git a/src/screen_settings/color_scheme.rs b/bootvine/src/screen_settings/color_scheme.rs similarity index 88% rename from src/screen_settings/color_scheme.rs rename to bootvine/src/screen_settings/color_scheme.rs index 0fdbcfe..be7f0c3 100644 --- a/src/screen_settings/color_scheme.rs +++ b/bootvine/src/screen_settings/color_scheme.rs @@ -8,6 +8,7 @@ pub struct Colors { pub stages: StageColors, pub tree: TreeColors, pub timeline: TimelineColors, + pub failures: FailureColors, pub footer: FooterColors, pub password: PasswordColors, } @@ -122,6 +123,30 @@ impl Default for TimelineColors { } } +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields, rename_all = "kebab-case")] +pub struct FailureColors { + pub rule: Color, + pub title: Color, + pub unit: Color, + pub reason: Color, + pub log: Color, + pub overflow: Color, +} + +impl Default for FailureColors { + fn default() -> Self { + Self { + rule: Color::DarkGray, + title: Color::LightRed, + unit: Color::LightRed, + reason: Color::White, + log: Color::Gray, + overflow: Color::DarkGray, + } + } +} + #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(default, deny_unknown_fields, rename_all = "kebab-case")] pub struct FooterColors { diff --git a/src/screen_settings/config_file.rs b/bootvine/src/screen_settings/config_file.rs similarity index 100% rename from src/screen_settings/config_file.rs rename to bootvine/src/screen_settings/config_file.rs diff --git a/src/screen_settings/font_glyphs.rs b/bootvine/src/screen_settings/font_glyphs.rs similarity index 100% rename from src/screen_settings/font_glyphs.rs rename to bootvine/src/screen_settings/font_glyphs.rs diff --git a/src/system_install.rs b/bootvine/src/system_install.rs similarity index 100% rename from src/system_install.rs rename to bootvine/src/system_install.rs diff --git a/src/system_install/install_error.rs b/bootvine/src/system_install/install_error.rs similarity index 100% rename from src/system_install/install_error.rs rename to bootvine/src/system_install/install_error.rs diff --git a/src/system_install/install_files.rs b/bootvine/src/system_install/install_files.rs similarity index 100% rename from src/system_install/install_files.rs rename to bootvine/src/system_install/install_files.rs diff --git a/src/systemd_link.rs b/bootvine/src/systemd_link.rs similarity index 85% rename from src/systemd_link.rs rename to bootvine/src/systemd_link.rs index dc87d22..466b59c 100644 --- a/src/systemd_link.rs +++ b/bootvine/src/systemd_link.rs @@ -1,4 +1,5 @@ pub mod boot_feed; +mod journal_tail; pub mod link_error; pub mod monotonic_clock; pub mod switch_root; diff --git a/src/systemd_link/boot_feed.rs b/bootvine/src/systemd_link/boot_feed.rs similarity index 76% rename from src/systemd_link/boot_feed.rs rename to bootvine/src/systemd_link/boot_feed.rs index 05d4625..05f3d9d 100644 --- a/src/systemd_link/boot_feed.rs +++ b/bootvine/src/systemd_link/boot_feed.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, iter::once, process, time::{Duration, Instant}, @@ -10,12 +10,16 @@ use futures::{Stream, StreamExt, stream}; use tokio::time::sleep; use crate::{ - boot_model::{Activity, BootChange, BootState, HANDOFF_UNIT, JobKind, Phase, UnitFacts}, + boot_model::{ + ActiveState, Activity, BootChange, BootState, FailureCause, HANDOFF_UNIT, JobKind, + JobResult, Phase, UnitFacts, + }, console_session::{ stream_sources::{BACKGROUND_CHECK_INTERVAL, CarriedItem, unfold_carrying}, watch_screen::ScreenInput, }, systemd_link::{ + journal_tail::recent_unit_messages, link_error::Error, monotonic_clock::monotonic_now, switch_root::Sysroot, @@ -27,6 +31,12 @@ use crate::{ }; const RECONNECT_PATIENCE: Duration = Duration::from_secs(60); +const JOURNAL_SETTLING_WAITS: [Duration; 3] = [ + Duration::from_millis(200), + Duration::from_millis(300), + Duration::from_millis(500), +]; +const LOG_LINES_KEPT: usize = 6; const SWITCH_ROOT_UNIT: &str = "initrd-switch-root.service"; const SHUTDOWN_ROOTS: [&str; 5] = [ "poweroff.target", @@ -108,6 +118,8 @@ struct BootTracker { target: RootRequest, phase: Phase, unit_paths: HashMap, String>, + explained_failures: HashSet, + units_in_failed_state: HashSet, root_switch: RootSwitch, } @@ -131,6 +143,8 @@ impl BusFeed { target, phase, unit_paths: HashMap::new(), + explained_failures: HashSet::new(), + units_in_failed_state: HashSet::new(), root_switch: RootSwitch::NotQueued, }, } @@ -196,6 +210,8 @@ impl BusFeed { } tracker.root_switch = RootSwitch::NotQueued; tracker.unit_paths.clear(); + tracker.explained_failures.clear(); + tracker.units_in_failed_state.clear(); tracker.phase = Phase::Host; self.link = Link::connecting(ConnectAttempt::Again); vec![ScreenInput::Boot(vec![BootChange::SwitchRoot { at }])] @@ -395,20 +411,68 @@ impl BootTracker { Signal::JobNew { id, job, unit } => { self.queue_new_job(connection, id, &job, unit, at).await } - Signal::JobRemoved { id, result } => vec![BootChange::FinishJob { id, result, at }], - Signal::UnitChanged { path, activity } => self - .unit_paths - .get(&path) - .map(|unit| BootChange::UpdateActivity { - unit: unit.clone(), - activity, - }) - .into_iter() - .collect(), + Signal::JobRemoved { id, unit, result } => { + let explanation = match result { + JobResult::Failed => self.explain_failure(connection, unit, "failed").await, + JobResult::Timeout => self.explain_failure(connection, unit, "timed out").await, + _ => None, + }; + once(BootChange::FinishJob { id, result, at }) + .chain(explanation) + .collect() + } + Signal::UnitChanged { path, activity } => { + let Some(unit) = self.unit_paths.get(&path).cloned() else { + return Vec::new(); + }; + let explanation = if self.enters_failed_state(&unit, activity) { + self.explain_failure(connection, unit.clone(), "failed") + .await + } else { + None + }; + once(BootChange::UpdateActivity { unit, activity }) + .chain(explanation) + .collect() + } Signal::BootFinished => self.boot_finished(), } } + fn enters_failed_state(&mut self, unit: &str, activity: Activity) -> bool { + match activity.state { + Some(ActiveState::Failed) => self.units_in_failed_state.insert(unit.to_owned()), + Some(_) => { + self.units_in_failed_state.remove(unit); + false + } + None => false, + } + } + + async fn explain_failure( + &mut self, + connection: &mut BusConnection, + unit: String, + fallback_reason: &str, + ) -> Option { + if !self.unit_paths.values().any(|known| *known == unit) { + return None; + } + self.explained_failures.insert(unit.clone()); + let reason = connection + .failure_reason(&unit) + .await + .ok() + .flatten() + .unwrap_or_else(|| fallback_reason.to_owned()); + let log_lines = settled_unit_messages(&unit).await; + Some(BootChange::ExplainFailure { + unit, + cause: FailureCause { reason, log_lines }, + }) + } + fn boot_finished(&self) -> Vec { (self.phase == Phase::Host) .then_some(BootChange::BootFinished) @@ -440,6 +504,17 @@ impl BootTracker { } } +async fn settled_unit_messages(unit: &str) -> Vec { + for settling in JOURNAL_SETTLING_WAITS { + sleep(settling).await; + let log_lines = recent_unit_messages(unit.to_owned(), LOG_LINES_KEPT).await; + if !log_lines.is_empty() { + return log_lines; + } + } + Vec::new() +} + pub fn bus_inputs( phase: Phase, scope: Scope, @@ -454,3 +529,38 @@ pub fn bus_inputs( }) .flat_map(stream::iter) } + +#[cfg(test)] +mod tests { + use super::*; + + fn host_tracker() -> BootTracker { + BootTracker { + scope: Scope::System, + target: RootRequest::ManagerDefault, + phase: Phase::Host, + unit_paths: HashMap::new(), + explained_failures: HashSet::new(), + units_in_failed_state: HashSet::new(), + root_switch: RootSwitch::NotQueued, + } + } + + fn in_state(state: ActiveState) -> Activity { + Activity { + state: Some(state), + ..Activity::default() + } + } + + #[test] + fn only_entering_the_failed_state_asks_for_a_fresh_explanation() { + let mut tracker = host_tracker(); + let unit = "http-listener.service"; + assert!(tracker.enters_failed_state(unit, in_state(ActiveState::Failed))); + assert!(!tracker.enters_failed_state(unit, in_state(ActiveState::Failed))); + assert!(!tracker.enters_failed_state(unit, Activity::default())); + assert!(!tracker.enters_failed_state(unit, in_state(ActiveState::Activating))); + assert!(tracker.enters_failed_state(unit, in_state(ActiveState::Failed))); + } +} diff --git a/bootvine/src/systemd_link/journal_tail.rs b/bootvine/src/systemd_link/journal_tail.rs new file mode 100644 index 0000000..7b871a5 --- /dev/null +++ b/bootvine/src/systemd_link/journal_tail.rs @@ -0,0 +1,132 @@ +use std::{io, str::Chars}; + +use systemd::{ + id128::Id128, + journal::{Journal, OpenOptions}, +}; +use tokio::task::spawn_blocking; + +const UNIT_FIELDS: [&str; 4] = ["_SYSTEMD_UNIT", "UNIT", "_SYSTEMD_USER_UNIT", "USER_UNIT"]; +const MANAGER_SUMMARY_MESSAGE_IDS: [&str; 10] = [ + "7d4958e842da4a758f6c1cdc7b36dcc5", + "39f53479d3a045ac8e11786248231fbf", + "7ad2d189f7e94e70a38c781354912448", + "5eb03494b6584870a536b337290809b3", + "be02cf6855d2428ba40df7e9d022f03d", + "de5b426a63be47a7b6ac3eaac82e2f6f", + "9d1aaa27d60140bd96365438aad20286", + "ae8f7b866b0347b9af31fe1c80b127c0", + "d9b373ed55a64feb8242e02dbe79a49c", + "98e322203f7a4ed290d09fe03c09fe15", +]; +const ENTRIES_SCANNED: usize = 200; +const ESCAPE: char = '\u{1b}'; + +pub async fn recent_unit_messages(unit: String, wanted: usize) -> Vec { + spawn_blocking(move || { + read_recent_messages(&unit, wanted).unwrap_or_else(|error| { + eprintln!("bootvine: reading the journal of {unit}: {error}"); + Vec::new() + }) + }) + .await + .unwrap_or_default() +} + +fn read_recent_messages(unit: &str, wanted: usize) -> io::Result> { + let mut journal = OpenOptions::default().open()?; + match_unit_in_this_boot(&mut journal, unit)?; + journal.seek_tail()?; + let mut newest_first = Vec::new(); + let mut scanned = 0; + while newest_first.len() < wanted + && scanned < ENTRIES_SCANNED + && journal.previous().is_ok_and(|moved| moved > 0) + { + scanned += 1; + if is_manager_summary(&mut journal).unwrap_or(true) { + continue; + } + if let Ok(Some(message)) = entry_message(&mut journal) { + newest_first.push(message); + } + } + newest_first.reverse(); + Ok(newest_first) +} + +fn match_unit_in_this_boot(journal: &mut Journal, unit: &str) -> io::Result<()> { + UNIT_FIELDS.iter().try_for_each(|field| { + journal.match_add(field, unit)?.match_or()?; + Ok::<_, io::Error>(()) + })?; + journal + .match_and()? + .match_add("_BOOT_ID", Id128::from_boot()?.to_string())?; + Ok(()) +} + +fn field_text(journal: &mut Journal, field: &str) -> io::Result> { + Ok(journal.get_data(field)?.and_then(|entry_field| { + entry_field + .value() + .map(|value| String::from_utf8_lossy(value).into_owned()) + })) +} + +fn is_manager_summary(journal: &mut Journal) -> io::Result { + Ok(field_text(journal, "MESSAGE_ID")? + .is_some_and(|message_id| MANAGER_SUMMARY_MESSAGE_IDS.contains(&message_id.as_str()))) +} + +fn entry_message(journal: &mut Journal) -> io::Result> { + Ok(field_text(journal, "MESSAGE")? + .map(|message| printable_line(&message)) + .filter(|line| !line.is_empty())) +} + +fn printable_line(message: &str) -> String { + let mut characters = message.lines().next().unwrap_or_default().chars(); + let mut printable = String::new(); + while let Some(character) = characters.next() { + if character == ESCAPE { + skip_escape_sequence(&mut characters); + } else if !character.is_control() { + printable.push(character); + } + } + printable.trim_end().to_owned() +} + +fn skip_escape_sequence(characters: &mut Chars<'_>) { + if characters.clone().next() == Some('[') { + characters.next(); + characters.find(|character| ('@'..='~').contains(character)); + } +} + +#[cfg(test)] +mod tests { + use std::env; + + use super::*; + + #[test] + #[ignore = "reads this machine's journal: cargo test journal_of_this_boot -- --ignored \ + --nocapture"] + fn journal_of_this_boot_has_lines_for_journald() { + let unit = env::var("BOOTVINE_JOURNAL_UNIT") + .unwrap_or_else(|_| "systemd-journald.service".to_owned()); + let lines = read_recent_messages(&unit, 6).expect("readable journal"); + lines.iter().for_each(|line| println!("{line}")); + assert!(!lines.is_empty()); + } + + #[test] + fn printable_line_keeps_the_first_line_without_colours() { + assert_eq!( + printable_line("\u{1b}[31mno such file\u{1b}[0m\nsecond line"), + "no such file" + ); + } +} diff --git a/src/systemd_link/link_error.rs b/bootvine/src/systemd_link/link_error.rs similarity index 100% rename from src/systemd_link/link_error.rs rename to bootvine/src/systemd_link/link_error.rs diff --git a/src/systemd_link/monotonic_clock.rs b/bootvine/src/systemd_link/monotonic_clock.rs similarity index 100% rename from src/systemd_link/monotonic_clock.rs rename to bootvine/src/systemd_link/monotonic_clock.rs diff --git a/src/systemd_link/switch_root.rs b/bootvine/src/systemd_link/switch_root.rs similarity index 100% rename from src/systemd_link/switch_root.rs rename to bootvine/src/systemd_link/switch_root.rs diff --git a/src/systemd_link/systemd_bus.rs b/bootvine/src/systemd_link/systemd_bus.rs similarity index 83% rename from src/systemd_link/systemd_bus.rs rename to bootvine/src/systemd_link/systemd_bus.rs index d17f17b..1042a38 100644 --- a/src/systemd_link/systemd_bus.rs +++ b/bootvine/src/systemd_link/systemd_bus.rs @@ -75,6 +75,7 @@ pub enum Signal { }, JobRemoved { id: u32, + unit: String, result: JobResult, }, UnitChanged { @@ -342,6 +343,18 @@ impl BusConnection { }) } + pub async fn failure_reason(&mut self, name: &str) -> Result, Error> { + let Some(interface) = kind_interface(UnitKind::of(name)) else { + return Ok(None); + }; + let path: Path<'static> = self + .call(manager_call("GetUnit", (name,))?) + .await? + .read1()?; + let properties = self.all_properties(&path, interface).await?; + Ok(failure_reason_from(&properties)) + } + async fn all_properties(&mut self, path: &Path<'_>, interface: &str) -> Result { Ok(self .call(method_call( @@ -376,6 +389,52 @@ fn text_property(properties: &PropMap, property: &str) -> Option { .map(ToOwned::to_owned) } +fn signal_name(number: i64) -> String { + let name = match number { + 1 => "SIGHUP", + 2 => "SIGINT", + 3 => "SIGQUIT", + 4 => "SIGILL", + 5 => "SIGTRAP", + 6 => "SIGABRT", + 7 => "SIGBUS", + 8 => "SIGFPE", + 9 => "SIGKILL", + 11 => "SIGSEGV", + 13 => "SIGPIPE", + 14 => "SIGALRM", + 15 => "SIGTERM", + other => return format!("signal {other}"), + }; + name.to_owned() +} + +fn failure_reason_from(properties: &PropMap) -> Option { + let result = text_property(properties, "Result").filter(|result| result != "success")?; + let status = properties + .get("ExecMainStatus") + .and_then(|value| value.0.as_i64()); + let reason = match (result.as_str(), status) { + ("exit-code", Some(status)) if status != 0 => format!("exited with status {status}"), + ("exit-code", _) => "exited with an error".to_owned(), + ("signal", Some(signal)) => format!("killed by {}", signal_name(signal)), + ("core-dump", Some(signal)) => format!("dumped core on {}", signal_name(signal)), + ("timeout", _) => "timed out".to_owned(), + ("watchdog", _) => "stopped answering the watchdog".to_owned(), + ("start-limit-hit", _) => "restarted too often".to_owned(), + ("oom-kill", _) => "killed for running out of memory".to_owned(), + ("resources", _) => "could not set up its resources".to_owned(), + ("protocol", _) => "broke the start-up protocol".to_owned(), + (other, _) => other.to_owned(), + }; + Some( + match text_property(properties, "StatusText").filter(|text| !text.is_empty()) { + Some(status_text) => format!("{reason}: {status_text}"), + None => reason, + }, + ) +} + fn unit_id(name: &str, properties: &PropMap) -> String { text_property(properties, "Id").unwrap_or_else(|| name.to_owned()) } @@ -474,10 +533,11 @@ fn parse_job_signal(message: &Message, member: &str) -> Option { Some(Signal::JobNew { id, job, unit }) } "JobRemoved" => { - let (id, _job, _unit, result): (u32, Path<'static>, String, String) = + let (id, _job, unit, result): (u32, Path<'static>, String, String) = message.read4().ok()?; Some(Signal::JobRemoved { id, + unit, result: serde_plain::from_str(&result).unwrap_or(JobResult::Failed), }) } @@ -567,6 +627,46 @@ mod tests { ); } + fn reason_properties(result: &str, status: i32, status_text: &str) -> PropMap { + [ + ("Result", Box::new(result.to_owned()) as Box), + ("ExecMainStatus", Box::new(status)), + ("StatusText", Box::new(status_text.to_owned())), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), Variant(value))) + .collect() + } + + #[test] + fn exit_codes_and_signals_become_a_reason() { + assert_eq!( + failure_reason_from(&reason_properties("exit-code", 1, "")).as_deref(), + Some("exited with status 1") + ); + assert_eq!( + failure_reason_from(&reason_properties("signal", 11, "")).as_deref(), + Some("killed by SIGSEGV") + ); + } + + #[test] + fn status_text_follows_the_reason() { + assert_eq!( + failure_reason_from(&reason_properties("timeout", 0, "waiting for the disk")) + .as_deref(), + Some("timed out: waiting for the disk") + ); + } + + #[test] + fn success_has_no_reason() { + assert_eq!( + failure_reason_from(&reason_properties("success", 0, "")), + None + ); + } + #[test] fn mounts_use_one_timeout_for_both_directions() { let mount = limit_properties(&[("TimeoutUSec", Duration::from_secs(20))]); diff --git a/system-files/dracut/.shellcheckrc b/bootvine/system-files/dracut/.shellcheckrc similarity index 100% rename from system-files/dracut/.shellcheckrc rename to bootvine/system-files/dracut/.shellcheckrc diff --git a/system-files/dracut/90bootvine/module-setup.sh b/bootvine/system-files/dracut/90bootvine/module-setup.sh similarity index 100% rename from system-files/dracut/90bootvine/module-setup.sh rename to bootvine/system-files/dracut/90bootvine/module-setup.sh diff --git a/system-files/mkinitcpio/install/sd-bootvine b/bootvine/system-files/mkinitcpio/install/sd-bootvine similarity index 100% rename from system-files/mkinitcpio/install/sd-bootvine rename to bootvine/system-files/mkinitcpio/install/sd-bootvine diff --git a/system-files/systemd/bootvine-shutdown.service b/bootvine/system-files/systemd/bootvine-shutdown.service similarity index 100% rename from system-files/systemd/bootvine-shutdown.service rename to bootvine/system-files/systemd/bootvine-shutdown.service diff --git a/system-files/systemd/bootvine.service b/bootvine/system-files/systemd/bootvine.service similarity index 100% rename from system-files/systemd/bootvine.service rename to bootvine/system-files/systemd/bootvine.service diff --git a/docs/architecture.md b/docs/architecture.md index 86acc21..c25d18e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,6 +11,8 @@ while it does. None of this is needed to install or configure it. PID 1 over systemd's private socket, which [zbus](https://github.com/dbus2/zbus) cannot use; see [systemd's private socket](#systemds-private-socket). +- **libsystemd.** It reads the last journal lines of a failed unit; see + [failures](#failures). - **A virtual terminal.** It draws on `tty12` and switches the console to it. ## Overview @@ -116,7 +118,7 @@ From then on it follows the signals: | Signal | Carries | Used for | | --- | --- | --- | | `JobNew` | job id, job path, unit name | a row in the tree | -| `JobRemoved` | job id, result | the row finishes as done, failed or skipped | +| `JobRemoved` | job id, unit name, result | the row finishes as done, failed or skipped | | `PropertiesChanged` on a unit | `ActiveState` and its timestamps | running since, and the timeline | | `StartupFinished` | | the boot reached its default target | @@ -151,6 +153,47 @@ systemd's `…TimestampMonotonic` properties count microseconds of clock, so the times it shows are measured from the kernel start, even for units that ran before bootvine did. +### Failures + +`JobRemoved` says how a job ended: `failed`, `timeout` or `dependency`. Like +systemd's own `[FAILED]`, `[ TIME ]` and `[DEPEND]`, bootvine tells them +apart: a unit that failed only because a dependency did is shown dimmed in the +tree and left out of the failure panel, so the panel points at the unit that +actually broke. A unit whose `ActiveState` turns `failed` without a failed job, +such as a service crashing after it started, counts as failed too. + +For a unit that broke or timed out, bootvine collects the cause once: + +```mermaid +sequenceDiagram + participant PID1 as systemd (PID 1) + participant Bootvine as bootvine + participant Journal as journal files + + PID1-->>Bootvine: ActiveState=failed, or JobRemoved failed/timeout + Bootvine->>PID1: GetUnit, Properties.GetAll on the unit type + PID1->>Bootvine: Result, ExecMainCode, ExecMainStatus, StatusText + Note over Bootvine: wait 200 ms for journald, longer while nothing is there yet + Bootvine->>Journal: sd_journal, matching the unit in this boot + Journal->>Bootvine: last messages, newest last + Note over Bootvine: reason and lines in the failure panel +``` + +- **The reason** comes from the unit type's `Result` (`exit-code`, `signal`, + `timeout`, `start-limit-hit`, `oom-kill`, …), with the exit status or signal + of the main process, and the service's `StatusText` when it set one. These + are read when the unit fails, since they change at that moment. +- **The journal lines** match the unit the same way `journalctl -u` does + (`_SYSTEMD_UNIT=` or `UNIT=`, in the current boot), without PID 1's own + "Starting…", "Failed with result…" and "Failed to start…" lines, which the + reason already covers. In the initrd they come from `/run/log/journal`. The + journal is opened without `SD_JOURNAL_LOCAL_ONLY`: libsystemd caches the + initrd's machine ID, which would hide the real root's files after the + switch, and the boot ID match already keeps other machines out. +- **libsystemd** is linked rather than calling `journalctl`, since dracut, + mkinitcpio and the NixOS initrd builder copy the libraries a binary links + into the initramfs along with it, while `journalctl` may not be there. + ## Surviving the switch to the real root At the end of the initrd, systemd mounts the real root at `/sysroot`, isolates diff --git a/docs/configuration.md b/docs/configuration.md index 7b573f6..590ecab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,6 +81,14 @@ queued-bar = "gray" running = "yellow" running-label = "white" +[colors.failures] +rule = "dark-gray" +title = "light-red" +unit = "light-red" +reason = "white" +log = "gray" +overflow = "dark-gray" + [colors.footer] rule = "dark-gray" done = "green" @@ -114,6 +122,7 @@ Each `[colors.*]` table covers one part of the screen: | `stages` | the finished stages, such as the initrd | | `tree` | the unit tree | | `timeline` | the timeline of the slowest units | +| `failures` | the failed units with their reason and last journal lines | | `footer` | the tallies of done, running, waiting and failed units | | `password` | the passphrase popup | diff --git a/docs/development.md b/docs/development.md index 348f1c3..a2e3100 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,5 +1,19 @@ # Development +## Layout + +| Folder | Holds | +| --- | --- | +| `bootvine/` | the published crate: the library, the `bootvine` binary and the unit and initrd files it installs | +| `scripted-demos/` | the scripted boot, shutdown and passphrase demos, and the VHS tapes that record them | +| `xtask/` | repository tasks run with `cargo xtask` | +| `vm-tests/` | the sample NixOS machines, one folder each with its VM check | +| `nix/` | the crane builds and the NixOS module | +| `recordings/` | the animations and screenshots the README and the [guide](guide.md) show | + +`cargo build` and `cargo test` act on every crate. `cargo run` needs the binary: +`--bin bootvine`, `--bin scripted-boot` and so on. + ## Setup Work inside the dev shell (`direnv allow` once, with @@ -57,24 +71,32 @@ locally from the dev shell with `release-plz update`. | What | Command | | --- | --- | | Model and render tests | `cargo test --workspace` | -| Scripted demos | `cargo run --example boot` (also `shutdown`, `luks`) | -| Sample machine in a window | `nix run .#sample-machine-vm` | +| Scripted demos | `cargo run --bin scripted-boot` (also `scripted-shutdown`, `scripted-luks-unlock`) | +| Sample machines in a window | `nix run .#sample-vm-`, see the [guide](guide.md#sample-machines) | | Boot and shutdown in a VM | `nix build .#checks.x86_64-linux.vm-boot-and-shutdown` | | Passphrase prompt in a VM | `nix build .#checks.x86_64-linux.vm-luks-passphrase` | +| Failure panel in a VM | `nix build .#checks.x86_64-linux.vm-failures` | To watch your own user manager start the sample units: ```sh units=$(nix build --no-link --print-out-paths .#sample-unit-files) systemctl --user link --runtime $units/* -systemctl --user start --no-block bootvine-demo.target -cargo run -- --user --target bootvine-demo.target --hold 3s +systemctl --user start --no-block bootvine-sample-services.target +cargo run --bin bootvine -- --user --target bootvine-sample-services.target --hold 3s systemctl --user stop $(ls $units) systemctl --user disable --runtime $(ls $units) ``` +Everything a sample machine needs sits in its own folder under `vm-tests/`: +`machine.nix`, and for the machines with a check, `check.nix` and `check.py`. +`vm-tests/shared/` holds the settings every machine shares, the window setup +for `nix run` and the console helpers for the checks. A new machine is a new +folder plus its name in `vm-tests/default.nix`. + The VM checks read the console as text, hold the boot at fixed points and wait -on marker files instead of sleeping. To step through one by hand, run +on marker files instead of sleeping. The failures check starts each failing +unit by hand, so it knows which one is newest in the panel. To step through one by hand, run `nix run .#checks.x86_64-linux.vm-boot-and-shutdown.driverInteractive` and call `test_script()`. @@ -83,12 +105,13 @@ exports libraries that clash with [QEMU](https://www.qemu.org) or Chromium. ## Repository tasks -- `nix run .#record-screencasts` re-records the README animations from the - scripted demos and copies the passphrase screenshot from the VM check. +- `nix run .#record-screencasts` re-records the animations from the scripted + demos and copies the passphrase and failures screenshots from the VM checks + into `recordings/`, for the README and the [guide](guide.md). - `cargo xtask sync-default-config` rewrites the defaults block in [configuration](configuration.md#defaults) from `bootvine defaults`. -- `cargo run -- install --destdir --initrd dracut` stages an install - without touching the system; `cargo run -- uninstall --destdir ` removes +- `cargo run --bin bootvine -- install --destdir --initrd dracut` stages an install + without touching the system; `cargo run --bin bootvine -- uninstall --destdir ` removes it again. ## Code style and design diff --git a/docs/guide.md b/docs/guide.md new file mode 100644 index 0000000..505d67d --- /dev/null +++ b/docs/guide.md @@ -0,0 +1,157 @@ +# Guide + +What bootvine shows on the console, part by part, with the recordings that go +with each. The animations come from the [scripted demos](#scripted-demos); the +screenshots come from the [VM checks](development.md#testing) on real NixOS +machines. + +## Reading the screen + +A boot through the initrd into stage 2, with one failed unit + +From top to bottom: + +1. **The header**: the host name, the system and its generation on NixOS + (otherwise the `PRETTY_NAME` from `os-release`), and the kernel. Once a + stage has finished, the total time since the kernel started sits on the + right. +2. **The stages**: each finished stage, such as `Initrd`, with how long it + took, how many units finished in it and how many of them failed. The stage + still running is the last line, ending in `Running`. +3. **The tree** of units with a pending job. Each unit is nested under the unit + waiting on it, and the milestone targets (`sysinit.target`, `basic.target`, + `multi-user.target`, `graphical.target`) open the levels: + - `◆` is the stage being worked on, `◇` one still to come, `√` one reached + (`*`, `o` and `+` on console fonts without those glyphs); + - `[|]` with a spinner is running, `[ ]` is waiting, `[√]` finished in the + last couple of seconds (`keep-finished-units-for`); + - `[×]` in bold red broke or timed out; dimmed red failed only because a + unit it needed did; + - the `Running` column counts up while a unit runs, and after 5 seconds the + `Timeout` column shows the limit systemd will stop it at, red once it + is passed; + - a unit waiting on exactly one other, which waits on exactly one other, + folds into one row joined by `─`; mounts and devices show their + description instead of their unit name. + + When the tree is taller than the screen, finished and waiting rows give way + first, and a last line counts the units left out. +4. **The failure panel**, only once a unit broke or timed out; see + [failures](#failures). +5. **The timeline**, in the room left below the tree: the milestones reached, + with how many units they took, the slowest finished units (up to five, and + none shorter than `hide-runs-shorter-than`) and the units still running, + as bars against seconds since the kernel started. A `│` marks the switch + from the initrd to the real root. +6. **The footer**: the units done, running, waiting and failed so far. + +## Booting through the initrd + +The same recording shows a boot from the start. With a systemd initrd, +bootvine starts about a second after the kernel: + +- **The initrd stage**: udev settling, the root device and its file systems + under `initrd.target`. +- **The switch to the real root**: the initrd moves into the stages at the + top, with its time and unit count, and the tree starts again under the + default target. The timeline keeps the initrd's runs, left of the `│`. +- **Stage 2** up to `graphical.target`. +- **The handover**: bootvine hands the console back once the default target is + reached, or earlier, once the display manager has switched to its own + terminal. + +How bootvine gets in that early and survives the switch is in +[architecture](architecture.md#starting-in-the-initrd). + +## Shutdown + +A poweroff: units stopping under the targets waiting on them + +On a poweroff, reboot or halt the tree turns around: + +- its root is the target being reached, such as `poweroff.target`; +- each unit being stopped sits under the units that have to stop before it, + so the unit holding up the shutdown is at the bottom of its branch; +- `shutdown.target`, `umount.target` and `final.target` open the levels; +- the screen stays until systemd hands over to `systemd-shutdown`. + +## Passphrase prompts + +A LUKS passphrase prompt floating over the initrd tree + +When a unit asks for a passphrase, such as `systemd-cryptsetup@….service` +unlocking a [LUKS](https://gitlab.com/cryptsetup/cryptsetup) volume, the +question floats over a dimmed tree: + +- typing shows one mark per character; +- `Enter` sends it, and the popup says it is checking; +- a wrong passphrase brings the prompt back with "Wrong passphrase, try + again"; +- once unlocked, the popup closes and the boot carries on. + +If another password agent, such as Plymouth's, was already asking when +bootvine started, bootvine waits behind it. + +## Failures + +The failure panel listing failed units with their reason and last journal lines + +A unit that broke or timed out gets an entry in the failure panel, newest +first: + +- **The headline**: the unit and why it failed, such as `exited with status 2`, + `killed by SIGSEGV`, `timed out`, `restarted too often` or + `exited with status 203` (systemd could not run the command). A service that + reported a status with `sd_notify` adds it, as in + `exited with status 1: Cannot reach database`. +- **The journal lines**: up to six of the unit's last messages from this boot, + the same ones `journalctl -u` shows, without systemd's own "Starting…" and + "Failed with result…" lines. +- **Out of room**: the newest failure keeps its lines, older ones shrink to + their headline, and a last line counts the rest. +- **What stays out**: a unit that failed only because one it needed did stays + dimmed in the tree, so the panel points at the unit that actually broke. + The footer counts both. + +Failures from the initrd stay in the panel after the switch to the real root. +The screenshot comes from the `failures` sample machine, with every kind of +failure systemd reports. + +## Trying it + +Neither of these touches your system's boot. + +### Scripted demos + +Play a scripted boot, poweroff and passphrase prompt in your terminal, with +[Rust](https://www.rust-lang.org/tools/install) installed: + +```sh +git clone https://codeberg.org/wvhulle/bootvine.git +cd bootvine +cargo run --bin scripted-boot +cargo run --bin scripted-shutdown +cargo run --bin scripted-luks-unlock +``` + +`q` quits. The encrypted root unlocks with the passphrase `demo`; `Esc` quits +while it asks. These are the scenarios the animations above were recorded +from. + +### Sample machines + +Each sample machine boots in a [QEMU](https://www.qemu.org) window with +[Nix](https://nixos.org/download/), and logs in as root at the end. `poweroff` +shows the shutdown. + +| Run | Shows | +| --- | --- | +| `nix run .#sample-vm-slow-services` | slow units in the initrd and stage 2, one failing unit, and the handover to a greetd login | +| `nix run .#sample-vm-text-console` | the same units without a display manager, handing over at the default target | +| `nix run .#sample-vm-scripted-initrd` | the same units with NixOS's scripted initrd, so bootvine starts in stage 2 only | +| `nix run .#sample-vm-encrypted-volume` | a LUKS volume unlocked in the initrd; the passphrase is `sample passphrase` | +| `nix run .#sample-vm-failures` | a firmware failure in the initrd, then units that exit with an error, crash, time out, cannot be run, restart too often, report a status, die after starting, and one that fails only through its dependency | + +Each machine keeps its disk in `~/.cache/bootvine-.qcow2`; delete it +to start fresh. The machines live in `vm-tests/`, one folder each, next to the +checks that drive them. diff --git a/docs/install.md b/docs/install.md index 7acec09..8924086 100644 --- a/docs/install.md +++ b/docs/install.md @@ -48,7 +48,7 @@ as well, the initrd has to run systemd. 1. Install the build dependencies and Rust: ```sh - sudo apt install build-essential pkg-config libdbus-1-dev git curl + sudo apt install build-essential pkg-config libdbus-1-dev libsystemd-dev git curl curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` @@ -88,7 +88,7 @@ as well, the initrd has to run systemd. ## Other distributions Install as in the Ubuntu steps, with your distribution's packages for a C -toolchain, `pkg-config` and the libdbus headers. +toolchain, `pkg-config` and the libdbus and libsystemd headers. For the initrd, use the dracut module as above, or on [mkinitcpio](https://wiki.archlinux.org/title/Mkinitcpio): diff --git a/flake.nix b/flake.nix index 777efef..6d4c639 100644 --- a/flake.nix +++ b/flake.nix @@ -46,11 +46,11 @@ craneLib rustToolchain ; - builds = import ./crane-builds.nix { inherit lib pkgs craneLib; }; + builds = import ./nix/crane-builds.nix { inherit lib pkgs craneLib; }; } ); - sampleMachine = import ./tests/sample-machine { + vmTests = import ./vm-tests { inherit lib; bootvineModule = self.nixosModules.bootvine; }; @@ -60,19 +60,22 @@ { pkgs, builds, ... }: { default = builds.bootvine; - inherit (builds) bootvine scripted-boot-examples repo-tasks; - sample-unit-files = pkgs.callPackage ./tests/sample-machine/sample-unit-files.nix { - inherit (sampleMachine) sampleUnits; + inherit (builds) bootvine scripted-demos repo-tasks; + sample-unit-files = pkgs.callPackage ./vm-tests/slow-services/sample-unit-files.nix { + inherit (vmTests) sampleUnits; }; } ); nixosModules = { default = self.nixosModules.bootvine; - bootvine = import ./nixos-module.nix { inherit self; }; + bootvine = import ./nix/nixos-module.nix { inherit self; }; }; - nixosConfigurations.sample-machine = sampleMachine.nixosSystemFor "x86_64-linux"; + nixosConfigurations = lib.mapAttrs' ( + machineName: _: + lib.nameValuePair "sample-${machineName}" (vmTests.nixosSystemFor "x86_64-linux" machineName) + ) vmTests.machines; apps = forEachSystem ( { @@ -83,6 +86,11 @@ ... }: let + runSampleMachine = machineName: description: { + type = "app"; + meta.description = "Boot the ${machineName} sample machine in QEMU: ${description}"; + program = lib.getExe (vmTests.nixosSystemFor system machineName).config.system.build.vm; + }; runReleasePlzOnCodeberg = subcommand: description: { type = "app"; meta.description = description; @@ -107,25 +115,25 @@ { release-plz-release = runReleasePlzOnCodeberg "release" "Publish unpublished versions to crates.io and tag them with a Codeberg release"; release-plz-release-pr = runReleasePlzOnCodeberg "release-pr" "Open or update the Codeberg pull request that bumps the version and changelog"; - sample-machine-vm = { - type = "app"; - meta.description = "Boot a sample machine with bootvine and slow sample units in QEMU"; - program = lib.getExe (sampleMachine.nixosSystemFor system).config.system.build.vm; - }; + sample-vm-slow-services = runSampleMachine "slow-services" "slow and failing units, a held initrd probe and a greetd login"; + sample-vm-text-console = runSampleMachine "text-console" "the slow units without a display manager, handing over at the default target"; + sample-vm-scripted-initrd = runSampleMachine "scripted-initrd" "the slow units with the scripted initrd, so bootvine starts in stage 2"; + sample-vm-encrypted-volume = runSampleMachine "encrypted-volume" "a LUKS volume unlocked in the initrd with the passphrase 'sample passphrase'"; + sample-vm-failures = runSampleMachine "failures" "units that fail in every way systemd reports, each with journal messages"; record-screencasts = { type = "app"; - meta.description = "Re-record the README screencasts and passphrase screenshot into recordings/"; + meta.description = "Re-record the screencasts and the passphrase and failures screenshots into recordings/"; program = lib.getExe ( pkgs.writeShellApplication { name = "record-screencasts"; runtimeInputs = [ pkgs.vhs - builds.scripted-boot-examples + builds.scripted-demos builds.repo-tasks ]; text = "exec xtask record-screencasts --passphrase-png ${ self.checks.${system}.vm-luks-passphrase - }/passphrase.png"; + }/passphrase.png --failures-png ${self.checks.${system}.vm-failures}/failures.png"; } ); }; @@ -163,13 +171,8 @@ inherit (builds) rustfmt-formatting clippy-lints; bootvine-build = builds.bootvine; repo-tasks-build = builds.repo-tasks; - vm-boot-and-shutdown = pkgs.testers.runNixOSTest ( - import ./tests/vm-checks/boot.nix { sampleMachine = sampleMachine.machine; } - ); - vm-luks-passphrase = pkgs.testers.runNixOSTest ( - import ./tests/vm-checks/luks.nix { bootvineModule = self.nixosModules.bootvine; } - ); } + // vmTests.checksFor pkgs ); devShells = forEachSystem ( diff --git a/crane-builds.nix b/nix/crane-builds.nix similarity index 56% rename from crane-builds.nix rename to nix/crane-builds.nix index 71a0aba..e5e0923 100644 --- a/crane-builds.nix +++ b/nix/crane-builds.nix @@ -4,18 +4,24 @@ craneLib, }: let + bootvineCrate = craneLib.crateNameFromCargoToml { cargoToml = ../bootvine/Cargo.toml; }; + commonArgs = { + inherit (bootvineCrate) pname version; src = lib.fileset.toSource { - root = ./.; + root = ../.; fileset = lib.fileset.unions [ - (craneLib.fileset.commonCargoSources ./.) - ./system-files - ./.rustfmt.toml + (craneLib.fileset.commonCargoSources ../.) + ../bootvine/system-files + ../.rustfmt.toml ]; }; strictDeps = true; nativeBuildInputs = [ pkgs.pkg-config ]; - buildInputs = [ pkgs.dbus ]; + buildInputs = [ + pkgs.dbus + pkgs.systemd + ]; }; cargoArtifacts = craneLib.buildDepsOnly (commonArgs // { pname = "bootvine-workspace"; }); @@ -25,7 +31,7 @@ let in { rustfmt-formatting = craneLib.cargoFmt { - inherit (commonArgs) src; + inherit (commonArgs) src pname version; cargoExtraArgs = "--all"; }; @@ -41,35 +47,33 @@ in cargoExtraArgs = "-p bootvine"; postInstall = '' for unit in bootvine.service bootvine-shutdown.service; do - install -Dm644 system-files/systemd/$unit $out/lib/systemd/system/$unit + install -Dm644 bootvine/system-files/systemd/$unit $out/lib/systemd/system/$unit substituteInPlace $out/lib/systemd/system/$unit \ --replace-fail /usr/bin/bootvine $out/bin/bootvine done - install -Dm755 system-files/dracut/90bootvine/module-setup.sh \ + install -Dm755 bootvine/system-files/dracut/90bootvine/module-setup.sh \ $out/lib/dracut/modules.d/90bootvine/module-setup.sh - install -Dm644 system-files/mkinitcpio/install/sd-bootvine \ + install -Dm644 bootvine/system-files/mkinitcpio/install/sd-bootvine \ $out/lib/initcpio/install/sd-bootvine ''; meta = { mainProgram = "bootvine"; platforms = lib.platforms.linux; - license = lib.getLicenseFromSpdxId (lib.importTOML ./Cargo.toml).package.license; + license = lib.getLicenseFromSpdxId (lib.importTOML ../bootvine/Cargo.toml).package.license; }; }; - scripted-boot-examples = buildWorkspaceMember { - pname = "bootvine-scripted-boot-examples"; - cargoExtraArgs = "-p bootvine --examples"; + scripted-demos = buildWorkspaceMember { + inherit (craneLib.crateNameFromCargoToml { cargoToml = ../scripted-demos/Cargo.toml; }) + pname + version + ; + cargoExtraArgs = "-p scripted-demos"; doCheck = false; - installPhaseCommand = '' - for example in boot shutdown; do - install -Dm755 target/release/examples/$example $out/bin/bootvine-$example-example - done - ''; }; repo-tasks = buildWorkspaceMember { - pname = "xtask"; + inherit (craneLib.crateNameFromCargoToml { cargoToml = ../xtask/Cargo.toml; }) pname version; cargoExtraArgs = "-p xtask"; doCheck = false; meta.mainProgram = "xtask"; diff --git a/nixos-module.nix b/nix/nixos-module.nix similarity index 100% rename from nixos-module.nix rename to nix/nixos-module.nix diff --git a/recordings/failures.png b/recordings/failures.png new file mode 100644 index 0000000000000000000000000000000000000000..99d860775cd43c41b48811c242ead3dd9f29c60c GIT binary patch literal 15468 zcmeAS@N?(olHy`uVBq!ia0y~yU}a!nU{+vZW?*3G+ji*$0|SFlfKQ0)j@@dq3W{kl zA-_FMe(lx2dSs!buHO;{1_cIB7srr_IdA9YZdtOp2eue#-dmX?2tNQ%-0-yJJ7!zfj=<&)2!fx2|PcwKr^vjOv3eH{I99 zi5s-`|5pk+XKQVd(e>-xQr$9zbp9NmROKYgU!d3t=t8BU|4cJiKem;nLSP8~7(o zZ&0~@W- z_Hzxo3tq)3{Vw`=<>37Ndjo17{`9{3=U-QSs!bWww-=3j+*dE~PQA6@+O7>|KPGK* z=ByB_Tec|oP15>g}}?ejQvdwjqM) zoc;FZ`z;O6ANewEdS<7Zu!>3SRl~~7Ik&FVre?oxxM-pF;Ds`a4R5|Sqp~XF-o=r! z20V>XLKiHiKZ@NT7U0h2o2j*7GVq_|HM) z`s;U1lFad^SzTm5xXd`;bXlS=QzA`U=kB-o@V72!y~XIyT6MT#88lj%7#)KZ6f(G2Sin&~vteoYmydg&o;W>s z*?#_eO>)j5Q;L3ToiyE)Ja@)En=R@KSkJ7Vu`xXF)6J}Dw^r?+D8+a>ZDNeJtF=D2 zb2Sq?hlgfSzfi!W+o_y8>aJ#(9@AX;Bd4uFdCGAUqcb_G92ze->oYKUZkI*Pv8ih&30XauG-E|-mAsO-TVL?L!(b85}LT5d8_{B!J@(1a(79jvV<6dvVY zSP^#M!gpV_-Rl~P;#M~`L~4ou+H3!rKS!iNV8fYfLKik3m1R+xF4}yRsq6mxh+;*_ z=@k;I&o<1giKISVYs9jps5GM02 z)AN{`>S^6c;hR{H)HnPn?5e7APvear}6Gzi_75 zX_3VtOa8JYS#SP4H{sHuovmi}wbdSJYc6kI&C=4q{wX_FCe*p;p%}}VvI&xJzwFUS zU==8sxzB%bmWJM=d->uY;!g{6R4Dvroiai2CDR(tAJ6?Ge!kydfA{^d9k#2C^M2i0 zP+WIh#O(O*_xAhUTmPEt*GNeCR{1sWbc!|T{XgUTvFEQdH+0;e_;&A&j)|RD=Ra6- zV}JdQhD{Q4$~GL8pWDdq9T>n=wWYyLc7pD&DL!wW*805g>h~{VJSs5J{(tMDH5v~@ zR69gPl8am!|Cb6}NSJh7Xv6cp9!;?#mp?ufQ#`86qOx=6T9zl-I?V4iSWGmyB))yy zwB@npeuo1g>@ng6>t4CuwE6$6>fBK+mN_c~46eMLx2$2M^RoUTZN|s31-%V-jHfv- ztZ8WwHf8#hG-0_YN-aO5|B=6}Deo=knDdJT*sV>m|CZIo{cVSh-N7gMaw^w0pRJmB&o{C5SC)RK zp-T%d`^0#M`L;g)_B5-o_{dhjnJ#?s(y~~lw7W(wQTNU!_g(K!(VG57{><6HV95yC z#7{;o@t(42F~J)ZzGr&Ov1Mgkz9E$HsgBZAreEeOgE|(ls7yO^awC7dM#4+^g5HKH zpA=^v+8J@-M5KaLOnA|!HNPwier^63c;}nYgq0qCF`R{l3J>%cBkfdzcT8${Vt20M zgu1lyOl;uG2h_vifidnxZ@*8Mr6jF0UfWHHYFdEueSydznTHP%!7+$91-*{8^8 zBzWl+H#TIo7e{*^nBrJ=boSJSm$T>nzW0ll{X~Pa~xk%9q zJ5-LWPm!Az3-*MD-pTn9K?$oHj%`l3+oC$3VfjJ6x_R9OJ3>vXeE*;AZBx8g92B1u+G(j@ z^W@;!O}?UzqLMG#w_bDLvptvgt8ZD?0#l(UQ6Zc+N(-+ZWh#u~u~TU+tuc0+_lVc; zjcngc6hLdQkO!jEc2lY}eFEr#w!fMZhj=7t^I_fC9KS)%bk{2uG%8=K~_{w_Pv&Oc{W>AuYOiQAdW6yL=- zf3bgQczAtUotVw_PtSJR&D)syK65+sCq=Wsduqyc@$7pO9gyopXmiNH;lh;IWT$^W z_iy;NzwmYa?EYQ0M{8eRwT>5C9=!49yhk=~UQPOMzmttIzVUnAvK=>$d=q*8(3NrR zSuTYHrPp=-vp3%WDUuRqV&rFG((=fW{L6Uv|KI!n|2&%iLHN)4ANTF|H`X2Zxb;VU zSwNtJKtZL|RLi&*ka^nB%ycQ_!V&6M`>&UelkYn=`-gVS3zi;HXl z_iCoPO4$olVlw$o%i0fJJ{10SP6j*A>^-st_eAB@n9GEN=J>T9pZvDr^SW$4_L)CC z@@LdvTij5!DP29BjVX?wV}~dxas?U|M~F_?#K9oL4sEGOvu;_=8=2l^`{ZEv?D(cv z=fdZ)xU@!o+34MTG)V97qi4TwXf4>e>zsi2(l=)5mEu)@bAw}a+9P$2N8kF*%k<3Q zz@!E)?K4T1_b%Pz=A2R5w=!Wvv!Vja8K;cQ%dg}lmj8QkC7SW-9EG}h6WaZbDb+7- zE?(dCrKfz?qe3BpZ2`i~J`u*H*wm~B!N-~V%={KMY-?#(&Rsr-C$} zc{RJ5aqO@Z-|w0K`$&^5)3jy0H?nUuK_ko~S~a0mFEekBf2rXe#Ra># z>tY?YS?>7|sCQ~5OUmpo+x&HI1T&qw>dgA*nAHPMR+~cEfOTRlM`8<9y_xjdt}b0G z{NYIX!AY04C_XJapt4}T`jdl;&QxFIJ8_%wsft1llc23(M{(tD*`-_jEjOj;POtpj z^0uLL$^47ERya(m``vf;(;tTe!3>PTThwP~FfXp*$0w*547Cc!)|?V0v97cD=2IEZd=LMkfp$$9X)SRFSquoKU>W0xY>-gnWPpw zf3Z=Vo_0L?Q@3o)rVSg8iB$+JU%7&LnZ~a3r(EZ4*WHjjPr&;?-n*xV4js9-MdIE^ zMLpm3yH0Oz&oI=<%VInkJ9qokhMM&e44qebuP#1x;?=J5(r;H7ud{xsTO@UokZoa?tW;b`7=!ENxo9BIR2v$~D&z!dR`TDc_${aWF0~1q>GJERd2lT6^lDbiBd? zs{*b)4A8vrVgeVq^{1c!Zphy>j^7~ntB@!Eh)EV_s-vKA=Gkowpns`COc#7y_^2~Vy<`YRe3z|gx2W? zS6CmMR$ZX0;61B~vCsa=s^Em@ZF+mQm3LUyi(9jN;q)ly+G4!GzozT%n{~`;jPjeB zrk`c|{($LJqvGTSSucm1(;PqrkbGj{mT30FFNjjw`rZ6SsC_aow0q(bg4y(4;c5v9+1Am`1RET@vG$r z4QGb3w=^hkd8lXM{jf)4`d)6ejj49rbuF#49xlGaXJNP_C!wSN6UWW^hdG7Cy8Zq< zLQ?Nfy=@SFvZ+x~SN+coPTYO1>RXJG%^SD*@v} zFd}-H*}dx*f?ox=8cyk%AqJ92WV*J=_UPG{)$UKOXDZG+`%OeWIXr*Hkqti^ZU^6y z;aH)tG4%`kmCr3c$~ITtC2*Y;VmDTngyHZPQ)o_0O@ldJeN zy%O`QdlTo}^i8w)!5lNs^MKvGo-Ye`uYAY*_)MPBl?Ofisn=~3gAORVB;IAN-JZd* z2fte=G%q%*KS?4%wiW{80@S&9HRizV~o#`YY zgR7rrROhhVDR}YnenaE|i&&^o=8Y5O94p>6IdfVVJz<(w<>KrxJAkP#KrdmX`xkT0 z6=8}G7CJbvcCKY*Tx*q3=)>`nZj6la%%B2i=3mCx?PmYm=b5EcoIZKK>5I+|i~sIsxg1-qZa%(w zRoSYn;3Mnf17@WnUm7D7zqWCHuKzKQ?dVVYJiUaYGbUtQe=sTK<{x>14P0spJ5^@R z`v16_Wy%cI2MX4+`1*3&LIgHUGSNC$Zn88brsvaBrl@46Ne%qZKmD{D=%5)5G zbi~{$v3pa^YBG=Y^iTT=>x3@t+l+<+OL+3xOA@!tp7Y;%J9Au}0!Om%f!YvTS;nn* z)`;(8V2Wda*U9FLsG|pnmXZjxKMF08pe`)3zeJ_dEn*Ox_Ji^ELF#=co7J$cLAo8R9{4^^ob@`cKlwt}5w(?qiOlQfKPzR$!2U>qbRvfbqd8YWN zHp1Z#OZ=Sq@wIK0rSFO-%=_KPZK}4Uo4+OUiN&NHvx7a@=le^|ef4+!(;bpx=PlcA z-S)3ej@mjydAWwbhevP5uAhR+`Rr6FEIrGo9OiorN*qJ&gU!rrtznfeE6OubJ#z{?&%G zYnru%a=chwb%fjV3S&OoZf-Hz`OdNiYp!$8nA&hogE4k$gO~L_@zMjc58Ptx7Yew} zI5&{Jr6J*%$cED_Z_GG${MFkL!@h;}NbHB&2mVgWR>?{Kzqry+@sC=IYu{oI7eT3n zw++F&@9&uMw&8Qyo!M6zpNshKTQ-a9VClvFM^?sRjlL5Z4H_CCtqIv6lo74nEK9aT z-Bj4KN4!QKDSbvCFkw;dy|h8xe{NFJJEH~e#WwS;(m#Bi!L&?I{o0}%+OA)dFKSjz zU-V@mKhKXMFD5ZvnT5VgZF`?6C|sxsII!dJ+1O8t3OAG%$e%ED^6P)C!KHDm^h1h( zKu?Y)<6Ps@#&XlRPh|HRrE_p7C}f;rf8X?#kAvgdZN|{lzWz%J_xNTUs$uiL$M@oR zmBln3rlyJJ-`X1E6J((o=Yparcn}tvU6Qz0sM{-OPhy;^q@TV)^S7_&ft#9B8n*0+ zZ1I{n-T20ozsp1(yxHS;z}MO=mH!>6ksK)y(EESJ3JIM}HV=<`yomSss_eF`{ao_7 z&&4k@&(6qYVfNK?k94>?Gj-9T*=M5NdRD|)e5($-s(4T}Li|F$pNJLLsfsP02R2NX zVfo|cr<`ynvGiIJt4{0pxy}coHz*}YT{2AM(qQk-5!z68go9;|ZZ=a|sl#fPo_Onz zTTGcg$%o9W^*mtll!Ijt>vORQvf@nlCSSh8`dU_%(ffEq!}TQ}bbh?tahFWn_#aL&%+1;gyT2NOeY@pN_1ymx2zU5^PH)85KzZfBl$XtVIc-seB>J)Ki6 zICG80rvo1)_StT}UaOveKAY*-$xE9JxVyS%Dzi*Eyj*Nvi$vyI;h#$yZW<=V_pIN! z(BA1l*9MlQp^v35hcSA#%=JFd)x0|S%ir&uHwsU$Q(wRn{o~r2EoqykF$KvN|CC~^ zua*4s>+wekapvffqT;w8Cm!GW@jpZ`WTNoPl?5_7k9sc3Xghm8G0uo{U9_X%VYGH% z7}8F1d62zCTy9Vy$JhU&p;PMuYqK(Fe{!5y#Dw zpJ?8DaO7U&^Z1M5$8ODjo3J-+Q=M65V(6M( z)+y}g>-Qc1KkuPe4QnK)#a}*up~~~Sc1)8gm1gbP>ygWt8G7L7_gPDOCs@tm{=3tQ zY1!tkWX>%OA2T$ywT-&P%hzyZ$SSKnPpk80Jn_20YmwjI55J=BDZ7EY^`T6gxMjt% z`X(-FNIJstUv9Hig3YXWov95U*K83oTKjAHq`up$Pi3!dxRfyUS|;Ps_F}yzL#6ctY1T8Z z8rZo#G=DwHU`+445aaN1C%Cr zXWmyHs*?3#?4Ii&_N(3};NJhm<%T}r{u{VEKC#rF!=b??KEI79A@CC(V1a z-5f*af`yNLlS=3EJ9L}1=c8F~(+%weD=C&eY+spPy>yuKaqFZXzv3lW154Sa%sn{A zr!T=v_Km^=&YM9@R@;3Yl(lseGDTdjN(Pj%X?cB`E#CXUXN%y3DKfPuCO_q2+~0Kg zUDI-dsP&C2D-Nx@{=O+W>JDps%hs|3Uyk0q+_ooM@M&(1(-PW_W5|{`qr*VS~!i*`H6hPcm42dFHRnFUox#*rfvMoLAko zUm73pee#k`OrpCLleT>IaekGq(-Y!*9-XOpa>R4plsykzazrPrx>j>s^wrN=<0a{sE${8S03Tml087eN&ln`c`)ui_m65Vin2{82TO0ylk{S{PpiHiyBUtj*< z5L1&i10%$-EG)i}ZzuT|^>avNd7dbIOgebjGb{ zN5$5vh)ssSyDD@}=`k9!gL9vR&VQeG`mJ$+n?v9}^A<)j_sT5iU^#Ptn1XY!<09lTQ5$9vj<^~|yC z6-8QdXFrOZS$E~_dWrsCd9n0ac0Rog{(_P+r}7wk<*)oGK5mqhX!~1r^S+azj6WNL z0&nXddn~V}UD^5kd4isI#QNxVrm6&>rXa zO@}WYVASEX2(7CVUn6V4(t7Ipj#;-({kg?qd2yBa;h2^Ss~Hmv8K!Kb@k!TkJYJQ=WI_ zrtHF6g$L87NVFAMpp8Xoa#%=YpB8x>7<1;!vwLe^`5%i)$z>BNkkr#m_%?$%W?Cp? zcBG$U=2d|*~%*KD+wvF&y%D zWYoM)^|dY)>0Gm%>*im-HFD;QPJ6DDKfUGVVwOF2_MWo34`l2rK0Z4d@-CEX#x93^ z(F;498ag+IELl03Yww4bj>TeL2P{h-FrDiWyT&0=z2FwprBw|#Gx!P_Rjzg2S+=}t_@fsQ`oHWx>-5(R z+ZtBQ;$X~g6uS`qbyxfKwbI!wk=75Iw=wM*Iw z2Er$DD;ApjY?otg5LHmfIF-kEb+b7`StGn`kGc>p^GU?Pf%{jN^Sq`|#*-1NeK#hq z&kFo@{!>WG&dhUnHr9Nwi96e<^4Ckm=$h6!SC_cFcyVzpo8tJ{$5l!Tjrs0P2 zS1;2wne9{Vdu`gz%y|0bn?IlPCy8y_HplzXZC21w*5>>dl6G?*_m^L_{`n(req+7K znj+4^=j&LjvL;DHK6^ef^5*8c7WYfnK1_PlnDkGxXvRO4B@*`T633m~V>gtTdVH%7 zS)UN{^Q+1_Ht&OSdwvTG8LvxuerG|x`$N{LD))q?s=_601E$4?@5-DfV0S_Hedp6t zvo&X4-FPf&Nz}aB1yiDGT11Woe{qjBC^M~EUiDw%WKh?J3z|tfrCF~Y+${g%xr$|p z=9C-XW#WtCzwe7F@~eCBX7fIc1e*mNM(YDLPW>o6sVm#Dl<%ZR%UX^d@0)JMH|&}8 z>ylu62BYRvb-&(t(?=7}Ydw(h`crjoMW=8V)2}a2vRFcjwXbk#uuoXxv%LQHvWBP3 zPj9YXuN`gusOjVCi}H#K_Uw9fE1R+OoP3g_!@pjueaDuibx#kfo2hqf^;~tATJ1Mo z8!NUbESTC<+fpHQO8&RQr;Jag*Hane_a*$vN{Lm;W|^{h-D##%I~&evJ=o*-^Xr*^ zb0hcX*)eG|i9P4kvy$An%+3F>*L2K&nlgs z@hs!=k}^E>=;%b%Yn&@zI&AV;Td|b6b+hPv-MXd;PMkk(u?ELG*gXGXxhdt-tX2I7 z^r~&v*{4^A?fubklcg^I`Gk80^Z$hxSNhc)^b2IldwS&h=Q(qJG{puu^z__0(W!sq zxj(Ng>Y%~$my`76e|9PIeLCioI^p?rxy_r(PpsJ>=RDunkX15Uns477u`}hdbrvU% zWU^cdJaI2vc|m+m=3Vpir@XF7Gfg{p{!Gc+2Va>c#ptMSTC_&xH;?+ON%3b=`x_o{ zemJ*ntt(^q$4|mh&P#4+B%D?B{<&7=H&49c1HVseVo#JP&YPNbxKLpMcfYm0t(wZ4 z0B4yi(yVJ0m5LU$)C)=P^Ze*CWq!&EwO`KPUS-5>`{$hO^r&^gA43D{UTa?VnHCpk zO!zTftZ&&S{SwQw%@!e=vLcR5%#P-E&lv* z%d;~C{_B@bZYbMhHzoA*ET6sp4&QegD#n}0q&#v-c9N04o~U*3o7MDK*PnAmy$%%I z`FZ5)@-to%AJ2Z?kn^^K`?-n&I z3I1T9rXX8(Fl0)r$fKfntC&~u*SaynS%y#OgRuNF8F8P>FV>(9)Er`c(^hCHQ^TSi+{N8iSiN9z!)-;c1B!ZlM@~(7 z)WpcRe`3sm?F)+3wE0;6xYb#ieg5)c+u;>+PIfUf?w>e6-n?yt;W{7DgPCTp3NIct zHFGY#^?P^8=7WXBO066J2;H>cx#VpBML4h^ye>wZ`#?nO>^~dDgmu=OHV>Zi^L>)6 z!B?e?zv>sT{9&rwA=4bsQaPzQ@w1_LWn%G!kIT-w>xe2UL@W(qy2WI*|JjYcm*#VF z&sDd-RR8f%K_NnAHDp@lJBR3=EhkxgW(VI2Jg{yXi%DCZ!-0Io{oHal&a3~{eOs}r z&vj;3z&6YExq=0E8XDreZVK=+9$c=$8Bs1SGdZ&K058)bz8_mV(hv4;NYJfa$d|F5 z*^kFBXOjm1{Q2>2zbt$d`<9-FV^ls?->|#so&luIiD-|4ItvO48D4k2PhECOG`X_$ zx|lpmr|H*gQ;wQ`%aeFqYVv?*k4A#wnOhqt&$#+$S?=tHT%0dsh9w^i;i@-S2Qy!<*AQ-D?y&6W3&%RE};H;C}ttw@hsp%bR4j zYqL}z*nD+c|CufQfd-@I(zJ zPkMNTKkLMlMPT}EXMU#I*$yS)e>;`57bux#Z3+) zzx?%wNp+`Z=H1#YQ~7hkGrq^q;#^*ywJcq`c@rO}4wtIJ(W8waDZ4V?7d?6EJj?nT z%bHh=aW*O&B7{EZ?wMq)o0n%bYiIKOhOO-^VgeHw8ST$EHMIATMH@C1q5B^LDGoN?m=>8uY*9c9RZ+!Bf#p!kyvDZv#w#%Pwy-`_tPen5S)|34G z+gZ+BTIH-3(LQs=A{~XLE^mruv>()*6Znw&jqAmqS;^LQ!UFf_d=Hw>udlget>A>O zx6&iO{!+QB$S-bCyG`Z6DaA(DsSWzQuN8i#PSRc;+MaSmOYysP%F#QMpB8OxxN1Cg zg79$xrggGH5h2qW{)D^5T>P5%Rh6-K%c}`YUqe5y`6N`ey5SFV*p!AsM#lJOcbK|P z2z=<1DHWHQ{LwdDDByX{`wj1!!da(SMb2Ec_~N1Cr~X~+FkT^Ukjfy^ntt&728)Ck z_SD1PNj7uCl>}0@@6iLdt5nt z-Dm8+dUDzci`VS)^zXD^XSrg$P+l$J-wWq2tj!JE89}RsOH{8$aa%Zjx@jNnaA5X_ z&#x~StUsq^CwKV${Nv8~=K_!UP8ah!aAWPd^KKJY9AA@cyUt`?a!RDb{Yrs54f5Ap z7v&k{#g%-1cEGH3%FYg%inu@5Cx!h!|K;3F>qE~t91raNJXs~Fj91T7%uIe^NKN1Y zf&M#94STpt_I#ha>x$28fe&UKLIoaWzO(yse`y@&l(-%eH7$A%XQ37=%O2iEd>X7Z zJ3`*=6iD6NaI=70F=35Np0Gf0-d(mb@fZ!pyQ@_fLm`gU34e5;7*9O%x%Y|A|*ds8FC-^J0<;&6WEYETX>i3hU$(pe-7VT z&b>G{bET|7-Z9f!@t7(x9>({Fx7{-c^>=6bF_Yyrj$sv5#*;Ue=uBJm?3(hP!y(Ug zE`7_%R}8-MEM~qM)3hS7%=tO`EPHgLSz}WB8x(t%zYS!nV_hyfLE%Ao0(ZomX@_Sp zB-b_Uwu$VRm)s)bpY8kPV)ybxdrkiz59ZTioPIzlap-3>81MM~AW3Oh`) znO(`VNu!+5DxTR9G&h5bSEdi5TWz-eQ*SpJp z_%!8R(_V3FmY(vp`kW`!RtYY@c>2$g!~?9A%?&#xZ?ir*$;otY#?;*pY?+SjPMCQ= zk?GcA-vg$RtC*!O7ro##NJ=rQbXzgYaQc^$UH*j&EE+p>6aJ?#b*UXtf7LMkdFwrc zude)DngL8qP0QYh6`anOXJJuAtB^N^9(b8+;4HUi$H|2|X1qMR_hv4)WVWCQTio=u zOnilk8CnW*?J+ZAnVwzYws6e68qlur;CsM4^&@GZj{E-JpVEIQS7{g1+f(8TDzh6k$iXS-oLvR&G=OgY<-Aqw|yJq+=`o4F|iMK+a4ku^TQ>liw z4?+QcuEra67Kk3r=li4Z?|W%b+n+AH zTo&$_vTNC?kD58XR~a93?T|{EHzAxgWj3?@i8n!EOuMugr^pI2(gR$7*65#SKZ zUg9SE!GJOIygK{xgxCZAcOgB?NK?TWK}-J=$qQu-I4!Pz2(eC>6Oyu4wjk^KZ1>p# zW}A*k$toRRzG17$P6O$~oO*3_BGc>K!dG^)9uN``FiG9s@G@Mg;V;^x`qH2SSCm&B z`E>78N)?k}L3Ny{fnBSD@|mvX3y%60RGV?^*wZ{s+1VlC)2fCkYj21}Cc0HO_BmeYgd_0I=<+Gjo{$`vsQe$c(a)W;h4>T}4OHlg2EoA++~KEETU z)pv!FBTBE5;#v?7%a^H+Z|v)9hy<2jkec$r1%f%lE;(#zue57nQTe)xbn zGoxehH;#y(ZZRAj*BqhiKa1iSUrx|?a28a{S}|7t-%{TAbnn!YXABj$?0D&AZYuIw zS#aA7(H|zIOhGXjRf4XqkssI0>7HF0c#7$j@OlfWA3D-3CO;b*R_mC&mYHZIEiQn(~#u4a(T;d^FA=_P4w}&ar5)zPnQFngAWO33H{AE^zv!& zm;8EhrCX!b&oa1twodxJy*Siz|IEKWNiUa0#iJEQ-}?DB(8DsJp?b7KpeaYm&4VEm~l>x81zRhcU# zEj8Tgj?WMfJ9dk4zN7kL*?{@;*UP4D-jC-fhyiv=k&07wFo@ z-fUD{d(YY756k@i`SHqZ`|Yls-<7HKYUMJsb*1}H7ruR%d}6P$VNTbbvnP-JuV7Q$ z)SdbF(ZTa-w#(156?Ht{SGeAEkBj}uwBE~wtaDoBuP-&qD2SUIv37RZGAE?t>jht)C z8pbOGK3o?n+Rak3P9dSiEKc@A*)^{NAM+QoJULnM#Ph(y+4b?EY;PodUNro#ojXNw z{|S~Qp^QHz7&$76Dr7$-Fj~hSJ31*yX7Oe(2eUU8I#(q67ByrY;fQvY{cxsZ_k)MA zOs7tyGCp6bhnW5UyhwVHzd}M7drO1)GgijRgFCen!cK{?#O&wRyUv))SSVhTwWHy& z@!iPImoMM#nDf=+ZA0eztXrQm_cpAYsGnl6A$fz^fgHIu?dYsGRg!MOej-sU+WowKUJF@N zw#EugSaU-4+GE#`YmbYs(buw6pEsl7^d`pxvnsmRxGxh}s`Egi)n5Ky3EP8`Io=Mc zvu(Ci9FsQxC|#2)#uB1z)Wd0Ykzdire%0rVX?W^Wy>T_?sXXDmm^7POqrQ^Qs4dTx=7*9TBI+xhi zAmYTubSy61uFpSlpQ=1l+-y_E`Mrq~n2#8Jik3!?yU$b$e0z^_3`t zZ=~Rbf0Z$!29?(~Z#N6pTVT78x7YRFss;_l_XlpXZZZCltMd|l&>EgMf*)iTODjH@-UJ_C1a}^$G&RgV zR;6-Gq{^mqs&`L1&*y-(Eq_|P+G9GJX3w6y`HJ!^v*&55$+NF9Pr0`;QSOzc=R9xC zYGW=I8_TJ&pR^Y~-NMTF)PSGkM`%rmgOK(Tx%W;7@{fpZs9MARlO-+UjB3cUO&mVOx?3QxNs@H#sv=LKXdANJ+8g<`O zk#ifbHoK>=`kkEB@c7iN2!8SWT1~wt^!d4P6<3Pe}B|`)*n%Bj1T2A+5|qV-|!=`_0j#+m4AL8 ziqmQh%v^u)v0=;AcY@cBEy(kjxNFy*1L+T${{5XE|L5%1XweVnxtU%?=Bjzj+`)Q|UHrrP{eO~XH~iP*-f_ah)g#frR&85Dl`zvhq0j0Svm3PSH=VxlOtx`} zEXy8)GFgMn@1H*#hjad)_o~(YnryMahyLI1_uoJIzP|q7@AvmB4gWfv&;I}S-C2eI z)AxSTKdWPOHY8}yhh&Z_L&oK|9&DMhy556Gug8UCr;$EZOW| zYPRw1h(FM{_6gG>zKXcIKf*gQZMS^cxGFA-ea@ZDw-}eZ?VD&{_t`%0&iwy-8q%y2 z%Y#vnR}j;{TFR`sK=;~H$x{@FyT1E!&3~u?bki3n(^`Wox6HBPVsF2lAQ8r zGEYawolWtvg|+FW;Xj&F8qDj1t?ZkE+-{~PWd2rZco(hxJ=f~P{eZ{U#9KB$Y`OjR z#q+GMPbX*GF`v`7GG$5jjd{Bv!He{8r^G>)O0VZQ=HiFts05R;@r zSZTi^b4M$GP(w0*-}|D?b9`7n7$(Geb}0D!Dfi|!u0FlTnmK%WWjWhAy~qE)d|SNu zZo>bbx2Mx5yq}wGwLtD;=3~2QYY$5=aq7Qnf7s=BaD(~o=lWIqmoY5gH*FdNcUa}* zchb}QQse(xU)Hk!dTEZoZ$0<=#$|UOd+X2rvO}`LzPC6;`~K?Z`_-8&_Io$b+l zW{=9gQs1(!yFMPctKX z*+#J$?WVd5SVNiJHB%VgKWFAR=gsKPP~dow*})Zg1{MM!)S*%q43#6fw%DHv9@%yMg(|D~B1ZD;{Vx_=z*E2xENH z$7+$=x?AGG%2y8ZH9>2bZMP*eCi!r*HBXp-WZv72f`#fee~xpSe0yJTiB)=56N8~b z>Y0Z>${D2RC(Tva#(euyq|x8db9PHkUf%vvgjt>Cfm>8!um-nx1)e9oGjF=Ho06++xKacurwlVk!K~c(-%F8{wDt z=WZ2IknXIQ@Kf`yqJ~t!>e-Qz*SZew*f4Q|ZuOZtS7e(G-RHXf_5O-IEK!e_%>92- z)_SEi%Tmqd%b7a1Z7q?SYrM-jU{-O}GAn~qVsoy|^5?R=`u@zZP{vioOg)=Qc?Js(m)@UiV7RkFl;Qk0vFRTBt(lVo?rJdZ z2n=p85B5@@by4KO&8iypaJCJ{XQmv=^sT=uZT*cyaj#Nmr-h%^y$c&1AM({HCGyts zPDztI&yaOMF)_~S-1`}8HCF$Awm9rFLvGs?$2_S&;yKeko@JQ4TV}aP!k-$J75}%+ zmJ(z5b>wykYrtPiO^3NppRzDat-roya@OC2u~$oNnniqN))v}MZ0|p7;nx+efA9m- z<7*BJC08Bayq&LjedarM8Jk;bk2f>y&dAfc=hRS_p>ZKaD*l0ctK9M5cb z@@c#M>*Ds`={8?Gd$wN^Zg|Fd;P-9DMUU0K-Al;*EAl{3a{iP9nhqTd_vAe7?gUA_ zXZE-!9sWa%(d_h(s)oPLrR58fCH8$PYsj&_E} z7YCmGCBv|_>+zaTLND|(rhMdd*vhaW(JpH$(;Gh)fq!$?Gw1$VqO~xgXFIdcbqCF> zjc1MQYQ^_G-n}8(jOBMD!xjFP(ltJOrDcpaU=Sw#$#!QRoQ0~uW39EVlwyg`6aE&VD43;B&}*U zvA61xaA#Yt@A|#dFUc-h;V$z@sQg8RA%E1w?K9^^ED>i^yJ>X!v+h*pHwyf<26l59 zbWR*z^qlK}*Hp%1MpOFEn_bdzs49N4bLAnY*9W&=aJ`)Ua1P7g`GVR zm!f^=qtJVE*EiRyzYxxx*dICH!%Ca&U;F2cleW$gFSGSI_CQKwkCh2ykM8#==@)#j zPn!F+bH^D6h8JQ{>0NfwDO`2@4#8|3j0=9N%9@mm=`bDf@n>L)G`i0`;Zu3_8D<^7 zhW(45F%-#q6bGj={QNI^mv`GeNj~-;tc(l%ycf(?J~rXv(Vw1u%r^?Pefrp%FM7Xw z8GnGGINNd#r$T|O$9-l#))nh`0{j_P9cJ9*zvKc}v7O&VuJX?em&1PCu)6TuCEO@-tTKvU#a%nYq?HVhQU5 zE?tH-^JjbIpYlrX^nWr(r~w@A40%^{4!qdVB|meR}{^Up*&PWb)VeZLqQ zf|I|nJSY@7{++pHOhZylNAs%X+;#?^-Ia^mhq7!z%fE8H2y^)M@3G z0teb}@3m*$u{}trZcUlwgU)4_k{VW8tDO7$oAKE`zQg}!8T%}K+PPA);e`v!mPOSK zJC0mBCmG2wWA-ashuXfd3dsd+nhv|ym!IokxU)iL!MW^-4~h*=-ZaFAp|6O#fDZIXMh z|7Na#<-6Nn40$1(3jc(XUVh10z30th&v&<9yQ&u$UGuyk1*MV0n`f%Ewtmz1 z_RfBy!}i|qyjftzX*K;|J^lJiGAal4<3d@N6`y}AKe>5wtoer!@k_c)S0p!;~vtw!@td}%pX6mZV_tN#tjgU{a^ zFiNo9*(tyyB~mNhG+FUTVn%Ys_k){5OebxXTk?;;CguNe-`)lWZ@IYw*NRquduwn& zZ(oksf};POA2UL>r0+BN`+8nsCZ1Ua4X)*l_Rm=*~Ef?11GdXxLy6x*c#$f!veeEOV2AdFjwhm^I7t#;I zD^`lOEfLbW@A#0Dm$jkf4W~mb!+|s3mN!llS4`UJA2@MpBjcgdsvEcxI$Spwr~5MQ zsuQelmrpLgJAHBFDyfw$0Y_B@LQXDb7P;0Gzwp}j!whGgWF^wi|GlvHbEvamf{fdO z_ig`+SITBBGi6A8HQ|Bql#}6%=dL#vev4z^D3CdDr)ECmmG^8JV$lzEtWti--n!nX z%`oltfefz=Sq*LZQt~>f=dZ>8&^z;bU9@`hSFHehfpn848j}Rt3#)(BDg20BC6)Q; z|AJW7IVVl3j&4|bmH%gB)O|j7#q}$ggI_+nc<1!~{g>-on|{W*&(91q*#fE`MHx=7 zxBj8r@s+RP;D5)NNAA864RZGGWG!8s#xTp-xG>e0RUi`#%1Y@)ac!XWmI~z5ylKGSed=7-q9N#ei$?210~PWr2o!)hjn4u*fbIRai9 z9Z2BvX56=Q`cAjij8gm;To`sU9q2pDui8*E-ETb;hdhhG@_-HTNAm8Td~~}~yhXRX zEVqx5@zIU^=(U+<8n=9ZfAisE?NzsNn75SeY{|(~sYfd17S0pA)@)^%y8ZNugeJGw zWqE%dB*%uR?ox@L=zMTN`GE)B`)p4yc7JJ~62&BZ+S^#>>nz3@{!2p{I6{_AoS(wB z;O}1R`Lbv0mu%Ebd#t!4d*_Lr!WllwyV)(hlp6{-8kfG?TE4w=}r+2Kq_m$cD!{gK(=Y;30rH)zKS!_I1 zxi(`858t|ve5`T>$^JZn0-L1o8?n4+W_rV(HD&!J{xz3&)&;YJnz?)o(S1qQ`_HAz z@0tN=<_a}Pi!xuiX=pt89nUA@MVIF1e7A@dQ+R%DN*R9$r^4woORSqWeYa$cSk37$ z$e1cjQci)$x?bnRn{7Z^_(c+@F27o;@vvN#Kge0=rc0H8<;fy2@A|aHTR_ ztz;(E)^(d(`S2;2KQ3+m~eyGZEZf)s)=I8p~Ix}1ywHTJ|j8@-v?8A;H z=X|}No^$)0vbEyhX)ZNoS?R|j&55C;IK`^oW@XRaGImy$iTt9~wYjEoC z^n0kg_O8|If6;pKCl<`U8_AG&MXN!*x_foF^Y%W85c5?ga#0}*H63Q^to90S__TOk z5SxL|G*iQw{q4<5ML)^!^(vozdjFisx}uxKdZv9pY&41SO5LIvnhu*;KD_X{vOn65 zp_b*(MEk{&4FdB!);=|^)e5$`<)d{^A%|&$Q!&HK7n}_Disv)4P56>`)a%RKf)#8( zB&QY1vEEG(yuVUpXF*<`+bqNE#3})qyC+u9d9X>5cWVH9nq7Ovxt@gYGu~~oU%FQ5 z^3=J)otxYEuO){6yt7@1d5+JtslQ}m7C(QkxQH_#Cdcjnhx^Ai?UrZ$Ajo*dKKa@% zzmFz`8x=3huw2+-ui9V`%vAU`S?bZ;8G8KsjmazJi`R;tJ+(O4`$HvT**S6PeZ8xh z`DfnIVy##_M~%UKA6ixWnsT zCq|`7E}#twIr{Ab}T%%w@I@>WzAM9F^GBDI(Ag1XHk?b_0R zitILP-TWE-Zavkr6LV+ZdSxl&r6~D5IlE$oTWXam%yd@W)0 z?VB%T*vGcr&w9go@o@kA>5K0M#|5%oSifR3P2sSL?SnzRO zE5n5iuD5)vlW#>cuIIUMy-~PfuJvh#=qjgm>;m=k9s26i<-Rja{oZN8Y&HEr+b#u% zzcLF9nF{0*(lxHAeJr*3-do<(qPvoL@?x`yBYU~a>!P%H_spw$UaK@Wo-KI$*Os0; zbK0Hf>%^okb^Q7%k?*YvgL#(dfz5mE=6XI7zPUoO;pV!GFn>m`8z(l+V-m5yyJ;Gu zTJ{=eLr=~R|1Ri17H%vFZ+HA{wf_>s-c>&5-|>GivJ>LrtPy5&_FS_z&F8$JWJ%e5hThXZTomqJm0RG&q(3Lr zV7gH0ZsV|B;p90+sq3r~*CNWjC5{S9X2vre-z=w~ z5zqAf^s`dNRnHlA`!VQkIrLKM^~sQ^W7`k0IoWiE1+gD{e<-oHbo;H2Mf2`e-1x!3 z`cDIa{;miChMh=GV_E)nS)Qy|wxUFVP z<8)aP%5c4v(m|eOgMe94gC2i@X3i6iSosBF4XJBccL$eT zCQ3wzpUaZHq#FJ*<8;IA=SOS;L^g4*n^uzAut8KIj+^O%y%Cc{qu1uen{|Jm-2YwV z!LB{~IVSWpy%1h#>b6Mq-0>-EZPmCAGi7L&n&cGJaQVKw|NkMYpFy^5*koDW!YezL zp4gH3I96mT;~qx7&v{${m6tp2{(89F+PHUbYH>g3^+pki1OC#*7cxIAWchYDpzI^} zkDl!L)#j~i^{fGjO{+sCBiRl3r4x78F?48q>{j5g-8SK@t=f)x9L!5XRk#=meZC7h zY&xy^kE@qqk{AP1MFSh(hZ!oW6T}=fm`binF`DF^mYB8oRO4Fz&ZE* zS=T3Tnr~moT2#VWQmrS%ThAQG*|LU3>(%`l#`AHe?{*dRZjqY*#q3!qXTVy96%R8S zs<;A-5A5L6VY<@Jv}D7?ZFZTJi0$!2Z%RUrU zeyiiZ&i`gM!D&A=Dt1Mv#r|?9oF3{d3HH#-j@Z3 zzTT{S>KnJR_P9a6UOaQ$>IZFsTdr+1s(I*Ji5W;hDhGY7@`TmvK}3vE+gEA`8lEWp*F4 zno@3TtJ|38X3qQO>33ndw>P=c8Mby5j*8KvsK`K&eIoG!F+qjED_RMFk@%U%CEyiQ+`d~BCi!P?L4>NlMyn?BenpK;gV z@os}%b`?**w^?0UP-Uckzz)=jS2!@aGO%pY4;QmnTf=rZx~*H)%b=Q*qU`W-QV!#c zmfjY%X}>pJbeyxp^SAB_wfp?~v1JY07_KCW%=osZge73*53aM(QmZ^)=$(C6d4*-c zRpExIS--8exmq)t$Svq(C`)ZPZLa_7Gvg+8mhJBx4oY;fJ!@Q8A=J!JDLLuu`;{{+ z3b)L0{ml?^PkG^i-Od-ToLjtG(Px*SYGkhFOslAKi>5P73zI&{aB0c!j|fE>$QKFxQWM8_rQ^hMPhF`CfF)YI~T#u$9Z6d zj@z<;z5Dpifb^I(ByJTndy>6qv52nfJcc9IuUD|FGcEM}%aVT4tRahWK^2q0>jU@s z45D*cda|olioRRMuvz7G^h3SUEpyzX8PrzyPEcprUUi*chB3*8v#rF2?G5{?gN$!p z|7!d@!7A$9;%{tMi~l|;JxAUw^A^rSev>1gV~htZfbR zZJD`Fuf8qxWB$j-`M-Wh|NHkx$YQ3>X$F7R7GbTWjf&@Xg-6BRoo0HQtN)FT8_SQk z4F4{>+-EQRJd4re(#MoY`v-TPMLlm`ba5X~NJ8nJEg$L`wfF4*SIF?c`{;&L&Odo& z&m}{d=ICwVd-Jg2(x+D}1uAW&5A<1XEM-XCe3uP;OU@~nykW01-~G8~ z8l%`!?Y#f@ySD6p@%sfk*MZ(u>=92xL>i>O-e$}*bLVzge|^tnrVFd*JebL%kabq- z^-^gGwhpz!KeK~h=Ggy8S;R5rkK>y6k94K8WE~0)Us_(Ttu{5((Y{4s``-A^0K=wKtV3Hstvf^IVwb_1gcS zk?%R1@SvArCf{wNQ;y#fu+`Rd5N~Xns-ftw_T<;Sz9oLzjCyM~p7ZvuVQjUzpz_a3 zzwzI6FIEi}U5&UnWupq_W7bnS0wz|OHE%M4icKe}wKbs=GUmw^a`poLX6NdJsrq}N7N^)j6 zVY~OEt{KCdzpLa+%8oeli7coPdc_{3{AjTU&-y8nN&yOAm=)(vcwNn|7Fu<3K~#nK zjEi2`XWpDn*t*X=E?yVh7FombR{)egcvsE-D$V##O5ziQe9fS@e$lzk!i%QN0bbKz z*)sT^h~Ff(dpcW`+%(td?uSyTsSICJBJAv4)H`=`Zp@kf#jat>!Y{703x&~jvbhFC(S=+}AE6!z~9zw%(qf$%H$-nuPR$!Cyw z&d>NM>di#`G>3-AziSKjEvQ!YIa~TY+r;*w%mIVbZC=;%|8gD+Hk+QJU4CStO1@@e z;6EqX7wW$|-Z*X%<9z;)dr`@rvTwnL2ez(WkEE<;NYXUX*?ssc_LC!^~Q9ggAL@@2F<%{4X0PHSgtxjYUhXg6T4^aos`eYAu!e9!8UXK z#tG9L9`GGs%`iPYP`qKM8pHfGi}vjkwPSR!&1_9>ICj0!YAy52)vH#i9`ZcBdWCkh z^d*_F#Fcz~5(jT?+!QeByNQzR^YdSs+cQq-esQ-xZ?djis3`xgwqopmr2{TUjrN3W z_?q(gdqlFAP=odsh7E6iMKBw*O!|}6Q2pXMd8Sa24boPraD3mvuz_Y7q` zOc@w>nLuj+$WITSgam`#txf}c{{wFvYUv0>!ES0TzFx%tRFN-~*YQABC<)sPiFMb_J-ZJZ_ z@ZN{9GujR;k(0grrg8nO_%FLs^0yq@l2fHD%P}E+YtE|t{iS&g+4aKjwHw{nCM(%Z z?46a}TE1&9!yO4TB?j|<$Lg&Q2$)^E$UaZQykKecT6T}m$|}b#t|`kDa*JG^^|r)@ zlj-NTmv>?>$82;xml}TA(W}S&>BNgy6ZvMoP`uS|na>w+{JigGLAmGm`d_dx`^@|O z_VPvEFovGD|Buer%hWyavL%u8&d~6OaGuQwq=!?0dd>b8cnBk2X0yZo7%I`!hBc|Hzmg9P|A0ii&S~ zEESKR-d7Gkb*g6ldghx_i(}7Sj5%^o(0;L7-F&Tv6x*MN?q=Lr{fYIYexuU^64AQ9EbY&*C7$osNZ@hAJQt!4UTzq?Lb zcy~HO=BknM`Jf@3EO}t4|bupCzF% z(Sw0+P6nI8xl>2^zt=sKQJ8v@MWJunwRdbkmTJdr^E=N_cAnu^!P3263YAyYR4-($ zV{u~H5X0Vf-z$pgt?$iudHc7RSu=;EGrYL1<|kAoQ^mGm9m9&GhN|?2Jzc+K4ydpF zP<8ftqo=kFqr|jqe$O<9bK4H7Mae9^EtJ4-y6{u#wg{rj-^)wSyK@4*s=Nj3Hb2WDp-4Q^U{ z=KJYy4u4IKW?s3nL!ZI?l{iDrYuU`4OaE(hnRDJ0{bGTP*=^-pP$-{xrgG9c{X3^} z&fnT}Be~dBAc3qBP^O29dw=y3sd3aqRf9pe@qv}#mleL0vs;wF} zFbJ?bSfs^}7QW@AyV~SAb6a|S#Q!jSIKA4w>)yn_#`7#QOP-t!c#wWZ$=EewUCQ?w zk7y>meSWzp81n+KoV?$%^wMEc;zj?YfNKosYTUvFVuA*$a;t zW<6$tENGY2{)K(p&8hEM_vv`qEZ#J&pg53Fm>{Dz1X-PhIT-{29BV zQzHG3A1f)2wYsciUEEdmtljSW&anPV*IQL4C!a0hf4zUk&ye>QTp8+0)vDgj+#$fZf&#r6b zSRmH$`mxIv2A%BOIUOI@7BFP6JosfHIC-6LQyf3HfCbaqwQhN4XAZYcn&Vaa&Ec+= z^!2F>cUEXM@WygapE~1w!}w)e)TSt|Ed z^?&UQ*!7IT{@{{$kEB0=$_&Ah-`(v$b|ySBYaFo2_53dbTFrPLRzbeX+&FxnDUMyzM43TxeyEEM7gEAuT;t{J|#w7RF1F z>Fe1Tcid7}Vle;ZBJtqMn{&Ig4y4-&zHo0yw!aX-`*XqO!}o6f$(Zl)@xl6u40l$@ z)+klYpA!?3bm=~*)_Np-Q{!W2&Xqbl!Hk=8{;i`Q;q(5$;(IMx0-O**2^ z#AGn3u>MQde2#Mw7q(aMmP!7UwVSBcCNX_t|M$+4SZ!~yS4GcsE>+F%xWu$!!B;!M zSq$zB%c9;3PZWx{QZ>KhN~7z`s`(vxlg_SYig_=5ux>%v*_*s*_+|~wcqSfk1xFcxt7!6sJGlQhSD#mgJZ!y>3>Hgd2Q}sQx z+omvlx}{hcrq6ysbcc)&#P=vT?B!Zu#q4nI0Vm_D zaK@&|ZL@W^H#jN!O!9a6o+6oZFI;z)NALZcW)~~J z_MY87?#EIZzuJXnFO#%SShi7o>7w(`U*1z=+!45()#LHKdC%VGJbvt=G|@vvsBOe8^wmbASBZJ`i*y59bIvZeT08YN({9yO z0g?}#x43;~a9!=n@SC0ajr{v_m$n{J6{z8?Deuz*c|3?c@5+te(ryjO@7HgSUa(YN z1M2mt)f<0Hyq)n}e2vGXRolvct*CSQBN~ zRK0M=uIt}}PT{HNT)^PWRei6grh}o)FuQzTD zzjCed>a+{DyefaDZF>;N8FQQK^)7XXr=m0F&tmWS|2Z^#g>Cesjq*1n7RZ(d6}GNo zi|CGd&l3=Sy|L8)dTH(B@EYsV@E5o84ldfj;+i#3lNpGB5mQc2J6e-tUWhGoY7^d;5%{haqGGpub}b>-yyLfHfNrks=0 zt(lnhDe(E)8lD-F>D-&kHxw%Im%lNKcpX=gedp@25APmN6+bukiok<_$^0@a`pZ~l z7Fy+%o>LR7wpg_J@wA_X)5QO>YU}FjUJ*zL6|9N4H|xwzxdl}`QcdlLBO{d>&b=wm z=M-R;7q^RIlF|C~(kp>)LZ2<;3VYE6U%Sax0)M4g8y?3Rb{%2-V_0^bG3NRIjD?r4 zH2(0E>pfRDjiK$6W!~N1yP-LZ=GK!VS8m>Nh`BC|k>y8DgL7?KT2!LO_Cs&}U+{f> z!24vA{4svhx*HvWuh?&Wzqj!*``+LCCC}S!n4YmizN6XsSNZdQDwmEf-uXE;X-3-8 zZgh z&&bVNqofO6C;A0h{=Oi( zWKms$uESmq1C^f{4IlJ<7}u>~j%WB$S*oya!Kd8@-Pt7zIf#Wnf@n N@O1TaS?83{1OUf=W6b~n literal 9916 zcmeAS@N?(olHy`uVBq!ia0y~yU}a!nU{+vaW?*1o{HC74z`(#0;1lA?z`*eT|Nm8? z456W+W};6y85k58JY5_^D(1YM>)rQy$EK~N{r~S;8Z7X8XPFc7y^M23+XVI%w)3UD z9&M|;qG8ftEy47T>q4r_1f`YJzi!ab+Pg3I=@!kCe7di9%A|;}GDuz!*?w#PrLMTp zU6))7A5WV-_Y&LZsn2e%T~>LkP42#wDCdn04N;k?DUYTVrP#T?H9fnSyQ(Jp(bxQ+ zPj@uP-Fl&=o$zrVli4n#-3u?J{=Jaro+Ev0(-x)BDBs;h*~?cnJu~jeNvp1@X1u-3 zpeQ75dFZLP9sS4l7_BXtqFw3DQ=aHL`NfIlyOS5_-TyA}BVg*a)+EN>88tnZ-5l(8 zWeRP`6?uQil&^771?dO{!4mevKoUSr}xTYB}w;=145nG6gS{!YS-ISfqiTv&8|?wb~~Kf2shw=cf0SP_tU(yXZwhGFAFd4VlA2YOJn6$n;3_sf0m0bY`FGy^TR2- zPFz#lx_ZIl@M%Q`-yP?_D`2ua^EY?3{9Hp-^X$rdi5=B9T~B@Y5j~n075&-cJpZ4P zV(s4>=7+8HbGW5@YSAkh*0n$FEKlcdSfv?vm2<_<`4zKNrDa|n*r9F6zeYRY&aG`< znTmv+?>%^Y!lF!p>6M>Dk0+xif34!t+UT{URF-=litqz1pt$laHD4`@;RR8*(3=U;XNc{Eytmm0N{5c1+&a zkd(jC{>AMewk=FMrxaYYOrN}V?X7z+<@Y!CaC=;vu)V#{ylb1H_-{Qc@tt>BvSv<; zad^0jX;*mOS|Jxb`K1>38)6Pz`qex~{=~lN=LOI39^I$<)X?mf_3USQtCm%-+_v)7 zW7|UyrVH*|U-FEpzS_;MsG#UBlPYi5Cka(seem{Oqbjkmhze8L# z6z)c5-8I|E<-xhA>$8NsmdJ)A#+UhAjM-H>TWjX0%5r&3<5_wjRa+sq-gN21qfA+1 z6CU}iDb%yPE>&f;-0A1fxgv<`#i5sdjequwZa6J?hBwz(Vk;L*h@XS+%LVP#EsYIR zm(8(%&Ob}wjOG&ELox^NOz6<;pD65<`($b7Zj=6JI&1pAb3A^1U?cyj5S!CKf4iwX z=!ls4JMOu=IMY^!{vGx}KUvPzqEmpY~Tr9bGa>u@D4Y#-|Uda~L@Z?Byz2J&dc=+h-Lx~gp zCT;97~D zN%+H)zYm*S%9W=03YAo;O4XH!b8XL?$uhIte`5Y@rQBV{p5_Jt*I)m9aHD!|Txt2s zOTD+$^m=*MN2@tLyBmBb!!gl4q-#~?lB;*+PE2V|idR}7KVL(Ceu3A4GPeUiZgNWq zZ@4&Rp@VI*wp{WAmMP(D-*&vHdcMzS(pQaL;j6sgCC1;ZaCPX4&z4;D{=tpoEGFeM zN}5kH+g|kjxOk=1t4AvD6rP?B*_M63yXi*9Bfp7P9#1%Sd5wir)a3RfYgG?5C-eV3 za4Dy{^=+Q+-B&Lb=F0gVtUYo>_KM^CJL0Sx1y2TC)k(PR(UQj#ahB=Rxtg3eAH`NA zF`j&9>g(rVxZR>dNu1G?d&Q2++O87ZCw{-&xKUpyAYJRhUg3aLr36d01wBrR#|2ZX zd&MVJeqW?uwkSklK@wxAzr)TB3!`s8oPI6ywoYUJuN3Z4A7d#VEFZbkM)$19$vxG9 z0@4dN=N{eLF5NdxiRV$c(8Gfp{qJ+Ho*91q(S;k=xo*UL$oqNW#&Xu2^tjk==j8Ue zp!_HNqtSZiCuZyaYszT5VS7NG~ zuWz>X$O$W(Q`y`e{c#Sp<-vYp0abk7_IKW$io4_ZdZXu3!P*@EJIgJ~{j*;w`W!qS zI4QoOVoEa8ExU|4)r)prKlN?tV<#P>zc>6s{0ra7edQ5je5|qa|I3SkeLou~rE2~y z&~%^hhGn1f^9i@x+}#{S7dWWzw?b66>V+riL8F{HVV%Nm1n@pBlg0w6|Y}_Yy zHpsN|v3v;>({h}W%;Z({AiyDO`@)wpEL)yU5&t0PzVb46MB8x(RnDHgIr|P82F%q} z_`ZJmF7+^nQnx9|bqP24w?AqwTzWKk^_t0QGn=oTT>RoLSK;>~zqPwQ^=PQSYZb3c zzh1z3{sXgkUAjPl(}AUb3)VlhJ-~lPl4A#`2s31SeCp88Z?a{_&5HitI<|fHD~%%i zngW4jrem#zpS?1<3)8RP;Qn&db_vttgN`-QjPLjTHDNsIyEw-xJThX=mYUyu+9&5E z=m!+;Pn6&P*)UUKg`511*_!XPk4rYL>iM$|q~P}9nTyu%a9YuS|K|)91?eL9Sl{-# z1>3ei4!M2MI`ii@*=cXQJ~D}3Us3uoSlvbcLG$ZA8JCKl3rS07F*xnlzrSw1(t^^Y zoUYk-nYtq!YVX{vh&gcK{ppAU)d~-4H5OcoJ8;;_&yDF;k3)E*#rJ)Ow=J9Wl}Xio zL-&2F*I@@1_ct=7vqY(1u*|glF8l0{gKnI{QxEr#msNcZo?CFq{hOwoRpZ0((yjXr zA6@X{_Cubz2MQV2Uo79@VJ~(50w>Si&8xr4zB@jj;fH}WCu_?L>4pYtNiGe0fwJ6B z5_6oAwr{vUZ_>K_;tS7yo0r!;d+=que$c+p*Djv^Q5m6Cu5(h?WcMn^YOgz@-ak%W z5>ea6tN;JI>#JQQ(cd^%{5DGqim`hj%=q55zjorKuP2lke@8YY=Pi7v^=IBgu^|0@ zdP<+C*LZ7YNv5$)YYh>p3|(?*=9=5zm$AyEaq`_g2)TM>X5*Fk89Z&_u2H7jcdc?ie~7v5 z{*@`w6>@q?o?Y+c-W+*sUmtehp}FZpv3VW69@UJ^EBwU@W~f>`6`R-I?eXW-wvNSq z0*W#-=JK$%ELlB6(4gJcq<$I8m)iEbp8t>S-#_X8HI~o!el#7Q7I^vG1FhG`Qr~>^ z@5@yV{u8qE$Ar|3$l~+H9Vs0$_cz^gS7ZGW8-F+Q?6QEC=VrT>JP}spOwdnU8M{jA zgUqeHfd*SAoR0iCU)D_b@oyXFXFI1glxsiOyZh2EcX9SzJ0?ek#@_4pQ_?Me-x@RN zJL?i@tzz@cX_Hg6_HBy)RQ+V9?P9k|ErtI7$zgq~SU*WPw60pBJhjn{>5`%>XuP2|Ge|(mPTzeIIBBt?yanu@ooN3?56)Z_|e=pU1qM-H8(DVGws3k(DPWf(KHsdZ+sUoMt zX_hS;T|A#IH!5{__lvJMY4(rZAr-fCj9W`>)->oaacfw{Y`W<+DN^x!gx5LShap#m z3tsB2;TANQZu?_p!zr#6D*XM|jl}{=nO=!9y?U>|;H~wUMH3V?9)H~F5~aW3*>ar) zMNShwhMlqu&z?Sa`Q4D*<0eoo;?gHdl4Fe{tIN$hiJm`a{h-*&fH!h1${=Ne1JDE$RoanElb zHFY_odp&THoPWnQ{RIW8jIEPdx0G|$%#BaI>u%wqDZeqKY(mblC5}I29b{~*KgDgU zZE;>@@aR^$u^R88Zz5rjzO6dMmp?sM{^10JB+sLhKx$c{7btb#gbQDA)v>ppqqjgNu;`t#%p>K#&61PZ1n0E=Toe6j&u@B=Au0O-`tnCd`}3J;N#J(z*v6)2w=C^fj8# zuJ_0dE%&c^A@zmzjaAi8gZ@81Vvh)3=Ks93lw;*t?ur*uU&uiar!Xsv>dIAO~ zr5q02b-a+Qu)ufeo+f1pH=Ui$a~re-KHMQ?krHwu0m`BeQ9C5sv;2Uj*a4H5R;9 zzEjM$bN1`p3m3n9@RIeUa;kK${3JIq&J1gbZ*v^^)=6+w{L5Rv>x7hQ@~W2# z$p(VSOk8stjQ-twP^7@jxcfxZZEs6q#^m%3Q?5Ug-1=(W_KUsMl9_T7^%D;;t}2_j zwRg$(iFKXI()U}P=lfdi=z8pCTOC_({&M58($c8?rt&uzPdt%Qac;@PNXF{gmc8fQ z@}#~CTr-KXEv;Dker6S8@pWgv)7GosubZ^*bcs-=iqT8CnI@YmxI;EJT)R<}XvO%m z`k8@up$%Vuk<3-sWw|LfSM0U)XX)|Lb(cI-8+R)D@|@p4mCtZUTz#nV_x0~e z?F(^U4yvNAa@*=W79Qc($=QF$xW-{ss)t>?n`O5#Q&xOqbMc!kceuqTH7`2%flB$b5wqE7;EPZ7Nk#q;FgO)1%zwGMN)H#Esx2JEq4sqD+F*41!DPgLmAd-v%= z6Wn(syL+&NggIojH^?S4l_|5AiCXMXU-y~$t-!JL9}ROGem>FZ<9OlD_rz|8-tl$Y z>JoC}=b>{EJ#M46I26?`Y&*~CxY&s&!6D2%xr@X{An{rmo_y8luOFNsvii*QG zXlTvuo^($3%pLwMy4Pqjtji}}k=@fGFY%nqZ+Fx7U0WaXEh}c%f5_fA>`Kjz?M5ci0vO(%@oCajf`T4f$)3`ap z3q(NeYH@)iyVQjZ1+2>>loH->-I8&Py*k&{ece;W`7a`4Vs~5-3Z9YuNUtWv z|*x!O3M6`({z_UTlh1&XllDt=vJNv{nDvtXRRuzIdXK3;Du}JpGfU9 z+N5w_{f7j!p-%s8;+!iJPtdweB#O?)zKmGYPC zsdiag-S)eRUz)2O@mp%&t(`40&n(Tvd+(Wr#YPlo_5?Vr`=qdQOUUbkZ?@R%S+e@) z0*&iydXBlroRMNYD$K*WHx!sq_U=(b@`4932RDF2 zEPYK|R^^;W!k2zDCSPqm+q$J_;>n#m^mYgp`qq2TQ{r15BD=8iwBF~BOVb}R{;X<> zR;oUFYmJCOynlgr%KC`erA*IWbLMrvQO#zyFHrYpX<1_IaA2;E_yoDyse;x^tDjEu zik!Y9JbuS#mfL#wSk{PM5IrgIM|*q7sZF~IT~=-05+V}Vq;T!cpWLeyGkPTJPv3M&62JRKX51Aq$L{iY&fS-qBlckjRLu>Y2ZfU5#ww-1rfqaO5Xb-Mu+h6Tk5^xG zli#*(dGA;g0n*aDDm6#+!-1aNlKWa4V)-8}hN;=wu#f#?AXNXyBxA+tZA#SB2y zdQpS>e-0jQkOz13BpDz6)=l&}@Npt@WtBnr#LFIa&kZymyg1QV=C#4aEtzBP;V{PI z@p~1|v(#*Gy(({e;hKnNl($OoJkA$cZATWf-*fCKYvoo+^vG#2EnBNPN%J(zn|`?$EH>*e6`1*OM?^7JiZtDO(Rk4C z!xFEUa;1L|`s(5|EmTtM zxIwFNMQ4@L`}Wp;au3V&58m)Xe)sh-Pu6L{ZU5aae{=Sf+aVN(=wvg5I`YVg|pEgVj>a4QxZ0P%fpG`}z zERcA@x6JufTSu>A;f${rHgRu0DeV&DQEqo{?drV0tJb|Ylq|8Sm|w)X)M!eh{L}&_ z?FW;0De^C!!s2#r{vUm&Z^i7vN~O+Pv1ObATlVg>sNE}g$=}U%OJ-n9uFD^9^ShiH zeakL=*|&#PPdH%4I_XEOS6ZL2zB-iK_&ikZT8w~Ou_TZBg0kp@ho@5mj#!CIC^y-7 z|4xa%g8s@S;#nVS4oe6ZM3}3HTsU`;vm)+kqL$rL)+vc>$5@^mzSPrjt+>9mLAca8 zL}7tF+o}%|6>hGAs`FQQKYgXOvEh^~bEw2d;eh+E&RrE@Je<5>lHGZoXpIM7s$7}; zuJG>koPR%RRr!w1A9{bY@)xsT4m`DdmGtov?G^n-dy7KdV)hF5UGg~mSngV?xVG!t zWz+W<-q|74Dcc<|)k_B-L*Kt;vs`|%z&en$C zr5Tm8)n91t0S&k;Z20nGb;+Hox(K(Km4B<0gs-;or+GPqr!0DN(^B#&%NM1&tIXyo zbZmS+x8b|Rj5{nmuR47~r2T|kL;^yVF4hSO=Zy4QKkr1w)`y3G2lT!@6gaids&wjv zr(cw1SDV#T=c6qal8GV+I}WasymlHVt#81+L`RbrQ9v+VJkI-!M6Rc6NA1fAt9hR&xjX)5#|{ss=f+aCmT{s>_}89e zn3L}4|8T`BgP%=PWIg_$n*9FlmyU)xddHZ8jHXIIouN5Co~JJg>F2%skGpZ$a77F<arAu zbDQ7AStwguB^j-Iy#K*_-2^7~;<*hwe{lyFsuma|roT<`ayYR=fMdtxO2*@N9R6o{ zUwdF5|8RAEZSVQ5tnXi|o)w8vO2|pOxcY5m^Zq(l{rzJ4KSc7qLE@XZ`#E-4W*c-i zXd1IVXO-E}c+K+mJkuYqugll}|M>Oz_3P{H_x*eR`hNWSNbWaIRq7L&AJyzb07E%oQReHyok z&_AR8|H&*SziMY*KBfJ``peg2#;oy6Z>)$3*8R(3B>rgInuGO!eVOWN|Neeo|NktL zn92j~1?AtiU8z|1AlokUKp5lp6kaRFvnwBK_cjDBdtj&|6yVCVYHeh*pM%xEzn}l} z3mLpnmr3W^_iQrT8SWeB4-}ufxowSO)YI0K;;zw4dwa*OC>?!`mb9F0|-uF9xf**QhUY5S!aHrVX`M_ag*5F@_%X1ptnHkF` z8QFhW%Q9!R$|ljsv-juQ|LeK^KilAW{KG5$-@otw*R%YGYOzScvPmbxQcUgbJUS-G zH(S~tTKSHZ`-kA!=hMza9$4w6Bz;5U!T-(vcE5u&JS*4FVU<~_qV$+;-Er<8pFgYr z%~$#P^XL5Y_ve3!5c$7x?f;*De*Ez_Ki==O?4QBzf9KEl^GCTo`;|HEO#A^y)rqJ2 zW^R>>JUc&asl9!d|Kdr4hc!NZJZ$stk3#0suZQ>ix_S2FXMgLd(|`Y1lKSGl|L6L7 zyFXm^Qu@vE$6)u}_xsOm3;AP~-g3WwV#5~?rl8e7kMFRo+i=3oO!-0k#L4#_evhA@ zw(frYQ%!~KXF`NDzj6P_eDdR?y6hE?4*4%0!FDUY9-qG^-khB~qN5>A{omy+7e1wR zp5NoIGw2eM{ zfA9YF)z6t5;Zp;V@7b&TFD#v{^H%9Y9FyO##(TnV*5xTY*pPaQe+$z+MTO1x_@}&5 zy0R|k{lkar?9NqlS$tq#y30fH@R9Pph0{OiG1+}&Ub>BWtM~iFm%Y1N$^|DxFZ=5d zF0*(|?)|ezPs-+NKDfXy!f08}!<6@tdFif>FI?Y28o`zY#kSm%ep>xl@8MdOo`{Ip zCz9{AA0&vkHpt%ob0F95@NKRiS_%*R?)f_WX!O4En7zuqK=Qrn119@|w>KlgKD)*` zyWRUK^6f|CZI(aZUr9{g_gORNJM&hHCT)``S8b9~i^U2q@rS(OSSNYw*t%!e|E*BJ z_mgMa??w}jdAF1vFtQ(9!y+>6N24_BpIg$OpIFa7Xw10$?}4>Ub-$Are7M8^#OFRQ zQ{C^)8$R6OUvfMxbZ?*ePCl8$bNn&&JT?t?Sm%6XHlO3dv;TVf2Hl$HPjmze&hf_> zJKg`C*Qn3L^p1t;-4khU4Q9F1s|)rVxPFbbX5TE6nBW)Yr=DA1zZesB)67~cT6}JG zy8RAL{&R~mPfGvYT>NYC&(p_(UaVA?m_nU^Us-1MLhodBE)>_mfWbHD<4|3 zFFu^%H&^4wv<0&-FSjt*dWeC6E#A|`F~m{Vx1-8SXFl zci#R=4DXTtZqy9RD+wFF0-HsPskLG(gbne*jGr5*g zKFp!G{=QAbIq@~O931w!E*3Cde$FW;6B~pQB>T=hTQpgfnQ^Odz*$a@ z!{0g^^mkl4;dbEnjE1{W+E4K;U?8Fyak;P}zB@Z+7@hDhORb6yf5dm$_;>YUug!To#SU>KNTxmV*~IBH zX@A47&Ia{x_j6k(b-16qdbu!exs?&aFa!!l&7v;GFWq8gJ@2&RA={AK9Sym`BDxVi>I?MVh*f<1 zUTLfFz;8{11zXCvLCL?uWrlt9Iq~)V)#B zi+vWh{{NAp)Bjdn)4y!Cdiu=|nh!5U%?=IDY0-^st22xEYZ>iln|0Wz>Z?(-ooYl> zu3*96iQ)N{QLA}Rzqooj=jW7!%;GP}jQnb}Ky{-ew@GuP>gSa*6) zUpedR^R;D%jjpyCf6x8;Jm;qf0|Uc>;Q6u+ZdankK0SXp+lEd3S@Z`9qnw{|IlI(8 z&P%YYm~K?DhVAYBe&2r?w{2&O8|-;z=j>3-S@&cAw@L};TeG(5o$|>zTUvU!T=dE3 z%&!l`vabKx^(dS_uX3Xmw}(=~UPXgXUtiAiTYc$IuJkEArTORYJTKzjK3lxs?b)gw zsjjtm(=ismI~BQdT``4C*#)@ z4V$>LZ!&Rna4=2}D4T9rRlp-7JYm`8cbP09f0GsBk6AEYoy}44T5pLa_l17>h>H=c zSb4P;hFGjiIO-I-?5$X@`HARwUv%sshm dV+TXQANdKHyw?x<)Qf|H(9_k=Wt~$(697D6dG!DQ diff --git a/scripted-demos/Cargo.toml b/scripted-demos/Cargo.toml new file mode 100644 index 0000000..675c488 --- /dev/null +++ b/scripted-demos/Cargo.toml @@ -0,0 +1,17 @@ +[package] +description = "Scripted boots, shutdowns and passphrase prompts that play bootvine's screen in a terminal" +edition = "2024" +license = "GPL-3.0-or-later" +name = "scripted-demos" +publish = false +version = "0.1.0" + +[lints] +workspace = true + +[dependencies] +bootvine = { path = "../bootvine" } +futures = { version = "0.3.34", default-features = false, features = ["std"] } +ratatui = "0.30.2" +tokio = { version = "1.54.0", default-features = false, features = ["rt", "time"] } +zeroize = "1.9.1" diff --git a/examples/boot.rs b/scripted-demos/src/bin/scripted-boot.rs similarity index 92% rename from examples/boot.rs rename to scripted-demos/src/bin/scripted-boot.rs index 5a812ef..619ff36 100644 --- a/examples/boot.rs +++ b/scripted-demos/src/bin/scripted-boot.rs @@ -1,9 +1,7 @@ use std::{io, time::Duration}; -use bootvine::{ - boot_model::Phase, - demo_script::scripted_boot::{Scenario, Step, failing, play, step}, -}; +use bootvine::boot_model::Phase; +use scripted_demos::scripted_boot::{Scenario, ScriptedFailure, Step, failing, play, step}; const INITRD: &[Step] = &[ step( @@ -127,6 +125,14 @@ const HOST: &[Step] = &[ "nvidia-powerd.service", &["basic.target"], Duration::from_millis(700), + ScriptedFailure { + reason: "exited with status 1", + log_lines: &[ + "nvidia-powerd version:1.0(build 1)", + "ERROR! Failed to get system power limits", + "ERROR! Client (presumably SBIOS) has not registered for D-Notify events", + ], + }, ), step( "sys-devices-virtual-misc-rfkill.device", diff --git a/examples/luks.rs b/scripted-demos/src/bin/scripted-luks-unlock.rs similarity index 98% rename from examples/luks.rs rename to scripted-demos/src/bin/scripted-luks-unlock.rs index 5eb859a..77bf15d 100644 --- a/examples/luks.rs +++ b/scripted-demos/src/bin/scripted-luks-unlock.rs @@ -1,9 +1,7 @@ use std::{io, time::Duration}; -use bootvine::{ - boot_model::Phase, - demo_script::scripted_boot::{Scenario, Step, play, step, unlocking}, -}; +use bootvine::boot_model::Phase; +use scripted_demos::scripted_boot::{Scenario, Step, play, step, unlocking}; const ROOT_PARTITION: &str = "dev-disk-by\\x2duuid-7c2e9f4a\\x2d1b3d\\x2d4e8f\\x2d9a6c\\x2d2d5b8e1f0c3a.device"; diff --git a/examples/shutdown.rs b/scripted-demos/src/bin/scripted-shutdown.rs similarity index 96% rename from examples/shutdown.rs rename to scripted-demos/src/bin/scripted-shutdown.rs index 0d0faf9..10465c9 100644 --- a/examples/shutdown.rs +++ b/scripted-demos/src/bin/scripted-shutdown.rs @@ -1,9 +1,7 @@ use std::{io, time::Duration}; -use bootvine::{ - boot_model::Phase, - demo_script::scripted_boot::{Scenario, Step, play, step, stopping}, -}; +use bootvine::boot_model::Phase; +use scripted_demos::scripted_boot::{Scenario, Step, play, step, stopping}; const SHUTDOWN: &[Step] = &[ step( diff --git a/scripted-demos/src/lib.rs b/scripted-demos/src/lib.rs new file mode 100644 index 0000000..02ce22d --- /dev/null +++ b/scripted-demos/src/lib.rs @@ -0,0 +1 @@ +pub mod scripted_boot; diff --git a/src/demo_script/scripted_boot.rs b/scripted-demos/src/scripted_boot.rs similarity index 91% rename from src/demo_script/scripted_boot.rs rename to scripted-demos/src/scripted_boot.rs index c0a97a2..9e161dc 100644 --- a/src/demo_script/scripted_boot.rs +++ b/scripted-demos/src/scripted_boot.rs @@ -7,15 +7,10 @@ use std::{ time::Duration, }; -use futures::stream; -use ratatui::{DefaultTerminal, crossterm::event::Event}; -use tokio::runtime::Builder; -use zeroize::Zeroizing; - -use crate::{ +use bootvine::{ boot_model::{ - ActiveState, Activity, BootChange, BootState, JobKind, JobResult, Phase, UnitFacts, - UnitTimeouts, boot_origin::BootOrigin, + ActiveState, Activity, BootChange, BootState, FailureCause, JobKind, JobResult, Phase, + UnitFacts, UnitTimeouts, boot_origin::BootOrigin, }, boot_view::PasswordPrompt, console_session::{ @@ -26,17 +21,33 @@ use crate::{ screen_settings::{color_scheme::Colors, config_file::Timing}, systemd_link::monotonic_clock::monotonic_now, }; +use futures::stream; +use ratatui::{DefaultTerminal, crossterm::event::Event}; +use tokio::runtime::Builder; +use zeroize::Zeroizing; const FINAL_FRAME: Duration = Duration::from_secs(3); pub const DEMO_PASSPHRASE: &str = "demo"; const DEMO_PASSPHRASE_HINT: &str = "This demo unlocks with the passphrase \"demo\""; +#[derive(Clone, Copy)] +pub struct ScriptedFailure { + pub reason: &'static str, + pub log_lines: &'static [&'static str], +} + +#[derive(Clone, Copy)] +enum StepEnding { + Succeeds, + Fails(ScriptedFailure), +} + pub struct Step { unit: &'static str, after: &'static [&'static str], work: Duration, - fails: bool, + ending: StepEnding, stops: bool, asks: Option<&'static str>, timeout: Option, @@ -58,7 +69,7 @@ pub const fn step(unit: &'static str, after: &'static [&'static str], work: Dura unit, after, work, - fails: false, + ending: StepEnding::Succeeds, stops: false, asks: None, timeout: None, @@ -66,9 +77,14 @@ pub const fn step(unit: &'static str, after: &'static [&'static str], work: Dura } #[must_use] -pub const fn failing(unit: &'static str, after: &'static [&'static str], work: Duration) -> Step { +pub const fn failing( + unit: &'static str, + after: &'static [&'static str], + work: Duration, + failure: ScriptedFailure, +) -> Step { Step { - fails: true, + ending: StepEnding::Fails(failure), ..step(unit, after, work) } } @@ -132,13 +148,26 @@ impl ScriptedJob { } fn outcome(&self) -> JobResult { - if self.step.fails { + if matches!(self.step.ending, StepEnding::Fails(_)) { JobResult::Failed } else { JobResult::Done } } + fn explanation(&self) -> Option { + let StepEnding::Fails(failure) = self.step.ending else { + return None; + }; + Some(BootChange::ExplainFailure { + unit: self.step.unit.to_owned(), + cause: FailureCause { + reason: failure.reason.to_owned(), + log_lines: failure.log_lines.iter().map(ToString::to_string).collect(), + }, + }) + } + fn initial_activity(&self) -> Activity { Activity { state: self.step.stops.then_some(ActiveState::Active), @@ -163,7 +192,7 @@ impl ScriptedJob { } fn settled_activity(&self, now: Duration) -> Activity { - let state = if self.step.fails { + let state = if matches!(self.step.ending, StepEnding::Fails(_)) { ActiveState::Failed } else if self.step.stops { ActiveState::Inactive @@ -329,6 +358,8 @@ impl Script { at: now, }, ] + .into_iter() + .chain(job.explanation()) }), ) .collect(); diff --git a/recordings/boot.tape b/scripted-demos/tapes/scripted-boot.tape similarity index 77% rename from recordings/boot.tape rename to scripted-demos/tapes/scripted-boot.tape index a50b3ff..71e1736 100644 --- a/recordings/boot.tape +++ b/scripted-demos/tapes/scripted-boot.tape @@ -1,6 +1,6 @@ Output recordings/boot.gif -Require bootvine-boot-example +Require scripted-boot Set Shell bash Set FontSize 22 @@ -12,7 +12,7 @@ Set TypingSpeed 0 Set Framerate 10 Hide -Type "clear && bootvine-boot-example" +Type "clear && scripted-boot" Enter Show diff --git a/recordings/shutdown.tape b/scripted-demos/tapes/scripted-shutdown.tape similarity index 76% rename from recordings/shutdown.tape rename to scripted-demos/tapes/scripted-shutdown.tape index 3b92cf3..2c09f67 100644 --- a/recordings/shutdown.tape +++ b/scripted-demos/tapes/scripted-shutdown.tape @@ -1,6 +1,6 @@ Output recordings/shutdown.gif -Require bootvine-shutdown-example +Require scripted-shutdown Set Shell bash Set FontSize 22 @@ -12,7 +12,7 @@ Set TypingSpeed 0 Set Framerate 10 Hide -Type "clear && bootvine-shutdown-example" +Type "clear && scripted-shutdown" Enter Show diff --git a/src/demo_script.rs b/src/demo_script.rs deleted file mode 100644 index c185991..0000000 --- a/src/demo_script.rs +++ /dev/null @@ -1,3 +0,0 @@ -#[cfg(test)] -pub(crate) mod boot_test_fixtures; -pub mod scripted_boot; diff --git a/tests/sample-machine/default.nix b/tests/sample-machine/default.nix deleted file mode 100644 index 67c8dbb..0000000 --- a/tests/sample-machine/default.nix +++ /dev/null @@ -1,25 +0,0 @@ -{ lib, bootvineModule }: -let - sampleUnits = import ./sample-units.nix; - - machine = { - imports = [ - bootvineModule - ./machine.nix - ]; - _module.args = { inherit sampleUnits; }; - }; -in -{ - inherit sampleUnits machine; - - nixosSystemFor = - system: - lib.nixosSystem { - inherit system; - modules = [ - machine - ./interactive-vm.nix - ]; - }; -} diff --git a/tests/sample-machine/machine.nix b/tests/sample-machine/machine.nix deleted file mode 100644 index 5f6eed2..0000000 --- a/tests/sample-machine/machine.nix +++ /dev/null @@ -1,91 +0,0 @@ -{ - sampleUnits, - lib, - pkgs, - ... -}: -let - sampleService = spec: { - inherit (spec) description; - after = map (dependency: "${dependency}.service") (spec.after or [ ]); - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Type = "oneshot"; - ExecStart = [ - "${pkgs.coreutils}/bin/sleep ${spec.seconds}" - ] - ++ lib.optional (spec.fails or false) "${pkgs.coreutils}/bin/false"; - } - // lib.optionalAttrs (spec ? stopSeconds) { - RemainAfterExit = true; - ExecStop = "${pkgs.coreutils}/bin/sleep ${spec.stopSeconds}"; - }; - }; -in -{ - networking.hostName = "bootvine-demo"; - system.stateVersion = "26.05"; - - boot.consoleLogLevel = lib.mkForce 3; - - boot.initrd.systemd = { - services.demo-initrd-probe = { - description = "Slow hardware probe"; - wantedBy = [ "initrd.target" ]; - before = [ "initrd.target" ]; - unitConfig.DefaultDependencies = false; - serviceConfig = { - Type = "oneshot"; - ExecStart = "/bin/sleep ${sampleUnits.initrdProbeSeconds}"; - }; - }; - }; - - services.bootvine = { - enable = true; - settings.colors = { - tree.current-stage = "light-yellow"; - tree.running-marker = "light-yellow"; - timeline.running = "light-yellow"; - footer.running = "light-yellow"; - password.frame = "light-yellow"; - }; - }; - - console = { - colors = [ - "1d2021" - "fb4934" - "b8bb26" - "fabd2f" - "83a598" - "d3869b" - "8ec07c" - "d5c4a1" - "665c54" - "fb4934" - "b8bb26" - "fabd2f" - "83a598" - "d3869b" - "8ec07c" - "fbf1c7" - ]; - earlySetup = true; - font = "${pkgs.terminus_font}/share/consolefonts/ter-v20n.psf.gz"; - }; - - systemd.services = lib.mapAttrs (_: sampleService) sampleUnits.services // { - greetd.after = [ "multi-user.target" ]; - }; - - services.greetd = { - enable = true; - settings.default_session.command = "${pkgs.greetd}/bin/agreety --cmd ${pkgs.bashInteractive}/bin/bash"; - }; - - users.users.demo = { - isNormalUser = true; - password = "demo"; - }; -} diff --git a/tests/vm-checks/luks.nix b/tests/vm-checks/luks.nix deleted file mode 100644 index 897428c..0000000 --- a/tests/vm-checks/luks.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ bootvineModule }: -let - passphrase = "quietly unlocked"; -in -{ - name = "bootvine-luks-passphrase"; - nodes.machine = - { lib, pkgs, ... }: - let - secretUuid = "9d1f2a8e-0b7c-4c55-9a8e-6b2f8f3c1d00"; - secretDisk = pkgs.runCommand "bootvine-luks-disk" { nativeBuildInputs = [ pkgs.cryptsetup ]; } '' - truncate -s 32M disk.img - echo -n '${passphrase}' | cryptsetup luksFormat -q --disable-locks --type luks2 \ - --pbkdf pbkdf2 --pbkdf-force-iterations 1000 --uuid ${secretUuid} disk.img - - mv disk.img $out - ''; - in - { - imports = [ bootvineModule ]; - - services.bootvine.enable = true; - - testing.initrdBackdoor = true; - boot.initrd.luks.devices = lib.mkVMOverride { - secret.device = "/dev/disk/by-uuid/${secretUuid}"; - }; - - virtualisation.qemu.options = [ - "-vga none" - "-device virtio-vga,xres=1280,yres=800" - ]; - virtualisation.qemu.drives = [ - { - name = "secret"; - file = "${secretDisk}"; - driveExtraOpts = { - format = "raw"; - snapshot = "on"; - }; - } - ]; - }; - - testScript = '' - PASSPHRASE = "${passphrase}" - '' - + builtins.readFile ./console.py - + builtins.readFile ./luks.py; -} diff --git a/vm-tests/default.nix b/vm-tests/default.nix new file mode 100644 index 0000000..e516868 --- /dev/null +++ b/vm-tests/default.nix @@ -0,0 +1,51 @@ +{ lib, bootvineModule }: +let + machineModule = machineName: { + imports = [ + bootvineModule + ./shared/common.nix + ./${machineName}/machine.nix + ]; + networking.hostName = "bootvine-${machineName}"; + }; + + machineNames = [ + "slow-services" + "text-console" + "scripted-initrd" + "encrypted-volume" + "failures" + ]; + + machines = lib.genAttrs machineNames machineModule; +in +{ + inherit machines; + + sampleUnits = import ./slow-services/sample-units.nix; + + nixosSystemFor = + system: machineName: + lib.nixosSystem { + inherit system; + modules = [ + machines.${machineName} + (import ./shared/interactive-vm.nix machineName) + ]; + }; + + checksFor = pkgs: { + vm-boot-and-shutdown = pkgs.testers.runNixOSTest ( + import ./slow-services/check.nix { machine = machines.slow-services; } + ); + vm-luks-passphrase = pkgs.testers.runNixOSTest ( + import ./encrypted-volume/check.nix { + machine = machines.encrypted-volume; + inherit pkgs; + } + ); + vm-failures = pkgs.testers.runNixOSTest ( + import ./failures/check.nix { machine = machines.failures; } + ); + }; +} diff --git a/vm-tests/encrypted-volume/check.nix b/vm-tests/encrypted-volume/check.nix new file mode 100644 index 0000000..086467b --- /dev/null +++ b/vm-tests/encrypted-volume/check.nix @@ -0,0 +1,24 @@ +{ machine, pkgs }: +let + secretDisk = import ./secret-disk.nix { inherit pkgs; }; +in +{ + name = "bootvine-luks-passphrase"; + nodes.machine = { + imports = [ machine ]; + + testing.initrdBackdoor = true; + + virtualisation.qemu.options = [ + "-vga none" + "-device virtio-vga,xres=1280,yres=800" + ]; + virtualisation.qemu.drives = [ secretDisk.qemuDrive ]; + }; + + testScript = '' + PASSPHRASE = "${secretDisk.passphrase}" + '' + + builtins.readFile ../shared/console.py + + builtins.readFile ./check.py; +} diff --git a/tests/vm-checks/luks.py b/vm-tests/encrypted-volume/check.py similarity index 91% rename from tests/vm-checks/luks.py rename to vm-tests/encrypted-volume/check.py index 6ec8f9b..795a697 100644 --- a/tests/vm-checks/luks.py +++ b/vm-tests/encrypted-volume/check.py @@ -11,13 +11,13 @@ machine.start() retry(lambda _: pending_prompt() != "", timeout_seconds=60) wait_for_screen("passphrase") -assert_text_visible() +assert_text_visible(font_has_512_glyphs=True) first_prompt = pending_prompt() machine.send_chars("not the passphrase\n") retry(lambda _: answered_prompts() == 1, timeout_seconds=30) retry(lambda _: pending_prompt() not in ("", first_prompt), timeout_seconds=30) wait_for_screen("passphrase") -assert_text_visible() +assert_text_visible(font_has_512_glyphs=True) machine.screenshot("passphrase") machine.send_chars(f"{PASSPHRASE}\n") machine.wait_for_unit("systemd-cryptsetup@secret.service") diff --git a/vm-tests/encrypted-volume/machine.nix b/vm-tests/encrypted-volume/machine.nix new file mode 100644 index 0000000..1d84ec7 --- /dev/null +++ b/vm-tests/encrypted-volume/machine.nix @@ -0,0 +1,11 @@ +{ lib, pkgs, ... }: +let + secretDisk = import ./secret-disk.nix { inherit pkgs; }; +in +{ + boot.initrd.luks.devices = lib.mkVMOverride { + secret.device = "/dev/disk/by-uuid/${secretDisk.uuid}"; + }; + + virtualisation.vmVariant.virtualisation.qemu.drives = [ secretDisk.qemuDrive ]; +} diff --git a/vm-tests/encrypted-volume/secret-disk.nix b/vm-tests/encrypted-volume/secret-disk.nix new file mode 100644 index 0000000..18c2d87 --- /dev/null +++ b/vm-tests/encrypted-volume/secret-disk.nix @@ -0,0 +1,22 @@ +{ pkgs }: +let + passphrase = "sample passphrase"; + uuid = "9d1f2a8e-0b7c-4c55-9a8e-6b2f8f3c1d00"; + image = pkgs.runCommand "bootvine-luks-disk" { nativeBuildInputs = [ pkgs.cryptsetup ]; } '' + truncate -s 32M disk.img + echo -n '${passphrase}' | cryptsetup luksFormat -q --disable-locks --type luks2 \ + --pbkdf pbkdf2 --pbkdf-force-iterations 1000 --uuid ${uuid} disk.img - + mv disk.img $out + ''; +in +{ + inherit passphrase uuid; + qemuDrive = { + name = "secret"; + file = "${image}"; + driveExtraOpts = { + format = "raw"; + snapshot = "on"; + }; + }; +} diff --git a/vm-tests/failures/check.nix b/vm-tests/failures/check.nix new file mode 100644 index 0000000..ac18a18 --- /dev/null +++ b/vm-tests/failures/check.nix @@ -0,0 +1,49 @@ +{ machine }: +{ + name = "bootvine-failures"; + + nodes.machine = + { pkgs, ... }: + let + holdUntilReleased = + marker: "${pkgs.bash}/bin/bash -c 'until [ -e /run/${marker} ]; do sleep 0.1; done'"; + in + { + imports = [ machine ]; + bootvineSample.startFailuresByHand = true; + testing.initrdBackdoor = true; + + boot.initrd.systemd = { + storePaths = [ "${pkgs.bash}/bin/bash" ]; + services.initrd-failures-hold = { + description = "Initrd held for the check"; + wantedBy = [ "initrd.target" ]; + before = [ "initrd.target" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + ExecStart = holdUntilReleased "release-initrd-failures-hold"; + }; + }; + }; + + systemd.services.boot-failures-hold = { + description = "Boot held for the check"; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = holdUntilReleased "release-boot-failures-hold"; + }; + }; + + virtualisation = { + memorySize = 2048; + qemu.options = [ + "-vga none" + "-device virtio-vga,xres=1280,yres=800" + ]; + }; + }; + + testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py; +} diff --git a/vm-tests/failures/check.py b/vm-tests/failures/check.py new file mode 100644 index 0000000..c61243c --- /dev/null +++ b/vm-tests/failures/check.py @@ -0,0 +1,59 @@ +def start_failing_unit(unit): + machine.succeed(f"systemctl start --no-block {unit}") + + +def unit_result(unit): + return machine.succeed(f"systemctl show -p Result --value {unit}").strip() + + +def failure_panel(): + lines = screen().splitlines() + start = next(index for index, line in enumerate(lines) if " Failures " in line) + return "\n".join(lines[start:]) + + +machine.start() + +wait_for_screen("Initrd", "Initrd held for the check") +start_failing_unit("initrd-nic-firmware-load") +wait_for_screen("Failures", "initrd-nic-firmware-load exited with status 1", "rtl8168h-2.fw not found") +assert_bootvine_in_front() +assert_text_visible(font_has_512_glyphs=True) +machine.succeed("touch /run/release-initrd-failures-hold") +machine.switch_root() + +wait_for_screen("Stage 2", "Boot held for the check") +wait_for_screen("initrd-nic-firmware-load exited with status 1") + +start_failing_unit("failure-report") +wait_for_screen("app-config-check exited with status 2", "missing key 'listen'") +retry(lambda _: unit_result("failure-report") != "", timeout_seconds=10) +assert "× failure-report" not in failure_panel(), f"a dependency failure reached the panel:\n{screen()}" +assert "failure-report" in screen(), f"the dependency failure left the tree:\n{screen()}" + +start_failing_unit("worker-pool") +wait_for_screen("worker-pool", "SIGSEGV", "starting worker pool with 4 threads") + +start_failing_unit("license-server-wait") +wait_for_screen("license-server-wait timed out", "waiting for license server at 10.0.0.9:27000") + +start_failing_unit("metrics-agent") +wait_for_screen("metrics-agent exited with status 203") + +start_failing_unit("http-listener") +wait_for_screen("http-listener restarted too often", "address already in use") + +start_failing_unit("database-client") +wait_for_screen("database-client exited with status 1: Cannot reach database", "connection refused") + +start_failing_unit("upstream-uplink") +wait_for_screen("upstream-uplink exited with status 1", "lost connection to upstream, giving up") + +wait_for_screen("× 9 failed", "connection refused") +assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}" +print(screen()) +machine.screenshot("failures") + +machine.succeed("touch /run/release-boot-failures-hold") +retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60) +print(bootvine_journal()) diff --git a/vm-tests/failures/machine.nix b/vm-tests/failures/machine.nix new file mode 100644 index 0000000..61c918f --- /dev/null +++ b/vm-tests/failures/machine.nix @@ -0,0 +1,148 @@ +{ + config, + lib, + pkgs, + ... +}: +let + startedByHand = config.bootvineSample.startFailuresByHand; + wantedAtBoot = target: lib.optional (!startedByHand) target; + + failingService = + { + description, + script, + serviceConfig ? { }, + unitConfig ? { }, + extraAttributes ? { }, + }: + { + inherit description unitConfig; + wantedBy = wantedAtBoot "multi-user.target"; + serviceConfig = { + Type = "oneshot"; + ExecStart = pkgs.writeShellScript "failing-service" script; + } + // serviceConfig; + } + // extraAttributes; + + initrdFirmwareScript = pkgs.writeShellScript "initrd-nic-firmware-load" '' + echo "loading firmware for enp1s0 (r8169)" + echo "firmware: rtl_nic/rtl8168h-2.fw not found" >&2 + exit 1 + ''; +in +{ + options.bootvineSample.startFailuresByHand = lib.mkEnableOption '' + leaving the failing sample units out of the boot targets, so a check can + start them one at a time + ''; + + config = { + boot.initrd.systemd = { + storePaths = [ + initrdFirmwareScript + "${pkgs.bash}/bin/bash" + ]; + services.initrd-nic-firmware-load = { + description = "Network card firmware"; + wantedBy = wantedAtBoot "initrd.target"; + before = [ "initrd.target" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${pkgs.bash}/bin/bash ${initrdFirmwareScript}"; + }; + }; + }; + + systemd.services = { + app-config-check = failingService { + description = "Application configuration check"; + script = '' + echo "reading /etc/app/app.toml" + echo "config: /etc/app/app.toml: missing key 'listen'" >&2 + exit 2 + ''; + }; + + worker-pool = failingService { + description = "Worker pool"; + script = '' + echo "starting worker pool with 4 threads" + kill -SEGV $$ + ''; + }; + + license-server-wait = failingService { + description = "License server check"; + script = '' + echo "waiting for license server at 10.0.0.9:27000" + sleep infinity + ''; + serviceConfig.TimeoutStartSec = 4; + }; + + metrics-agent = { + description = "Metrics agent"; + wantedBy = wantedAtBoot "multi-user.target"; + serviceConfig = { + Type = "oneshot"; + ExecStart = "/usr/bin/metrics-agent --listen :9100"; + }; + }; + + http-listener = failingService { + description = "HTTP listener"; + script = '' + echo "bind 0.0.0.0:8080: address already in use" >&2 + exit 1 + ''; + serviceConfig = { + Restart = "on-failure"; + RestartSec = "200ms"; + }; + unitConfig = { + StartLimitBurst = 3; + StartLimitIntervalSec = 30; + }; + }; + + database-client = failingService { + description = "Database client"; + script = '' + systemd-notify --status="Cannot reach database at db.internal:5432" + echo "connect db.internal:5432: connection refused" >&2 + exit 1 + ''; + serviceConfig = { + Type = "notify"; + NotifyAccess = "all"; + }; + }; + + upstream-uplink = failingService { + description = "Upstream uplink"; + script = '' + echo "connected to upstream" + sleep 3 + echo "lost connection to upstream, giving up" >&2 + exit 1 + ''; + serviceConfig.Type = "simple"; + }; + + failure-report = failingService { + description = "Failure report"; + script = '' + echo "all checks passed" + ''; + extraAttributes = { + requires = [ "app-config-check.service" ]; + after = [ "app-config-check.service" ]; + }; + }; + }; + }; +} diff --git a/vm-tests/scripted-initrd/machine.nix b/vm-tests/scripted-initrd/machine.nix new file mode 100644 index 0000000..d3c53d5 --- /dev/null +++ b/vm-tests/scripted-initrd/machine.nix @@ -0,0 +1,6 @@ +{ lib, ... }: +{ + imports = [ ../slow-services/machine.nix ]; + + boot.initrd.systemd.enable = lib.mkForce false; +} diff --git a/vm-tests/shared/common.nix b/vm-tests/shared/common.nix new file mode 100644 index 0000000..10fdfa0 --- /dev/null +++ b/vm-tests/shared/common.nix @@ -0,0 +1,45 @@ +{ lib, pkgs, ... }: +{ + system.stateVersion = "26.05"; + + boot.consoleLogLevel = lib.mkForce 3; + + services.bootvine = { + enable = true; + settings.colors = { + tree.current-stage = "light-yellow"; + tree.running-marker = "light-yellow"; + timeline.running = "light-yellow"; + footer.running = "light-yellow"; + password.frame = "light-yellow"; + }; + }; + + console = { + colors = [ + "1d2021" + "fb4934" + "b8bb26" + "fabd2f" + "83a598" + "d3869b" + "8ec07c" + "d5c4a1" + "665c54" + "fb4934" + "b8bb26" + "fabd2f" + "83a598" + "d3869b" + "8ec07c" + "fbf1c7" + ]; + earlySetup = true; + font = "${pkgs.terminus_font}/share/consolefonts/ter-v20n.psf.gz"; + }; + + users.users.sample-user = { + isNormalUser = true; + password = "sample-user"; + }; +} diff --git a/tests/vm-checks/console.py b/vm-tests/shared/console.py similarity index 98% rename from tests/vm-checks/console.py rename to vm-tests/shared/console.py index 75adb3b..f466410 100644 --- a/tests/vm-checks/console.py +++ b/vm-tests/shared/console.py @@ -51,6 +51,7 @@ def wait_for_screen(*fragments): text = screen() if last_try: print(text) + print(bootvine_journal()) return all(fragment in text for fragment in fragments) retry(shows_all, timeout_seconds=60) diff --git a/tests/sample-machine/interactive-vm.nix b/vm-tests/shared/interactive-vm.nix similarity index 79% rename from tests/sample-machine/interactive-vm.nix rename to vm-tests/shared/interactive-vm.nix index e876556..5f062a6 100644 --- a/tests/sample-machine/interactive-vm.nix +++ b/vm-tests/shared/interactive-vm.nix @@ -1,4 +1,4 @@ -{ +machineName: { boot.loader.grub.enable = false; services.getty.autologinUser = "root"; @@ -11,7 +11,7 @@ memorySize = 2048; cores = 2; graphics = true; - diskImage = "\${XDG_CACHE_HOME:-$HOME/.cache}/bootvine-sample-machine.qcow2"; + diskImage = "\${XDG_CACHE_HOME:-$HOME/.cache}/bootvine-${machineName}.qcow2"; qemu.options = [ "-vga none" "-device virtio-vga,xres=1920,yres=1080" diff --git a/tests/vm-checks/boot.nix b/vm-tests/slow-services/check.nix similarity index 54% rename from tests/vm-checks/boot.nix rename to vm-tests/slow-services/check.nix index 768351a..b8bdf04 100644 --- a/tests/vm-checks/boot.nix +++ b/vm-tests/slow-services/check.nix @@ -1,4 +1,4 @@ -{ sampleMachine }: +{ machine }: { name = "bootvine-boot-and-shutdown"; @@ -8,21 +8,21 @@ waitFor = flag: "/bin/sh -c 'until [ -e /run/${flag} ]; do sleep 0.1; done'"; in { - imports = [ sampleMachine ]; + imports = [ machine ]; testing.initrdBackdoor = true; - boot.initrd.systemd.services.demo-initrd-probe.serviceConfig.ExecStart = lib.mkForce ( - waitFor "release-initrd-probe" + boot.initrd.systemd.services.initrd-hardware-probe.serviceConfig.ExecStart = lib.mkForce ( + waitFor "release-initrd-hardware-probe" ); systemd = { settings.Manager.ShowStatus = lib.mkForce true; - services.demo-db.serviceConfig.ExecStop = lib.mkForce (waitFor "release-demo-db"); - services.backdoor.before = [ "demo-db.service" ]; - services.demo-late-sensor = { + services.sample-database.serviceConfig.ExecStop = lib.mkForce (waitFor "release-sample-database"); + services.backdoor.before = [ "sample-database.service" ]; + services.late-sensor-probe = { wantedBy = [ "graphical.target" ]; after = [ "greetd.service" ]; serviceConfig = { Type = "oneshot"; - ExecStart = waitFor "release-late-sensor"; + ExecStart = waitFor "release-late-sensor-probe"; }; }; }; @@ -35,5 +35,5 @@ }; }; - testScript = builtins.readFile ./console.py + builtins.readFile ./boot.py; + testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py; } diff --git a/tests/vm-checks/boot.py b/vm-tests/slow-services/check.py similarity index 81% rename from tests/vm-checks/boot.py rename to vm-tests/slow-services/check.py index e8bd0ee..f327ed6 100644 --- a/tests/vm-checks/boot.py +++ b/vm-tests/slow-services/check.py @@ -12,20 +12,20 @@ wait_for_screen("Initrd", "Slow hardware probe") assert_bootvine_in_front() assert_text_visible(font_has_512_glyphs=True) assert console_level() == "1", f"the kernel console was not quieted: {console_level()}" -machine.succeed("touch /run/release-initrd-probe") +machine.succeed("touch /run/release-initrd-hardware-probe") machine.switch_root() -retry(lambda _: unit_state("demo-db") == "activating", timeout_seconds=60) -wait_for_screen("Stage 2", "demo-db") +retry(lambda _: unit_state("sample-database") == "activating", timeout_seconds=60) +wait_for_screen("Stage 2", "sample-database") assert_bootvine_in_front() assert_text_visible(font_has_512_glyphs=True) assert console_level() == "1", f"stage 2 sysctl reopened the kernel console: {console_level()}" machine.wait_for_unit("greetd.service") retry(lambda _: unit_state("bootvine") == "inactive", timeout_seconds=60) -assert unit_state("demo-late-sensor") == "activating", "the boot finished before the display manager took over" +assert unit_state("late-sensor-probe") == "activating", "the boot finished before the display manager took over" assert foreground_terminal() != f"tty{BOOTVINE_TERMINAL}", "bootvine took the console back from the display manager" -machine.succeed("touch /run/release-late-sensor") +machine.succeed("touch /run/release-late-sensor-probe") machine.wait_for_unit("graphical.target") journal = bootvine_journal() @@ -50,9 +50,9 @@ assert "boot already finished" in bootvine_journal(), "a restart after boot did assert foreground_terminal() == foreground, "a restart after boot took the console" machine.succeed("systemctl poweroff --no-block") -retry(lambda _: unit_state("demo-db") == "deactivating", timeout_seconds=60) -wait_for_screen("Shutdown", "demo-db") +retry(lambda _: unit_state("sample-database") == "deactivating", timeout_seconds=60) +wait_for_screen("Shutdown", "sample-database") assert_bootvine_in_front() assert_text_visible(font_has_512_glyphs=True) -machine.succeed("touch /run/release-demo-db") +machine.succeed("touch /run/release-sample-database") machine.wait_for_shutdown() diff --git a/vm-tests/slow-services/machine.nix b/vm-tests/slow-services/machine.nix new file mode 100644 index 0000000..1a4d0ed --- /dev/null +++ b/vm-tests/slow-services/machine.nix @@ -0,0 +1,42 @@ +{ lib, pkgs, ... }: +let + sampleUnits = import ./sample-units.nix; + + sampleService = spec: { + inherit (spec) description; + after = map (dependency: "${dependency}.service") (spec.after or [ ]); + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = [ + "${pkgs.coreutils}/bin/sleep ${spec.seconds}" + ] + ++ lib.optional (spec.fails or false) "${pkgs.coreutils}/bin/false"; + } + // lib.optionalAttrs (spec ? stopSeconds) { + RemainAfterExit = true; + ExecStop = "${pkgs.coreutils}/bin/sleep ${spec.stopSeconds}"; + }; + }; +in +{ + boot.initrd.systemd.services.initrd-hardware-probe = { + description = "Slow hardware probe"; + wantedBy = [ "initrd.target" ]; + before = [ "initrd.target" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + ExecStart = "/bin/sleep ${sampleUnits.initrdProbeSeconds}"; + }; + }; + + systemd.services = lib.mapAttrs (_: sampleService) sampleUnits.services // { + greetd.after = [ "multi-user.target" ]; + }; + + services.greetd = { + enable = true; + settings.default_session.command = "${pkgs.greetd}/bin/agreety --cmd ${pkgs.bashInteractive}/bin/bash"; + }; +} diff --git a/tests/sample-machine/sample-unit-files.nix b/vm-tests/slow-services/sample-unit-files.nix similarity index 100% rename from tests/sample-machine/sample-unit-files.nix rename to vm-tests/slow-services/sample-unit-files.nix diff --git a/tests/sample-machine/sample-units.nix b/vm-tests/slow-services/sample-units.nix similarity index 63% rename from tests/sample-machine/sample-units.nix rename to vm-tests/slow-services/sample-units.nix index d3408a1..cec8124 100644 --- a/tests/sample-machine/sample-units.nix +++ b/vm-tests/slow-services/sample-units.nix @@ -1,51 +1,51 @@ { - target = "bootvine-demo"; + target = "bootvine-sample-services"; initrdProbeSeconds = "6"; services = { - demo-disk = { + scratch-disk-prepare = { description = "Scratch disk preparation"; seconds = "2"; stopSeconds = "2"; }; - demo-db = { - description = "Demo database"; + sample-database = { + description = "Sample database"; seconds = "3"; stopSeconds = "3"; - after = [ "demo-disk" ]; + after = [ "scratch-disk-prepare" ]; }; - demo-cache = { + memory-cache = { description = "In-memory cache"; seconds = "1.5"; stopSeconds = "1"; }; - demo-web = { + web-frontend = { description = "Web frontend"; seconds = "2"; stopSeconds = "2"; after = [ - "demo-db" - "demo-cache" + "sample-database" + "memory-cache" ]; }; - demo-fetch = { + remote-content-sync = { description = "Remote content sync"; seconds = "4"; }; - demo-flaky = { + flaky-health-check = { description = "Flaky health check"; seconds = "1"; fails = true; }; - demo-slow = { + slow-warm-up = { description = "Slow warm-up job"; seconds = "8"; }; - demo-report = { + startup-report = { description = "Startup report"; seconds = "1"; after = [ - "demo-web" - "demo-fetch" + "web-frontend" + "remote-content-sync" ]; }; }; diff --git a/vm-tests/text-console/machine.nix b/vm-tests/text-console/machine.nix new file mode 100644 index 0000000..9043e74 --- /dev/null +++ b/vm-tests/text-console/machine.nix @@ -0,0 +1,6 @@ +{ lib, ... }: +{ + imports = [ ../slow-services/machine.nix ]; + + services.greetd.enable = lib.mkForce false; +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 90d35b4..f61c226 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -16,8 +16,8 @@ struct RepoTasks { #[derive(Subcommand)] enum Task { #[command( - about = "Record the README screencasts with vhs and copy the passphrase screenshot into \ - recordings/" + about = "Record the screencasts with vhs and copy the passphrase and failures screenshots \ + into recordings/" )] RecordScreencasts(RecordScreencastsOptions), #[command( @@ -30,12 +30,17 @@ enum Task { struct RecordScreencastsOptions { #[arg(long, help = "passphrase.png produced by the vm-luks-passphrase check")] passphrase_png: PathBuf, + #[arg(long, help = "failures.png produced by the vm-failures check")] + failures_png: PathBuf, } fn main() -> ExitCode { let outcome = match RepoTasks::parse().task { Task::RecordScreencasts(options) => { - record_screencasts::record_screencasts(&options.passphrase_png) + record_screencasts::record_screencasts(&record_screencasts::CheckScreenshots { + passphrase_png: &options.passphrase_png, + failures_png: &options.failures_png, + }) } Task::SyncDefaultConfig => sync_default_config::sync_default_config(), }; diff --git a/xtask/src/record_screencasts.rs b/xtask/src/record_screencasts.rs index 3c77800..ec2ea50 100644 --- a/xtask/src/record_screencasts.rs +++ b/xtask/src/record_screencasts.rs @@ -8,23 +8,40 @@ use xshell::{Shell, cmd}; use crate::task_error::TaskError; -const TAPES: [&str; 2] = ["boot.tape", "shutdown.tape"]; +const TAPES: [&str; 2] = [ + "scripted-demos/tapes/scripted-boot.tape", + "scripted-demos/tapes/scripted-shutdown.tape", +]; -pub fn record_screencasts(passphrase_png: &Path) -> Result<(), TaskError> { - let recordings = Path::new("recordings"); - if !recordings.is_dir() { - return Err(TaskError::NotInRepositoryRoot); - } - let shell = Shell::new()?; - TAPES.iter().try_for_each(|tape| { - let tape_path = recordings.join(tape); - cmd!(shell, "vhs {tape_path}").run() - })?; - let copied_png = recordings.join("passphrase.png"); - fs::copy(passphrase_png, &copied_png) +pub struct CheckScreenshots<'a> { + pub passphrase_png: &'a Path, + pub failures_png: &'a Path, +} + +fn copy_screenshot( + screenshot: &Path, + recordings: &Path, + published_name: &str, +) -> Result<(), TaskError> { + let copied_png = recordings.join(published_name); + fs::copy(screenshot, &copied_png) .and_then(|_| fs::set_permissions(&copied_png, Permissions::from_mode(0o644))) .map_err(|source| TaskError::File { path: copied_png, source, }) } + +pub fn record_screencasts(screenshots: &CheckScreenshots) -> Result<(), TaskError> { + let recordings = Path::new("recordings"); + if !recordings.is_dir() { + return Err(TaskError::NotInRepositoryRoot); + } + let shell = Shell::new()?; + TAPES.iter().try_for_each(|tape| { + let tape_path = Path::new(tape); + cmd!(shell, "vhs {tape_path}").run() + })?; + copy_screenshot(screenshots.passphrase_png, recordings, "passphrase.png")?; + copy_screenshot(screenshots.failures_png, recordings, "failures.png") +}