diff --git a/Cargo.lock b/Cargo.lock index 3262ae3..55c2eda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,6 +158,7 @@ dependencies = [ "ser_nix", "serde", "serde_plain", + "systemd", "thiserror 2.0.21", "tokio", "toml", @@ -166,6 +167,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "build-env" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1522ac6ee801a11bf9ef3f80403f4ede6eb41291fac3dde3de09989679305f25" + [[package]] name = "bumpalo" version = "3.20.3" @@ -365,6 +372,16 @@ dependencies = [ "phf", ] +[[package]] +name = "cstr-argument" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bd9c8e659a473bce955ae5c35b116af38af11a7acb0b480e01f3ed348aeb40" +dependencies = [ + "cfg-if", + "memchr", +] + [[package]] name = "darling" version = "0.24.1" @@ -588,6 +605,33 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5190182e6915eb873ddbc16e23b711b6eb1f9c00a0d0a3a91b5f6228475225" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "futures" version = "0.3.34" @@ -880,6 +924,18 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libsystemd-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976306de183e6046819ef6505888d00996214766a3f4660a2ed5761c84a20aed" +dependencies = [ + "build-env", + "cfg-if", + "libc", + "pkg-config", +] + [[package]] name = "line-clipping" version = "0.3.8" @@ -1532,6 +1588,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scripted-demos" +version = "0.1.0" +dependencies = [ + "bootvine", + "futures", + "ratatui", + "tokio", + "zeroize", +] + [[package]] name = "semver" version = "1.0.28" @@ -1750,6 +1817,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "systemd" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e9d1976a15b86245def55d20d52b5818e1a1e81aa030b6a608d3ce57709423" +dependencies = [ + "cstr-argument", + "foreign-types", + "libc", + "libsystemd-sys", + "log", + "memchr", + "utf8-cstr", +] + [[package]] name = "termina" version = "0.3.3" @@ -1993,6 +2075,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "utf8-cstr" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55bcbb425141152b10d5693095950b51c3745d019363fc2929ffd8f61449b628" + [[package]] name = "utf8parse" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index 6dbf32e..52c9239 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,18 +1,6 @@ -[package] -categories = ["command-line-utilities", "os::linux-apis"] -description = "Boot and shutdown progress for systemd on the console, with each job nested under the units waiting on it" -edition = "2024" -include = ["/src", "/examples", "/system-files", "/README.md", "/LICENSE"] -keywords = ["systemd", "boot", "initrd", "tui", "console"] -license = "GPL-3.0-or-later" -name = "bootvine" -readme = "README.md" -repository = "https://codeberg.org/wvhulle/bootvine" -rust-version = "1.91" -version = "0.1.0" - [workspace] -members = [".", "xtask"] +members = ["bootvine", "scripted-demos", "xtask"] +resolver = "3" [workspace.lints.clippy] absolute_paths = "warn" @@ -20,60 +8,6 @@ missing_errors_doc = "allow" needless_for_each = "allow" pedantic = { priority = -1, level = "warn" } -[lints] -workspace = true - -[dependencies] -clap = { version = "4.6.7", default-features = false, features = [ - "std", - "derive", - "help", - "usage", - "error-context", -] } -crossterm = "0.29.0" -dbus = "0.9.12" -futures = { version = "0.3.34", default-features = false, features = [ - "std", - "async-await", -] } -heck = "0.5.0" -jiff = { version = "0.2.38", default-features = false, features = [ - "serde", - "std", -] } -petgraph = { version = "0.8.3", default-features = false, features = [ - "std", - "graphmap", -] } -ratatui = { version = "0.30.2", features = [ - "serde", - "unstable-rendered-line-info", -] } -rustix = { version = "1.1.5", features = [ - "event", - "fs", - "process", - "stdio", - "system", - "termios", - "time", -] } -ser_nix = "0.2.4" -serde = { version = "1.0.229", features = ["derive"] } -serde_plain = "1.0.2" -thiserror = "2.0.12" -tokio = { version = "1.54.0", default-features = false, features = [ - "rt", - "time", - "net", - "signal", -] } -toml = { version = "1.1.8", features = ["preserve_order"] } -unicode-truncate = "2.0.1" -unicode-width = "0.2.2" -zeroize = "1.9.1" - [profile.release] codegen-units = 1 lto = "fat" diff --git a/README.md b/README.md index d791983..6e8433a 100644 --- a/README.md +++ b/README.md @@ -11,21 +11,16 @@ gave me any idea of what was going on. A boot through the initrd into stage 2, with one failed unit -It follows shutdowns and reboots too: the units being stopped, nested under the -units that have to stop first. - -A poweroff: units stopping under the targets waiting on them - -Passphrase prompts, such as a -[LUKS](https://gitlab.com/cryptsetup/cryptsetup) unlock in the initrd, are -answered in place over the tree. - -A LUKS passphrase prompt floating over the initrd tree +It also follows shutdowns, answers passphrase prompts such as a +[LUKS](https://gitlab.com/cryptsetup/cryptsetup) unlock in place, and shows why +a unit failed with its last journal lines. The +[guide](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/guide.md) +walks through each part of the screen. ## Requirements - systemd ≥ 246 as PID 1 -- [libdbus](https://www.freedesktop.org/wiki/Software/dbus/) +- [libdbus](https://www.freedesktop.org/wiki/Software/dbus/) and libsystemd - a virtual terminal - a systemd initrd, to show the boot from the initrd as well as stage 2 @@ -34,33 +29,23 @@ for each distribution. ## Try it without installing -Neither of these touches your system's boot. - -Play a scripted boot, poweroff and encrypted-root boot in your terminal, with -[Rust](https://www.rust-lang.org/tools/install) installed: +With [Nix](https://nixos.org/download/), boot a sample [NixOS](https://nixos.org) machine with slow and +failing units in a [QEMU](https://www.qemu.org) window, without touching your +system's boot: ```sh -git clone https://codeberg.org/wvhulle/bootvine.git -cd bootvine -cargo run --example boot -cargo run --example shutdown -cargo run --example luks +nix run git+https://codeberg.org/wvhulle/bootvine#sample-vm-slow-services ``` -`q` quits. The encrypted root unlocks with the passphrase `demo`; `Esc` quits -while it asks. - -Or, with [Nix](https://nixos.org/download/), boot a sample [NixOS](https://nixos.org) machine with slow and -failing units in a [QEMU](https://www.qemu.org) window: - -```sh -nix run git+https://codeberg.org/wvhulle/bootvine#sample-machine-vm -``` - -It logs in as root; `poweroff` shows the shutdown. +It logs in as root; `poweroff` shows the shutdown. The +[guide](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/guide.md#trying-it) +lists the other sample machines and the scripted demos that play in a terminal. ## Documentation +- [Guide](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/guide.md) + to the screen: the tree, the timeline, failures, shutdown and passphrase + prompts. - [Install](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/install.md) on NixOS, Ubuntu and other distributions. - [Configuration](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/configuration.md) diff --git a/bootvine/Cargo.toml b/bootvine/Cargo.toml new file mode 100644 index 0000000..4745a15 --- /dev/null +++ b/bootvine/Cargo.toml @@ -0,0 +1,69 @@ +[package] +categories = ["command-line-utilities", "os::linux-apis"] +description = "Boot and shutdown progress for systemd on the console, with each job nested under the units waiting on it" +edition = "2024" +include = ["/src", "/system-files", "/LICENSE"] +keywords = ["systemd", "boot", "initrd", "tui", "console"] +license = "GPL-3.0-or-later" +name = "bootvine" +readme = "../README.md" +repository = "https://codeberg.org/wvhulle/bootvine" +rust-version = "1.91" +version = "0.1.0" + +[lints] +workspace = true + +[dependencies] +clap = { version = "4.6.7", default-features = false, features = [ + "std", + "derive", + "help", + "usage", + "error-context", +] } +crossterm = "0.29.0" +dbus = "0.9.12" +futures = { version = "0.3.34", default-features = false, features = [ + "std", + "async-await", +] } +heck = "0.5.0" +jiff = { version = "0.2.38", default-features = false, features = [ + "serde", + "std", +] } +petgraph = { version = "0.8.3", default-features = false, features = [ + "std", + "graphmap", +] } +ratatui = { version = "0.30.2", features = [ + "serde", + "unstable-rendered-line-info", +] } +rustix = { version = "1.1.5", features = [ + "event", + "fs", + "process", + "stdio", + "system", + "termios", + "time", +] } +ser_nix = "0.2.4" +serde = { version = "1.0.229", features = ["derive"] } +serde_plain = "1.0.2" +systemd = { version = "0.10.1", default-features = false, features = [ + "journal", +] } +thiserror = "2.0.12" +tokio = { version = "1.54.0", default-features = false, features = [ + "rt", + "time", + "net", + "signal", +] } +toml = { version = "1.1.8", features = ["preserve_order"] } +unicode-truncate = "2.0.1" +unicode-width = "0.2.2" +zeroize = "1.9.1" diff --git a/bootvine/LICENSE b/bootvine/LICENSE new file mode 120000 index 0000000..ea5b606 --- /dev/null +++ b/bootvine/LICENSE @@ -0,0 +1 @@ +../LICENSE \ No newline at end of file diff --git a/src/boot_model.rs b/bootvine/src/boot_model.rs similarity index 79% rename from src/boot_model.rs rename to bootvine/src/boot_model.rs index 4329907..f0628b8 100644 --- a/src/boot_model.rs +++ b/bootvine/src/boot_model.rs @@ -41,10 +41,44 @@ struct PendingJob { kind: JobKind, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FailureKind { + Broke, + TimedOut, + DependencyFailed, +} + +impl FailureKind { + #[must_use] + pub fn has_own_cause(self) -> bool { + self != Self::DependencyFailed + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FailureCause { + pub reason: String, + pub log_lines: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FailureExplanation { + Unexplained, + Explained(FailureCause), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FailedUnit { + pub unit: String, + pub label: String, + pub kind: FailureKind, + pub explanation: FailureExplanation, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Outcome { Succeeded { at: Duration }, - Failed, + Failed(FailureKind), Abandoned, } @@ -109,7 +143,7 @@ pub struct BootState { pub phase: Phase, pub origin: Duration, pub root: Root, - pub failed: Vec, + pub failed: Vec, pub finished_stages: Vec, pub finished_runs: Vec, pub handoff_unit: Option, diff --git a/src/boot_model/boot_change.rs b/bootvine/src/boot_model/boot_change.rs similarity index 91% rename from src/boot_model/boot_change.rs rename to bootvine/src/boot_model/boot_change.rs index 1c63e08..84b9f49 100644 --- a/src/boot_model/boot_change.rs +++ b/bootvine/src/boot_model/boot_change.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use super::{Activity, BootState, JobKind, JobResult, UnitFacts}; +use super::{Activity, BootState, FailureCause, JobKind, JobResult, UnitFacts}; pub enum BootChange { TrackUnit { @@ -22,6 +22,10 @@ pub enum BootChange { result: JobResult, at: Duration, }, + ExplainFailure { + unit: String, + cause: FailureCause, + }, SetRoot(String), SetHandoff(String), SetOrigin(Duration), @@ -42,6 +46,7 @@ impl BootState { BootChange::UpdateActivity { unit, activity } => self.update_activity(&unit, activity), BootChange::QueueJob { id, unit, kind, at } => self.queue_job(id, &unit, kind, at), BootChange::FinishJob { id, result, at } => self.finish_job(id, result, at), + BootChange::ExplainFailure { unit, cause } => self.explain_failure(&unit, cause), BootChange::SetRoot(root) => self.set_root(root), BootChange::SetHandoff(unit) => { if let Some(handoff) = &mut self.handoff_unit { @@ -62,7 +67,7 @@ mod tests { use super::*; use crate::{ boot_model::Phase, - demo_script::boot_test_fixtures::{MS, facts, waiting_rows}, + boot_test_fixtures::{MS, facts, waiting_rows}, }; #[test] diff --git a/src/boot_model/boot_origin.rs b/bootvine/src/boot_model/boot_origin.rs similarity index 100% rename from src/boot_model/boot_origin.rs rename to bootvine/src/boot_model/boot_origin.rs diff --git a/src/boot_model/job_ingestion.rs b/bootvine/src/boot_model/job_ingestion.rs similarity index 76% rename from src/boot_model/job_ingestion.rs rename to bootvine/src/boot_model/job_ingestion.rs index a5ef5f5..42266ac 100644 --- a/src/boot_model/job_ingestion.rs +++ b/bootvine/src/boot_model/job_ingestion.rs @@ -1,9 +1,10 @@ use std::{collections::HashMap, time::Duration}; use super::{ - ActiveState, Activity, BootState, FinishedRun, FinishedStage, HANDOFF_UNIT, HOST_MILESTONES, - JobKind, JobResult, Outcome, PendingJob, Phase, Progress, Root, SHUTDOWN_MILESTONES, Tally, - UnitFacts, UnitKind, UnitRecord, unit_facts::display_label, + ActiveState, Activity, BootState, FailedUnit, FailureCause, FailureExplanation, FailureKind, + FinishedRun, FinishedStage, HANDOFF_UNIT, HOST_MILESTONES, JobKind, JobResult, Outcome, + PendingJob, Phase, Progress, Root, SHUTDOWN_MILESTONES, Tally, UnitFacts, UnitKind, UnitRecord, + unit_facts::display_label, }; impl UnitRecord { @@ -116,11 +117,40 @@ impl BootState { } pub fn update_activity(&mut self, unit: &str, activity: Activity) { - if let Some(record) = self.units.get_mut(unit) { - record.activity = record.activity.merge(activity); + let Some(record) = self.units.get_mut(unit) else { + return; + }; + record.activity = record.activity.merge(activity); + if activity.state == Some(ActiveState::Failed) { + let label = display_label(&record.facts); + self.record_failure(unit, label, FailureKind::Broke); } } + fn record_failure(&mut self, unit: &str, label: String, kind: FailureKind) { + match self.failed.iter_mut().find(|failed| failed.unit == unit) { + Some(known) if kind != FailureKind::Broke => known.kind = kind, + Some(_) => {} + None => self.failed.push(FailedUnit { + unit: unit.to_owned(), + label, + kind, + explanation: FailureExplanation::Unexplained, + }), + } + } + + pub fn explain_failure(&mut self, unit: &str, cause: FailureCause) { + if let Some(failed) = self.failed.iter_mut().find(|failed| failed.unit == unit) { + failed.explanation = FailureExplanation::Explained(cause); + } + } + + #[must_use] + pub fn has_failed(&self, unit: &str) -> bool { + self.failed.iter().any(|failed| failed.unit == unit) + } + pub fn queue_job(&mut self, id: u32, unit: &str, kind: JobKind, at: Duration) { if self.enqueue(id, unit, kind, at) && kind == JobKind::BringsUp @@ -150,18 +180,26 @@ impl BootState { let Some(unit) = self.jobs.remove(&id) else { return; }; - let Some(record) = self.units.get_mut(&unit) else { + let Some(label) = self + .units + .get(&unit) + .map(|record| display_label(&record.facts)) + else { return; }; let outcome = match result { JobResult::Done => Outcome::Succeeded { at }, JobResult::Skipped | JobResult::Canceled => Outcome::Abandoned, - JobResult::Failed | JobResult::Timeout | JobResult::Dependency => Outcome::Failed, + JobResult::Failed => Outcome::Failed(FailureKind::Broke), + JobResult::Timeout => Outcome::Failed(FailureKind::TimedOut), + JobResult::Dependency => Outcome::Failed(FailureKind::DependencyFailed), }; - let label = display_label(&record.facts); - if outcome == Outcome::Failed && !self.failed.contains(&label) { - self.failed.push(label.clone()); + if let Outcome::Failed(kind) = outcome { + self.record_failure(&unit, label.clone(), kind); } + let Some(record) = self.units.get_mut(&unit) else { + return; + }; let job = match record.progress { Progress::Pending(job) if job.id == id => job, _ => return, @@ -176,7 +214,7 @@ impl BootState { label, started_at, finished_at: at, - failed: outcome == Outcome::Failed, + failed: matches!(outcome, Outcome::Failed(_)), }); } } @@ -193,7 +231,7 @@ impl BootState { failed_units: self .units .values() - .filter(|record| record.progress == Progress::Finished(Outcome::Failed)) + .filter(|record| matches!(record.progress, Progress::Finished(Outcome::Failed(_)))) .count(), started_at: self.origin, reached_at: at, @@ -254,7 +292,7 @@ impl BootState { Progress::Finished(Outcome::Succeeded { .. } | Outcome::Abandoned) => { tally.done += 1; } - Progress::Finished(Outcome::Failed) | Progress::Untouched => {} + Progress::Finished(Outcome::Failed(_)) | Progress::Untouched => {} } tally }) @@ -266,17 +304,82 @@ mod tests { use super::*; use crate::{ boot_model::{BootChange, RowStatus, UnitTimeouts}, - demo_script::boot_test_fixtures::{ + boot_test_fixtures::{ MS, activating, facts, nested_boot_sample as sample, queue_unit, queued_boot, running, waiting, waiting_rows, }, }; + fn failure_kinds(state: &BootState) -> Vec<(&str, FailureKind)> { + state + .failed + .iter() + .map(|failed| (failed.unit.as_str(), failed.kind)) + .collect() + } + + #[test] + fn job_results_keep_their_failure_kind() { + let mut state = sample(); + state.finish_job(4, JobResult::Timeout, 2000 * MS); + state.finish_job(3, JobResult::Dependency, 2000 * MS); + assert_eq!( + failure_kinds(&state), + [ + ("nm.service", FailureKind::TimedOut), + ("nm-wait.service", FailureKind::DependencyFailed), + ] + ); + } + + #[test] + fn a_unit_entering_the_failed_state_counts_without_a_failed_job() { + let mut state = sample(); + state.update_activity( + "nm.service", + Activity { + state: Some(ActiveState::Failed), + ..Activity::default() + }, + ); + assert_eq!(failure_kinds(&state), [("nm.service", FailureKind::Broke)]); + state.finish_job(4, JobResult::Timeout, 2000 * MS); + assert_eq!( + failure_kinds(&state), + [("nm.service", FailureKind::TimedOut)] + ); + assert_eq!(state.tally().failed, 1); + } + + #[test] + fn an_explanation_attaches_to_its_failed_unit() { + let mut state = sample(); + let cause = FailureCause { + reason: "exited with status 1".to_owned(), + log_lines: vec!["no carrier".to_owned()], + }; + state.explain_failure("nm.service", cause.clone()); + assert!(state.failed.is_empty()); + state.finish_job(4, JobResult::Failed, 2000 * MS); + state.explain_failure("nm.service", cause.clone()); + assert_eq!( + state.failed[0].explanation, + FailureExplanation::Explained(cause) + ); + } + #[test] fn failures_are_listed_and_not_counted_as_done() { let mut state = sample(); state.finish_job(4, JobResult::Failed, 2000 * MS); - assert_eq!(state.failed, ["nm"]); + assert_eq!( + state + .failed + .iter() + .map(|failed| failed.label.as_str()) + .collect::>(), + ["nm"] + ); let tally = state.tally(); assert_eq!(tally.failed, 1); assert_eq!(tally.done, 0); diff --git a/src/boot_model/reached_milestones.rs b/bootvine/src/boot_model/reached_milestones.rs similarity index 96% rename from src/boot_model/reached_milestones.rs rename to bootvine/src/boot_model/reached_milestones.rs index 48cfa16..efa0bf8 100644 --- a/src/boot_model/reached_milestones.rs +++ b/bootvine/src/boot_model/reached_milestones.rs @@ -23,7 +23,9 @@ impl BootState { Progress::Untouched => record.activity.active_enter.filter(|entered| { record.activity.state == Some(ActiveState::Active) && *entered >= self.origin }), - Progress::Pending(_) | Progress::Finished(Outcome::Failed | Outcome::Abandoned) => None, + Progress::Pending(_) | Progress::Finished(Outcome::Failed(_) | Outcome::Abandoned) => { + None + } } } @@ -81,9 +83,7 @@ mod tests { use super::*; use crate::{ boot_model::{Activity, JobResult, Phase}, - demo_script::boot_test_fixtures::{ - MS, nested_boot_sample, queued_boot, waiting, waiting_rows, - }, + boot_test_fixtures::{MS, nested_boot_sample, queued_boot, waiting, waiting_rows}, }; #[test] diff --git a/src/boot_model/unit_facts.rs b/bootvine/src/boot_model/unit_facts.rs similarity index 98% rename from src/boot_model/unit_facts.rs rename to bootvine/src/boot_model/unit_facts.rs index d986ffc..b4dc35b 100644 --- a/src/boot_model/unit_facts.rs +++ b/bootvine/src/boot_model/unit_facts.rs @@ -152,7 +152,7 @@ mod tests { use serde_plain::from_str; use super::*; - use crate::demo_script::boot_test_fixtures::{facts, waiting}; + use crate::boot_test_fixtures::{facts, waiting}; #[test] fn unknown_systemd_strings_fall_back_instead_of_failing() { diff --git a/src/boot_model/waiting_tree.rs b/bootvine/src/boot_model/waiting_tree.rs similarity index 97% rename from src/boot_model/waiting_tree.rs rename to bootvine/src/boot_model/waiting_tree.rs index 7cf6115..0cfc78a 100644 --- a/src/boot_model/waiting_tree.rs +++ b/bootvine/src/boot_model/waiting_tree.rs @@ -7,7 +7,7 @@ use petgraph::{ }; use super::{ - BootState, Outcome, Phase, Progress, UnitKind, UnitRecord, + BootState, FailureKind, Outcome, Phase, Progress, UnitKind, UnitRecord, unit_facts::{display_label, summary}, }; @@ -31,7 +31,7 @@ pub enum RowStatus { since: Duration, deadline: Option, }, - Failed, + Failed(FailureKind), Done, } @@ -87,7 +87,7 @@ impl UnitRecord { deadline: self.deadline(), }) } - Progress::Finished(Outcome::Failed) => RowStatus::Failed, + Progress::Finished(Outcome::Failed(kind)) => RowStatus::Failed(kind), Progress::Finished(_) | Progress::Untouched => RowStatus::Done, } } @@ -100,7 +100,7 @@ impl BootState { self.units .iter() .filter(|(name, record)| match record.progress { - Progress::Pending(_) | Progress::Finished(Outcome::Failed) => true, + Progress::Pending(_) | Progress::Finished(Outcome::Failed(_)) => true, Progress::Finished(Outcome::Succeeded { at }) => { at >= finished_since || self.is_milestone(name) } @@ -240,7 +240,7 @@ impl BootState { Progress::Finished(Outcome::Succeeded { at }) => { SiblingRank::RunningSince(record.activity.inactive_exit.unwrap_or(at)) } - Progress::Finished(Outcome::Failed) => SiblingRank::Failed, + Progress::Finished(Outcome::Failed(_)) => SiblingRank::Failed, Progress::Finished(Outcome::Abandoned) | Progress::Untouched => { SiblingRank::Waiting } @@ -253,7 +253,7 @@ impl BootState { #[cfg(test)] mod tests { use super::*; - use crate::demo_script::boot_test_fixtures::{ + use crate::boot_test_fixtures::{ nested_boot_sample, queued_boot, tree_outline, waiting, waiting_rows, }; diff --git a/src/demo_script/boot_test_fixtures.rs b/bootvine/src/boot_test_fixtures.rs similarity index 100% rename from src/demo_script/boot_test_fixtures.rs rename to bootvine/src/boot_test_fixtures.rs diff --git a/src/boot_view.rs b/bootvine/src/boot_view.rs similarity index 77% rename from src/boot_view.rs rename to bootvine/src/boot_view.rs index 488f292..cf2bf57 100644 --- a/src/boot_view.rs +++ b/bootvine/src/boot_view.rs @@ -1,3 +1,4 @@ +mod failure_panel; #[cfg(test)] mod frame_cost_bench; #[cfg(test)] @@ -27,6 +28,7 @@ use unicode_width::UnicodeWidthStr; pub use self::screen_chrome::PasswordPrompt; use self::{ + failure_panel::FailurePanel, run_timeline::{RunningRun, Timeline}, screen_chrome::{FOOTER_HEIGHT, Footer, Header, PasswordPopup}, stage_summary::Stages, @@ -46,6 +48,9 @@ const MIN_BAR_WIDTH: u16 = 12; const MIN_HEIGHT_FOR_TIMELINE: u16 = 12; const MIN_TIMELINE_HEIGHT: u16 = run_timeline::CHROME_HEIGHT + 4; const HEADER_HEIGHT: u16 = 1; +const MIN_FAILURE_PANEL_HEIGHT: u16 = failure_panel::RULE_HEIGHT + 1; +const RESERVED_FAILURE_PANEL_HEIGHT: u16 = failure_panel::RULE_HEIGHT + 3; +const MIN_TREE_HEIGHT_BESIDE_FAILURES: u16 = 4; const BRANCH: &str = "├─ "; const LAST_BRANCH: &str = "└─ "; const PASSING_TRUNK: &str = "│ "; @@ -75,6 +80,20 @@ pub struct BootView<'a> { } impl BootView<'_> { + fn failure_panel(&self) -> FailurePanel<'_> { + FailurePanel { + newest_first: self + .state + .failed + .iter() + .rev() + .filter(|failed| failed.kind.has_own_cause()) + .collect(), + colors: &self.colors.failures, + glyphs: self.glyphs, + } + } + fn timeline<'a>(&'a self, rows: &'a [TreeRow]) -> Timeline<'a> { let mut running: Vec = rows .iter() @@ -135,6 +154,7 @@ impl Widget for BootView<'_> { glyphs: self.glyphs, }; let timeline = self.timeline(&all_rows); + let failure_panel = self.failure_panel(); let footer = Footer { tally: self.state.tally(), colors: &self.colors.footer, @@ -148,6 +168,7 @@ impl Widget for BootView<'_> { let heights = Heights::share( free, UnitTree::height_of(pending_count, 0), + failure_panel.height(), if fits_timeline { timeline.height() } else { 0 }, ); let budget = usize::from(heights.tree.saturating_sub(1)); @@ -168,6 +189,7 @@ impl Widget for BootView<'_> { stage_area, tree_area, _, + failure_area, timeline_area, footer_area, ] = Layout::vertical([ @@ -175,6 +197,7 @@ impl Widget for BootView<'_> { Constraint::Length(stages.height()), Constraint::Length(tree_height), Constraint::Fill(1), + Constraint::Length(heights.failures), Constraint::Length(heights.timeline), Constraint::Length(FOOTER_HEIGHT), ]) @@ -183,6 +206,9 @@ impl Widget for BootView<'_> { header.render(header_area, buf); stages.render(stage_area, buf); tree.render(tree_area, buf); + if heights.failures > 0 { + failure_panel.render(failure_area, buf); + } if heights.timeline > 0 { timeline.render(timeline_area, buf); } @@ -200,19 +226,32 @@ impl Widget for BootView<'_> { struct Heights { tree: u16, + failures: u16, timeline: u16, } impl Heights { - fn share(free: u16, tree_need: u16, timeline_need: u16) -> Self { - let leftover = free.saturating_sub(tree_need); + fn share(free: u16, tree_need: u16, failures_need: u16, timeline_need: u16) -> Self { + let reserved = RESERVED_FAILURE_PANEL_HEIGHT + .min(failures_need) + .min(free.saturating_sub(MIN_TREE_HEIGHT_BESIDE_FAILURES)); + let failures = failures_need + .min(free.saturating_sub(tree_need)) + .max(reserved); + let failures = if failures >= MIN_FAILURE_PANEL_HEIGHT { + failures + } else { + 0 + }; + let leftover = free.saturating_sub(tree_need).saturating_sub(failures); let timeline = if leftover >= MIN_TIMELINE_HEIGHT { timeline_need.min(leftover) } else { 0 }; Self { - tree: free - timeline, + tree: free - failures - timeline, + failures, timeline, } } @@ -264,13 +303,32 @@ mod tests { #[test] fn the_tree_takes_the_room_and_the_timeline_only_what_is_left() { - let heights = Heights::share(30, 40, 50); + let heights = Heights::share(30, 40, 0, 50); assert_eq!((heights.tree, heights.timeline), (30, 0)); - let heights = Heights::share(30, 12, 50); + let heights = Heights::share(30, 12, 0, 50); assert_eq!((heights.tree, heights.timeline), (12, 18)); - let heights = Heights::share(30, 25, 50); + let heights = Heights::share(30, 25, 0, 50); assert_eq!((heights.tree, heights.timeline), (30, 0)); - let heights = Heights::share(30, 12, 8); + let heights = Heights::share(30, 12, 0, 8); assert_eq!((heights.tree, heights.timeline), (22, 8)); } + + #[test] + fn failures_come_before_the_timeline_and_keep_a_few_rows_beside_a_full_tree() { + let heights = Heights::share(30, 12, 6, 50); + assert_eq!( + (heights.tree, heights.failures, heights.timeline), + (12, 6, 12) + ); + let heights = Heights::share(30, 40, 9, 50); + assert_eq!( + (heights.tree, heights.failures, heights.timeline), + (26, 4, 0) + ); + let heights = Heights::share(5, 40, 9, 50); + assert_eq!( + (heights.tree, heights.failures, heights.timeline), + (5, 0, 0) + ); + } } diff --git a/bootvine/src/boot_view/failure_panel.rs b/bootvine/src/boot_view/failure_panel.rs new file mode 100644 index 0000000..8b8b9b9 --- /dev/null +++ b/bootvine/src/boot_view/failure_panel.rs @@ -0,0 +1,227 @@ +use std::iter::once; + +use ratatui::{ + buffer::Buffer, + layout::Rect, + style::Stylize, + text::{Line, Span}, + widgets::{Paragraph, Widget}, +}; +use unicode_width::UnicodeWidthStr; + +use super::{fit_width, rule}; +use crate::{ + boot_model::{FailedUnit, FailureExplanation}, + screen_settings::{color_scheme::FailureColors, font_glyphs::Glyphs}, +}; + +pub const RULE_HEIGHT: u16 = 1; +const LOG_INDENT: &str = " "; +const REASON_GAP: &str = " "; + +pub struct FailurePanel<'a> { + pub newest_first: Vec<&'a FailedUnit>, + pub colors: &'a FailureColors, + pub glyphs: &'a Glyphs, +} + +fn log_lines(failed: &FailedUnit) -> &[String] { + match &failed.explanation { + FailureExplanation::Explained(cause) => &cause.log_lines, + FailureExplanation::Unexplained => &[], + } +} + +impl FailurePanel<'_> { + pub fn height(&self) -> u16 { + if self.newest_first.is_empty() { + return 0; + } + let rows: usize = self + .newest_first + .iter() + .map(|failed| 1 + log_lines(failed).len()) + .sum(); + RULE_HEIGHT.saturating_add(u16::try_from(rows).unwrap_or(u16::MAX)) + } + + fn headline(&self, failed: &FailedUnit, width: usize) -> Line<'static> { + let marker = format!("{} ", self.glyphs.failed); + let label_room = width.saturating_sub(marker.width()); + let label = fit_width(&failed.label, label_room, self.glyphs.ellipsis); + let reason_room = label_room.saturating_sub(label.width() + REASON_GAP.width()); + let reason = match &failed.explanation { + FailureExplanation::Explained(cause) if reason_room > 0 => { + fit_width(&cause.reason, reason_room, self.glyphs.ellipsis) + } + _ => String::new(), + }; + let reason_span = + (!reason.is_empty()).then(|| format!("{REASON_GAP}{reason}").fg(self.colors.reason)); + Line::from( + [ + marker.fg(self.colors.unit), + label.fg(self.colors.unit).bold(), + ] + .into_iter() + .chain(reason_span) + .collect::>(), + ) + } + + fn block(&self, failed: &FailedUnit, width: usize) -> Vec> { + let log_room = width.saturating_sub(LOG_INDENT.width()); + once(self.headline(failed, width)) + .chain(log_lines(failed).iter().map(|line| { + format!( + "{LOG_INDENT}{}", + fit_width(line, log_room, self.glyphs.ellipsis) + ) + .fg(self.colors.log) + .into() + })) + .collect() + } + + fn lines_within(&self, rows: usize, width: usize) -> Vec> { + if self.newest_first.len() > rows { + return self.headlines_with_overflow(rows, width); + } + let mut spare_rows = rows - self.newest_first.len(); + self.newest_first + .iter() + .flat_map(|failed| { + let mut block = self.block(failed, width); + let kept_log_lines = (block.len() - 1).min(spare_rows); + spare_rows -= kept_log_lines; + block.truncate(1 + kept_log_lines); + block + }) + .collect() + } + + fn headlines_with_overflow(&self, rows: usize, width: usize) -> Vec> { + let Some(headline_rows) = rows.checked_sub(1) else { + return Vec::new(); + }; + let hidden = self.newest_first.len() - headline_rows; + self.newest_first + .iter() + .take(headline_rows) + .map(|failed| self.headline(failed, width)) + .chain(once( + Line::from(format!("{} {hidden} more failed", self.glyphs.ellipsis)) + .fg(self.colors.overflow), + )) + .collect() + } +} + +impl Widget for FailurePanel<'_> { + fn render(self, area: Rect, buf: &mut Buffer) { + let block = rule( + self.colors.rule, + Line::from(" Failures ".fg(self.colors.title)), + ); + let inner = block.inner(area); + block.render(area, buf); + Paragraph::new(self.lines_within(usize::from(inner.height), usize::from(inner.width))) + .render(inner, buf); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::boot_model::{FailureCause, FailureKind}; + + fn explained(label: &str, log_lines: &[&str]) -> FailedUnit { + FailedUnit { + unit: format!("{label}.service"), + label: label.to_owned(), + kind: FailureKind::Broke, + explanation: FailureExplanation::Explained(FailureCause { + reason: "exited with status 1".to_owned(), + log_lines: log_lines.iter().map(ToString::to_string).collect(), + }), + } + } + + fn panel_text(failures: &[FailedUnit], rows: usize) -> Vec { + FailurePanel { + newest_first: failures.iter().collect(), + colors: &FailureColors::default(), + glyphs: &Glyphs::fitting(|_| true), + } + .lines_within(rows, 60) + .iter() + .map(ToString::to_string) + .collect() + } + + #[test] + fn the_newest_failure_keeps_its_lines_and_older_ones_shrink_to_a_headline() { + let failures = [ + explained("web", &["bind: address in use", "giving up"]), + explained("db", &["disk full"]), + explained("cache", &["no memory"]), + ]; + assert_eq!( + panel_text(&failures, 5), + [ + "× web exited with status 1", + " bind: address in use", + " giving up", + "× db exited with status 1", + "× cache exited with status 1", + ] + ); + } + + #[test] + fn room_left_by_the_newest_failure_goes_to_older_ones() { + let failures = [ + explained("web", &["giving up"]), + explained("db", &["disk full", "read-only file system"]), + explained("cache", &["no memory"]), + ]; + assert_eq!( + panel_text(&failures, 6), + [ + "× web exited with status 1", + " giving up", + "× db exited with status 1", + " disk full", + " read-only file system", + "× cache exited with status 1", + ] + ); + } + + #[test] + fn failures_beyond_the_room_are_counted() { + let failures = [ + explained("web", &["giving up"]), + explained("db", &[]), + explained("cache", &[]), + explained("queue", &[]), + ]; + assert_eq!( + panel_text(&failures, 3), + [ + "× web exited with status 1", + "× db exited with status 1", + "… 2 more failed" + ] + ); + } + + #[test] + fn an_unexplained_failure_shows_only_its_name() { + let failures = [FailedUnit { + explanation: FailureExplanation::Unexplained, + ..explained("web", &[]) + }]; + assert_eq!(panel_text(&failures, 4), ["× web"]); + } +} diff --git a/src/boot_view/frame_cost_bench.rs b/bootvine/src/boot_view/frame_cost_bench.rs similarity index 97% rename from src/boot_view/frame_cost_bench.rs rename to bootvine/src/boot_view/frame_cost_bench.rs index f2d1efd..3b8d989 100644 --- a/src/boot_view/frame_cost_bench.rs +++ b/bootvine/src/boot_view/frame_cost_bench.rs @@ -9,7 +9,7 @@ use ratatui::{Terminal, backend::TestBackend}; use super::{BootProjection, BootView, project_boot, screen_render_fixtures::ryzen_origin}; use crate::{ boot_model::{BootState, JobResult, Phase, flatten_forest}, - demo_script::boot_test_fixtures::{MS, QueuedUnit, queued_boot, running, waiting}, + boot_test_fixtures::{MS, QueuedUnit, queued_boot, running, waiting}, screen_settings::{color_scheme::Colors, config_file::Timing, font_glyphs::Glyphs}, }; diff --git a/src/boot_view/panel_render_tests.rs b/bootvine/src/boot_view/panel_render_tests.rs similarity index 82% rename from src/boot_view/panel_render_tests.rs rename to bootvine/src/boot_view/panel_render_tests.rs index 98193fe..a770195 100644 --- a/src/boot_view/panel_render_tests.rs +++ b/bootvine/src/boot_view/panel_render_tests.rs @@ -5,8 +5,8 @@ use super::{ screen_render_fixtures::{boot_in_progress, render, render_prompted, screen_lines}, }; use crate::{ - boot_model::{ActiveState, Activity, BootState, HANDOFF_UNIT, JobResult, Phase}, - demo_script::boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting}, + boot_model::{ActiveState, Activity, BootState, FailureCause, HANDOFF_UNIT, JobResult, Phase}, + boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting}, screen_settings::color_scheme::Colors, }; @@ -391,3 +391,82 @@ fn the_timeline_keeps_only_the_slowest_finished_runs() { "{screen}" ); } + +fn explained_nvidia_failure() -> BootState { + let mut state = boot_in_progress(); + state.explain_failure( + "nvidia-powerd.service", + FailureCause { + reason: "exited with status 1".to_owned(), + log_lines: vec![ + "ERROR! Failed to get system power limits".to_owned(), + "giving up".to_owned(), + ], + }, + ); + state +} + +#[test] +fn failures_show_their_reason_and_journal_lines_above_the_timeline() { + let lines = render(&explained_nvidia_failure(), 12_400 * MS, 110, 30); + let screen = lines.join("\n"); + let failures = lines + .iter() + .position(|line| line.contains(" Failures ")) + .expect("failure panel"); + let timeline = lines + .iter() + .position(|line| line.contains(" Timeline ")) + .expect("timeline panel"); + assert!(failures < timeline, "{screen}"); + assert_eq!( + lines[failures + 1], + "× nvidia-powerd exited with status 1", + "{screen}" + ); + assert_eq!( + lines[failures + 2], + " ERROR! Failed to get system power limits", + "{screen}" + ); + assert_eq!(lines[failures + 3], " giving up", "{screen}"); +} + +#[test] +fn failure_lines_are_cut_to_a_narrow_screen() { + let lines = render(&explained_nvidia_failure(), 12_400 * MS, 30, 30); + let screen = lines.join("\n"); + let failures = lines + .iter() + .position(|line| line.contains(" Failures ")) + .expect("failure panel"); + assert_eq!( + lines[failures + 1], + "× nvidia-powerd exited with …", + "{screen}" + ); + assert_eq!( + lines[failures + 2], + " ERROR! Failed to get syst…", + "{screen}" + ); +} + +#[test] +fn dependency_failures_stay_out_of_the_failure_panel() { + let mut state = queued_boot( + Phase::Host, + 1000 * MS, + [ + waiting("multi-user.target", &["web.service"]), + waiting("web.service", &[]), + ], + ); + state.finish_job(2, JobResult::Dependency, 2000 * MS); + let lines = render(&state, 3000 * MS, 110, 24); + let screen = lines.join("\n"); + assert!(!screen.contains(" Failures "), "{screen}"); + assert!(screen.contains("[×] web"), "{screen}"); + assert!(screen.contains("× 1 failed"), "{screen}"); +} diff --git a/src/boot_view/run_timeline.rs b/bootvine/src/boot_view/run_timeline.rs similarity index 100% rename from src/boot_view/run_timeline.rs rename to bootvine/src/boot_view/run_timeline.rs diff --git a/src/boot_view/screen_chrome.rs b/bootvine/src/boot_view/screen_chrome.rs similarity index 100% rename from src/boot_view/screen_chrome.rs rename to bootvine/src/boot_view/screen_chrome.rs diff --git a/src/boot_view/screen_render_fixtures.rs b/bootvine/src/boot_view/screen_render_fixtures.rs similarity index 98% rename from src/boot_view/screen_render_fixtures.rs rename to bootvine/src/boot_view/screen_render_fixtures.rs index feff27a..ee322a1 100644 --- a/src/boot_view/screen_render_fixtures.rs +++ b/bootvine/src/boot_view/screen_render_fixtures.rs @@ -5,7 +5,7 @@ use ratatui::{Terminal, backend::TestBackend, buffer::Buffer}; use super::{BootView, PasswordPrompt, project_boot}; use crate::{ boot_model::{BootState, JobResult, Phase, boot_origin::BootOrigin}, - demo_script::boot_test_fixtures::{MS, queued_boot, running, waiting}, + boot_test_fixtures::{MS, queued_boot, running, waiting}, screen_settings::{color_scheme::Colors, config_file::Timing, font_glyphs::Glyphs}, }; diff --git a/src/boot_view/stage_summary.rs b/bootvine/src/boot_view/stage_summary.rs similarity index 100% rename from src/boot_view/stage_summary.rs rename to bootvine/src/boot_view/stage_summary.rs diff --git a/src/boot_view/tree_height_fitting.rs b/bootvine/src/boot_view/tree_height_fitting.rs similarity index 98% rename from src/boot_view/tree_height_fitting.rs rename to bootvine/src/boot_view/tree_height_fitting.rs index f63c22e..16fe8db 100644 --- a/src/boot_view/tree_height_fitting.rs +++ b/bootvine/src/boot_view/tree_height_fitting.rs @@ -34,7 +34,7 @@ impl TreeNode { fn leads_to_running(&self) -> bool { matches!( self.row.status, - RowStatus::Running { .. } | RowStatus::Failed + RowStatus::Running { .. } | RowStatus::Failed(_) ) || self.children.iter().any(Self::leads_to_running) } @@ -95,7 +95,7 @@ mod tests { use super::*; use crate::{ boot_model::{BootState, Phase}, - demo_script::boot_test_fixtures::{ + boot_test_fixtures::{ activating, nested_boot_sample, queued_boot, running, tree_outline, waiting, }, }; diff --git a/src/boot_view/tree_render_tests.rs b/bootvine/src/boot_view/tree_render_tests.rs similarity index 98% rename from src/boot_view/tree_render_tests.rs rename to bootvine/src/boot_view/tree_render_tests.rs index 8990f7b..88879a8 100644 --- a/src/boot_view/tree_render_tests.rs +++ b/bootvine/src/boot_view/tree_render_tests.rs @@ -5,7 +5,7 @@ use unicode_width::UnicodeWidthStr; use super::screen_render_fixtures::{boot_in_progress, chain_boot, described_boot, render}; use crate::{ boot_model::{BootState, JobResult, Phase, UnitTimeouts}, - demo_script::boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting}, + boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting}, }; #[test] diff --git a/src/boot_view/unit_tree.rs b/bootvine/src/boot_view/unit_tree.rs similarity index 97% rename from src/boot_view/unit_tree.rs rename to bootvine/src/boot_view/unit_tree.rs index 55389a3..c8f8dda 100644 --- a/src/boot_view/unit_tree.rs +++ b/bootvine/src/boot_view/unit_tree.rs @@ -166,7 +166,10 @@ impl UnitTree<'_> { fn unit_label_style(&self, row: &TreeRow) -> Style { match row.status { - RowStatus::Failed => Style::from(self.colors.failed_unit).bold(), + RowStatus::Failed(kind) if kind.has_own_cause() => { + Style::from(self.colors.failed_unit).bold() + } + RowStatus::Failed(_) => Style::from(self.colors.failed_unit).dim(), _ if row.kind == UnitKind::Target => self.colors.target.into(), RowStatus::Running { .. } => self.colors.running_unit.into(), RowStatus::Waiting => self.colors.waiting_unit.into(), @@ -187,7 +190,9 @@ impl UnitTree<'_> { fn unit_marker(&self, row: &TreeRow) -> Span<'static> { match row.status { - RowStatus::Failed => format!("[{}] ", self.glyphs.failed).fg(self.colors.failed_unit), + RowStatus::Failed(_) => { + format!("[{}] ", self.glyphs.failed).fg(self.colors.failed_unit) + } _ if row.kind == UnitKind::Target => Span::default(), RowStatus::Running { .. } => { format!("[{}] ", spinner_at(self.now)).fg(self.colors.running_marker) diff --git a/src/console_session.rs b/bootvine/src/console_session.rs similarity index 100% rename from src/console_session.rs rename to bootvine/src/console_session.rs diff --git a/src/console_session/ask_password.rs b/bootvine/src/console_session/ask_password.rs similarity index 100% rename from src/console_session/ask_password.rs rename to bootvine/src/console_session/ask_password.rs diff --git a/src/console_session/console_palette.rs b/bootvine/src/console_session/console_palette.rs similarity index 100% rename from src/console_session/console_palette.rs rename to bootvine/src/console_session/console_palette.rs diff --git a/src/console_session/console_takeover.rs b/bootvine/src/console_session/console_takeover.rs similarity index 100% rename from src/console_session/console_takeover.rs rename to bootvine/src/console_session/console_takeover.rs diff --git a/src/console_session/live_desk.rs b/bootvine/src/console_session/live_desk.rs similarity index 100% rename from src/console_session/live_desk.rs rename to bootvine/src/console_session/live_desk.rs diff --git a/src/console_session/passphrase_input.rs b/bootvine/src/console_session/passphrase_input.rs similarity index 100% rename from src/console_session/passphrase_input.rs rename to bootvine/src/console_session/passphrase_input.rs diff --git a/src/console_session/stream_sources.rs b/bootvine/src/console_session/stream_sources.rs similarity index 100% rename from src/console_session/stream_sources.rs rename to bootvine/src/console_session/stream_sources.rs diff --git a/src/console_session/watch_loop.rs b/bootvine/src/console_session/watch_loop.rs similarity index 100% rename from src/console_session/watch_loop.rs rename to bootvine/src/console_session/watch_loop.rs diff --git a/src/console_session/watch_screen.rs b/bootvine/src/console_session/watch_screen.rs similarity index 99% rename from src/console_session/watch_screen.rs rename to bootvine/src/console_session/watch_screen.rs index 270b91d..cb1c59e 100644 --- a/src/console_session/watch_screen.rs +++ b/bootvine/src/console_session/watch_screen.rs @@ -273,7 +273,7 @@ mod tests { use super::*; use crate::{ boot_model::{ActiveState, Activity, HANDOFF_UNIT}, - demo_script::boot_test_fixtures::{MS, nested_boot_sample, queue_unit, waiting}, + boot_test_fixtures::{MS, nested_boot_sample, queue_unit, waiting}, }; #[derive(Default)] diff --git a/src/lib.rs b/bootvine/src/lib.rs similarity index 78% rename from src/lib.rs rename to bootvine/src/lib.rs index 1a5c142..7ab6ec2 100644 --- a/src/lib.rs +++ b/bootvine/src/lib.rs @@ -1,7 +1,8 @@ pub mod boot_model; +#[cfg(test)] +mod boot_test_fixtures; pub mod boot_view; pub mod console_session; -pub mod demo_script; pub mod screen_settings; pub mod system_install; pub mod systemd_link; diff --git a/src/main.rs b/bootvine/src/main.rs similarity index 100% rename from src/main.rs rename to bootvine/src/main.rs diff --git a/src/screen_settings.rs b/bootvine/src/screen_settings.rs similarity index 100% rename from src/screen_settings.rs rename to bootvine/src/screen_settings.rs diff --git a/src/screen_settings/color_scheme.rs b/bootvine/src/screen_settings/color_scheme.rs similarity index 88% rename from src/screen_settings/color_scheme.rs rename to bootvine/src/screen_settings/color_scheme.rs index 0fdbcfe..be7f0c3 100644 --- a/src/screen_settings/color_scheme.rs +++ b/bootvine/src/screen_settings/color_scheme.rs @@ -8,6 +8,7 @@ pub struct Colors { pub stages: StageColors, pub tree: TreeColors, pub timeline: TimelineColors, + pub failures: FailureColors, pub footer: FooterColors, pub password: PasswordColors, } @@ -122,6 +123,30 @@ impl Default for TimelineColors { } } +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields, rename_all = "kebab-case")] +pub struct FailureColors { + pub rule: Color, + pub title: Color, + pub unit: Color, + pub reason: Color, + pub log: Color, + pub overflow: Color, +} + +impl Default for FailureColors { + fn default() -> Self { + Self { + rule: Color::DarkGray, + title: Color::LightRed, + unit: Color::LightRed, + reason: Color::White, + log: Color::Gray, + overflow: Color::DarkGray, + } + } +} + #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(default, deny_unknown_fields, rename_all = "kebab-case")] pub struct FooterColors { diff --git a/src/screen_settings/config_file.rs b/bootvine/src/screen_settings/config_file.rs similarity index 100% rename from src/screen_settings/config_file.rs rename to bootvine/src/screen_settings/config_file.rs diff --git a/src/screen_settings/font_glyphs.rs b/bootvine/src/screen_settings/font_glyphs.rs similarity index 100% rename from src/screen_settings/font_glyphs.rs rename to bootvine/src/screen_settings/font_glyphs.rs diff --git a/src/system_install.rs b/bootvine/src/system_install.rs similarity index 100% rename from src/system_install.rs rename to bootvine/src/system_install.rs diff --git a/src/system_install/install_error.rs b/bootvine/src/system_install/install_error.rs similarity index 100% rename from src/system_install/install_error.rs rename to bootvine/src/system_install/install_error.rs diff --git a/src/system_install/install_files.rs b/bootvine/src/system_install/install_files.rs similarity index 100% rename from src/system_install/install_files.rs rename to bootvine/src/system_install/install_files.rs diff --git a/src/systemd_link.rs b/bootvine/src/systemd_link.rs similarity index 85% rename from src/systemd_link.rs rename to bootvine/src/systemd_link.rs index dc87d22..466b59c 100644 --- a/src/systemd_link.rs +++ b/bootvine/src/systemd_link.rs @@ -1,4 +1,5 @@ pub mod boot_feed; +mod journal_tail; pub mod link_error; pub mod monotonic_clock; pub mod switch_root; diff --git a/src/systemd_link/boot_feed.rs b/bootvine/src/systemd_link/boot_feed.rs similarity index 76% rename from src/systemd_link/boot_feed.rs rename to bootvine/src/systemd_link/boot_feed.rs index 05d4625..05f3d9d 100644 --- a/src/systemd_link/boot_feed.rs +++ b/bootvine/src/systemd_link/boot_feed.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, iter::once, process, time::{Duration, Instant}, @@ -10,12 +10,16 @@ use futures::{Stream, StreamExt, stream}; use tokio::time::sleep; use crate::{ - boot_model::{Activity, BootChange, BootState, HANDOFF_UNIT, JobKind, Phase, UnitFacts}, + boot_model::{ + ActiveState, Activity, BootChange, BootState, FailureCause, HANDOFF_UNIT, JobKind, + JobResult, Phase, UnitFacts, + }, console_session::{ stream_sources::{BACKGROUND_CHECK_INTERVAL, CarriedItem, unfold_carrying}, watch_screen::ScreenInput, }, systemd_link::{ + journal_tail::recent_unit_messages, link_error::Error, monotonic_clock::monotonic_now, switch_root::Sysroot, @@ -27,6 +31,12 @@ use crate::{ }; const RECONNECT_PATIENCE: Duration = Duration::from_secs(60); +const JOURNAL_SETTLING_WAITS: [Duration; 3] = [ + Duration::from_millis(200), + Duration::from_millis(300), + Duration::from_millis(500), +]; +const LOG_LINES_KEPT: usize = 6; const SWITCH_ROOT_UNIT: &str = "initrd-switch-root.service"; const SHUTDOWN_ROOTS: [&str; 5] = [ "poweroff.target", @@ -108,6 +118,8 @@ struct BootTracker { target: RootRequest, phase: Phase, unit_paths: HashMap, String>, + explained_failures: HashSet, + units_in_failed_state: HashSet, root_switch: RootSwitch, } @@ -131,6 +143,8 @@ impl BusFeed { target, phase, unit_paths: HashMap::new(), + explained_failures: HashSet::new(), + units_in_failed_state: HashSet::new(), root_switch: RootSwitch::NotQueued, }, } @@ -196,6 +210,8 @@ impl BusFeed { } tracker.root_switch = RootSwitch::NotQueued; tracker.unit_paths.clear(); + tracker.explained_failures.clear(); + tracker.units_in_failed_state.clear(); tracker.phase = Phase::Host; self.link = Link::connecting(ConnectAttempt::Again); vec![ScreenInput::Boot(vec![BootChange::SwitchRoot { at }])] @@ -395,20 +411,68 @@ impl BootTracker { Signal::JobNew { id, job, unit } => { self.queue_new_job(connection, id, &job, unit, at).await } - Signal::JobRemoved { id, result } => vec![BootChange::FinishJob { id, result, at }], - Signal::UnitChanged { path, activity } => self - .unit_paths - .get(&path) - .map(|unit| BootChange::UpdateActivity { - unit: unit.clone(), - activity, - }) - .into_iter() - .collect(), + Signal::JobRemoved { id, unit, result } => { + let explanation = match result { + JobResult::Failed => self.explain_failure(connection, unit, "failed").await, + JobResult::Timeout => self.explain_failure(connection, unit, "timed out").await, + _ => None, + }; + once(BootChange::FinishJob { id, result, at }) + .chain(explanation) + .collect() + } + Signal::UnitChanged { path, activity } => { + let Some(unit) = self.unit_paths.get(&path).cloned() else { + return Vec::new(); + }; + let explanation = if self.enters_failed_state(&unit, activity) { + self.explain_failure(connection, unit.clone(), "failed") + .await + } else { + None + }; + once(BootChange::UpdateActivity { unit, activity }) + .chain(explanation) + .collect() + } Signal::BootFinished => self.boot_finished(), } } + fn enters_failed_state(&mut self, unit: &str, activity: Activity) -> bool { + match activity.state { + Some(ActiveState::Failed) => self.units_in_failed_state.insert(unit.to_owned()), + Some(_) => { + self.units_in_failed_state.remove(unit); + false + } + None => false, + } + } + + async fn explain_failure( + &mut self, + connection: &mut BusConnection, + unit: String, + fallback_reason: &str, + ) -> Option { + if !self.unit_paths.values().any(|known| *known == unit) { + return None; + } + self.explained_failures.insert(unit.clone()); + let reason = connection + .failure_reason(&unit) + .await + .ok() + .flatten() + .unwrap_or_else(|| fallback_reason.to_owned()); + let log_lines = settled_unit_messages(&unit).await; + Some(BootChange::ExplainFailure { + unit, + cause: FailureCause { reason, log_lines }, + }) + } + fn boot_finished(&self) -> Vec { (self.phase == Phase::Host) .then_some(BootChange::BootFinished) @@ -440,6 +504,17 @@ impl BootTracker { } } +async fn settled_unit_messages(unit: &str) -> Vec { + for settling in JOURNAL_SETTLING_WAITS { + sleep(settling).await; + let log_lines = recent_unit_messages(unit.to_owned(), LOG_LINES_KEPT).await; + if !log_lines.is_empty() { + return log_lines; + } + } + Vec::new() +} + pub fn bus_inputs( phase: Phase, scope: Scope, @@ -454,3 +529,38 @@ pub fn bus_inputs( }) .flat_map(stream::iter) } + +#[cfg(test)] +mod tests { + use super::*; + + fn host_tracker() -> BootTracker { + BootTracker { + scope: Scope::System, + target: RootRequest::ManagerDefault, + phase: Phase::Host, + unit_paths: HashMap::new(), + explained_failures: HashSet::new(), + units_in_failed_state: HashSet::new(), + root_switch: RootSwitch::NotQueued, + } + } + + fn in_state(state: ActiveState) -> Activity { + Activity { + state: Some(state), + ..Activity::default() + } + } + + #[test] + fn only_entering_the_failed_state_asks_for_a_fresh_explanation() { + let mut tracker = host_tracker(); + let unit = "http-listener.service"; + assert!(tracker.enters_failed_state(unit, in_state(ActiveState::Failed))); + assert!(!tracker.enters_failed_state(unit, in_state(ActiveState::Failed))); + assert!(!tracker.enters_failed_state(unit, Activity::default())); + assert!(!tracker.enters_failed_state(unit, in_state(ActiveState::Activating))); + assert!(tracker.enters_failed_state(unit, in_state(ActiveState::Failed))); + } +} diff --git a/bootvine/src/systemd_link/journal_tail.rs b/bootvine/src/systemd_link/journal_tail.rs new file mode 100644 index 0000000..7b871a5 --- /dev/null +++ b/bootvine/src/systemd_link/journal_tail.rs @@ -0,0 +1,132 @@ +use std::{io, str::Chars}; + +use systemd::{ + id128::Id128, + journal::{Journal, OpenOptions}, +}; +use tokio::task::spawn_blocking; + +const UNIT_FIELDS: [&str; 4] = ["_SYSTEMD_UNIT", "UNIT", "_SYSTEMD_USER_UNIT", "USER_UNIT"]; +const MANAGER_SUMMARY_MESSAGE_IDS: [&str; 10] = [ + "7d4958e842da4a758f6c1cdc7b36dcc5", + "39f53479d3a045ac8e11786248231fbf", + "7ad2d189f7e94e70a38c781354912448", + "5eb03494b6584870a536b337290809b3", + "be02cf6855d2428ba40df7e9d022f03d", + "de5b426a63be47a7b6ac3eaac82e2f6f", + "9d1aaa27d60140bd96365438aad20286", + "ae8f7b866b0347b9af31fe1c80b127c0", + "d9b373ed55a64feb8242e02dbe79a49c", + "98e322203f7a4ed290d09fe03c09fe15", +]; +const ENTRIES_SCANNED: usize = 200; +const ESCAPE: char = '\u{1b}'; + +pub async fn recent_unit_messages(unit: String, wanted: usize) -> Vec { + spawn_blocking(move || { + read_recent_messages(&unit, wanted).unwrap_or_else(|error| { + eprintln!("bootvine: reading the journal of {unit}: {error}"); + Vec::new() + }) + }) + .await + .unwrap_or_default() +} + +fn read_recent_messages(unit: &str, wanted: usize) -> io::Result> { + let mut journal = OpenOptions::default().open()?; + match_unit_in_this_boot(&mut journal, unit)?; + journal.seek_tail()?; + let mut newest_first = Vec::new(); + let mut scanned = 0; + while newest_first.len() < wanted + && scanned < ENTRIES_SCANNED + && journal.previous().is_ok_and(|moved| moved > 0) + { + scanned += 1; + if is_manager_summary(&mut journal).unwrap_or(true) { + continue; + } + if let Ok(Some(message)) = entry_message(&mut journal) { + newest_first.push(message); + } + } + newest_first.reverse(); + Ok(newest_first) +} + +fn match_unit_in_this_boot(journal: &mut Journal, unit: &str) -> io::Result<()> { + UNIT_FIELDS.iter().try_for_each(|field| { + journal.match_add(field, unit)?.match_or()?; + Ok::<_, io::Error>(()) + })?; + journal + .match_and()? + .match_add("_BOOT_ID", Id128::from_boot()?.to_string())?; + Ok(()) +} + +fn field_text(journal: &mut Journal, field: &str) -> io::Result> { + Ok(journal.get_data(field)?.and_then(|entry_field| { + entry_field + .value() + .map(|value| String::from_utf8_lossy(value).into_owned()) + })) +} + +fn is_manager_summary(journal: &mut Journal) -> io::Result { + Ok(field_text(journal, "MESSAGE_ID")? + .is_some_and(|message_id| MANAGER_SUMMARY_MESSAGE_IDS.contains(&message_id.as_str()))) +} + +fn entry_message(journal: &mut Journal) -> io::Result> { + Ok(field_text(journal, "MESSAGE")? + .map(|message| printable_line(&message)) + .filter(|line| !line.is_empty())) +} + +fn printable_line(message: &str) -> String { + let mut characters = message.lines().next().unwrap_or_default().chars(); + let mut printable = String::new(); + while let Some(character) = characters.next() { + if character == ESCAPE { + skip_escape_sequence(&mut characters); + } else if !character.is_control() { + printable.push(character); + } + } + printable.trim_end().to_owned() +} + +fn skip_escape_sequence(characters: &mut Chars<'_>) { + if characters.clone().next() == Some('[') { + characters.next(); + characters.find(|character| ('@'..='~').contains(character)); + } +} + +#[cfg(test)] +mod tests { + use std::env; + + use super::*; + + #[test] + #[ignore = "reads this machine's journal: cargo test journal_of_this_boot -- --ignored \ + --nocapture"] + fn journal_of_this_boot_has_lines_for_journald() { + let unit = env::var("BOOTVINE_JOURNAL_UNIT") + .unwrap_or_else(|_| "systemd-journald.service".to_owned()); + let lines = read_recent_messages(&unit, 6).expect("readable journal"); + lines.iter().for_each(|line| println!("{line}")); + assert!(!lines.is_empty()); + } + + #[test] + fn printable_line_keeps_the_first_line_without_colours() { + assert_eq!( + printable_line("\u{1b}[31mno such file\u{1b}[0m\nsecond line"), + "no such file" + ); + } +} diff --git a/src/systemd_link/link_error.rs b/bootvine/src/systemd_link/link_error.rs similarity index 100% rename from src/systemd_link/link_error.rs rename to bootvine/src/systemd_link/link_error.rs diff --git a/src/systemd_link/monotonic_clock.rs b/bootvine/src/systemd_link/monotonic_clock.rs similarity index 100% rename from src/systemd_link/monotonic_clock.rs rename to bootvine/src/systemd_link/monotonic_clock.rs diff --git a/src/systemd_link/switch_root.rs b/bootvine/src/systemd_link/switch_root.rs similarity index 100% rename from src/systemd_link/switch_root.rs rename to bootvine/src/systemd_link/switch_root.rs diff --git a/src/systemd_link/systemd_bus.rs b/bootvine/src/systemd_link/systemd_bus.rs similarity index 83% rename from src/systemd_link/systemd_bus.rs rename to bootvine/src/systemd_link/systemd_bus.rs index d17f17b..1042a38 100644 --- a/src/systemd_link/systemd_bus.rs +++ b/bootvine/src/systemd_link/systemd_bus.rs @@ -75,6 +75,7 @@ pub enum Signal { }, JobRemoved { id: u32, + unit: String, result: JobResult, }, UnitChanged { @@ -342,6 +343,18 @@ impl BusConnection { }) } + pub async fn failure_reason(&mut self, name: &str) -> Result, Error> { + let Some(interface) = kind_interface(UnitKind::of(name)) else { + return Ok(None); + }; + let path: Path<'static> = self + .call(manager_call("GetUnit", (name,))?) + .await? + .read1()?; + let properties = self.all_properties(&path, interface).await?; + Ok(failure_reason_from(&properties)) + } + async fn all_properties(&mut self, path: &Path<'_>, interface: &str) -> Result { Ok(self .call(method_call( @@ -376,6 +389,52 @@ fn text_property(properties: &PropMap, property: &str) -> Option { .map(ToOwned::to_owned) } +fn signal_name(number: i64) -> String { + let name = match number { + 1 => "SIGHUP", + 2 => "SIGINT", + 3 => "SIGQUIT", + 4 => "SIGILL", + 5 => "SIGTRAP", + 6 => "SIGABRT", + 7 => "SIGBUS", + 8 => "SIGFPE", + 9 => "SIGKILL", + 11 => "SIGSEGV", + 13 => "SIGPIPE", + 14 => "SIGALRM", + 15 => "SIGTERM", + other => return format!("signal {other}"), + }; + name.to_owned() +} + +fn failure_reason_from(properties: &PropMap) -> Option { + let result = text_property(properties, "Result").filter(|result| result != "success")?; + let status = properties + .get("ExecMainStatus") + .and_then(|value| value.0.as_i64()); + let reason = match (result.as_str(), status) { + ("exit-code", Some(status)) if status != 0 => format!("exited with status {status}"), + ("exit-code", _) => "exited with an error".to_owned(), + ("signal", Some(signal)) => format!("killed by {}", signal_name(signal)), + ("core-dump", Some(signal)) => format!("dumped core on {}", signal_name(signal)), + ("timeout", _) => "timed out".to_owned(), + ("watchdog", _) => "stopped answering the watchdog".to_owned(), + ("start-limit-hit", _) => "restarted too often".to_owned(), + ("oom-kill", _) => "killed for running out of memory".to_owned(), + ("resources", _) => "could not set up its resources".to_owned(), + ("protocol", _) => "broke the start-up protocol".to_owned(), + (other, _) => other.to_owned(), + }; + Some( + match text_property(properties, "StatusText").filter(|text| !text.is_empty()) { + Some(status_text) => format!("{reason}: {status_text}"), + None => reason, + }, + ) +} + fn unit_id(name: &str, properties: &PropMap) -> String { text_property(properties, "Id").unwrap_or_else(|| name.to_owned()) } @@ -474,10 +533,11 @@ fn parse_job_signal(message: &Message, member: &str) -> Option { Some(Signal::JobNew { id, job, unit }) } "JobRemoved" => { - let (id, _job, _unit, result): (u32, Path<'static>, String, String) = + let (id, _job, unit, result): (u32, Path<'static>, String, String) = message.read4().ok()?; Some(Signal::JobRemoved { id, + unit, result: serde_plain::from_str(&result).unwrap_or(JobResult::Failed), }) } @@ -567,6 +627,46 @@ mod tests { ); } + fn reason_properties(result: &str, status: i32, status_text: &str) -> PropMap { + [ + ("Result", Box::new(result.to_owned()) as Box), + ("ExecMainStatus", Box::new(status)), + ("StatusText", Box::new(status_text.to_owned())), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), Variant(value))) + .collect() + } + + #[test] + fn exit_codes_and_signals_become_a_reason() { + assert_eq!( + failure_reason_from(&reason_properties("exit-code", 1, "")).as_deref(), + Some("exited with status 1") + ); + assert_eq!( + failure_reason_from(&reason_properties("signal", 11, "")).as_deref(), + Some("killed by SIGSEGV") + ); + } + + #[test] + fn status_text_follows_the_reason() { + assert_eq!( + failure_reason_from(&reason_properties("timeout", 0, "waiting for the disk")) + .as_deref(), + Some("timed out: waiting for the disk") + ); + } + + #[test] + fn success_has_no_reason() { + assert_eq!( + failure_reason_from(&reason_properties("success", 0, "")), + None + ); + } + #[test] fn mounts_use_one_timeout_for_both_directions() { let mount = limit_properties(&[("TimeoutUSec", Duration::from_secs(20))]); diff --git a/system-files/dracut/.shellcheckrc b/bootvine/system-files/dracut/.shellcheckrc similarity index 100% rename from system-files/dracut/.shellcheckrc rename to bootvine/system-files/dracut/.shellcheckrc diff --git a/system-files/dracut/90bootvine/module-setup.sh b/bootvine/system-files/dracut/90bootvine/module-setup.sh similarity index 100% rename from system-files/dracut/90bootvine/module-setup.sh rename to bootvine/system-files/dracut/90bootvine/module-setup.sh diff --git a/system-files/mkinitcpio/install/sd-bootvine b/bootvine/system-files/mkinitcpio/install/sd-bootvine similarity index 100% rename from system-files/mkinitcpio/install/sd-bootvine rename to bootvine/system-files/mkinitcpio/install/sd-bootvine diff --git a/system-files/systemd/bootvine-shutdown.service b/bootvine/system-files/systemd/bootvine-shutdown.service similarity index 100% rename from system-files/systemd/bootvine-shutdown.service rename to bootvine/system-files/systemd/bootvine-shutdown.service diff --git a/system-files/systemd/bootvine.service b/bootvine/system-files/systemd/bootvine.service similarity index 100% rename from system-files/systemd/bootvine.service rename to bootvine/system-files/systemd/bootvine.service diff --git a/docs/architecture.md b/docs/architecture.md index 86acc21..c25d18e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,6 +11,8 @@ while it does. None of this is needed to install or configure it. PID 1 over systemd's private socket, which [zbus](https://github.com/dbus2/zbus) cannot use; see [systemd's private socket](#systemds-private-socket). +- **libsystemd.** It reads the last journal lines of a failed unit; see + [failures](#failures). - **A virtual terminal.** It draws on `tty12` and switches the console to it. ## Overview @@ -116,7 +118,7 @@ From then on it follows the signals: | Signal | Carries | Used for | | --- | --- | --- | | `JobNew` | job id, job path, unit name | a row in the tree | -| `JobRemoved` | job id, result | the row finishes as done, failed or skipped | +| `JobRemoved` | job id, unit name, result | the row finishes as done, failed or skipped | | `PropertiesChanged` on a unit | `ActiveState` and its timestamps | running since, and the timeline | | `StartupFinished` | | the boot reached its default target | @@ -151,6 +153,47 @@ systemd's `…TimestampMonotonic` properties count microseconds of clock, so the times it shows are measured from the kernel start, even for units that ran before bootvine did. +### Failures + +`JobRemoved` says how a job ended: `failed`, `timeout` or `dependency`. Like +systemd's own `[FAILED]`, `[ TIME ]` and `[DEPEND]`, bootvine tells them +apart: a unit that failed only because a dependency did is shown dimmed in the +tree and left out of the failure panel, so the panel points at the unit that +actually broke. A unit whose `ActiveState` turns `failed` without a failed job, +such as a service crashing after it started, counts as failed too. + +For a unit that broke or timed out, bootvine collects the cause once: + +```mermaid +sequenceDiagram + participant PID1 as systemd (PID 1) + participant Bootvine as bootvine + participant Journal as journal files + + PID1-->>Bootvine: ActiveState=failed, or JobRemoved failed/timeout + Bootvine->>PID1: GetUnit, Properties.GetAll on the unit type + PID1->>Bootvine: Result, ExecMainCode, ExecMainStatus, StatusText + Note over Bootvine: wait 200 ms for journald, longer while nothing is there yet + Bootvine->>Journal: sd_journal, matching the unit in this boot + Journal->>Bootvine: last messages, newest last + Note over Bootvine: reason and lines in the failure panel +``` + +- **The reason** comes from the unit type's `Result` (`exit-code`, `signal`, + `timeout`, `start-limit-hit`, `oom-kill`, …), with the exit status or signal + of the main process, and the service's `StatusText` when it set one. These + are read when the unit fails, since they change at that moment. +- **The journal lines** match the unit the same way `journalctl -u` does + (`_SYSTEMD_UNIT=` or `UNIT=`, in the current boot), without PID 1's own + "Starting…", "Failed with result…" and "Failed to start…" lines, which the + reason already covers. In the initrd they come from `/run/log/journal`. The + journal is opened without `SD_JOURNAL_LOCAL_ONLY`: libsystemd caches the + initrd's machine ID, which would hide the real root's files after the + switch, and the boot ID match already keeps other machines out. +- **libsystemd** is linked rather than calling `journalctl`, since dracut, + mkinitcpio and the NixOS initrd builder copy the libraries a binary links + into the initramfs along with it, while `journalctl` may not be there. + ## Surviving the switch to the real root At the end of the initrd, systemd mounts the real root at `/sysroot`, isolates diff --git a/docs/configuration.md b/docs/configuration.md index 7b573f6..590ecab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,6 +81,14 @@ queued-bar = "gray" running = "yellow" running-label = "white" +[colors.failures] +rule = "dark-gray" +title = "light-red" +unit = "light-red" +reason = "white" +log = "gray" +overflow = "dark-gray" + [colors.footer] rule = "dark-gray" done = "green" @@ -114,6 +122,7 @@ Each `[colors.*]` table covers one part of the screen: | `stages` | the finished stages, such as the initrd | | `tree` | the unit tree | | `timeline` | the timeline of the slowest units | +| `failures` | the failed units with their reason and last journal lines | | `footer` | the tallies of done, running, waiting and failed units | | `password` | the passphrase popup | diff --git a/docs/development.md b/docs/development.md index 348f1c3..a2e3100 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,5 +1,19 @@ # Development +## Layout + +| Folder | Holds | +| --- | --- | +| `bootvine/` | the published crate: the library, the `bootvine` binary and the unit and initrd files it installs | +| `scripted-demos/` | the scripted boot, shutdown and passphrase demos, and the VHS tapes that record them | +| `xtask/` | repository tasks run with `cargo xtask` | +| `vm-tests/` | the sample NixOS machines, one folder each with its VM check | +| `nix/` | the crane builds and the NixOS module | +| `recordings/` | the animations and screenshots the README and the [guide](guide.md) show | + +`cargo build` and `cargo test` act on every crate. `cargo run` needs the binary: +`--bin bootvine`, `--bin scripted-boot` and so on. + ## Setup Work inside the dev shell (`direnv allow` once, with @@ -57,24 +71,32 @@ locally from the dev shell with `release-plz update`. | What | Command | | --- | --- | | Model and render tests | `cargo test --workspace` | -| Scripted demos | `cargo run --example boot` (also `shutdown`, `luks`) | -| Sample machine in a window | `nix run .#sample-machine-vm` | +| Scripted demos | `cargo run --bin scripted-boot` (also `scripted-shutdown`, `scripted-luks-unlock`) | +| Sample machines in a window | `nix run .#sample-vm-`, see the [guide](guide.md#sample-machines) | | Boot and shutdown in a VM | `nix build .#checks.x86_64-linux.vm-boot-and-shutdown` | | Passphrase prompt in a VM | `nix build .#checks.x86_64-linux.vm-luks-passphrase` | +| Failure panel in a VM | `nix build .#checks.x86_64-linux.vm-failures` | To watch your own user manager start the sample units: ```sh units=$(nix build --no-link --print-out-paths .#sample-unit-files) systemctl --user link --runtime $units/* -systemctl --user start --no-block bootvine-demo.target -cargo run -- --user --target bootvine-demo.target --hold 3s +systemctl --user start --no-block bootvine-sample-services.target +cargo run --bin bootvine -- --user --target bootvine-sample-services.target --hold 3s systemctl --user stop $(ls $units) systemctl --user disable --runtime $(ls $units) ``` +Everything a sample machine needs sits in its own folder under `vm-tests/`: +`machine.nix`, and for the machines with a check, `check.nix` and `check.py`. +`vm-tests/shared/` holds the settings every machine shares, the window setup +for `nix run` and the console helpers for the checks. A new machine is a new +folder plus its name in `vm-tests/default.nix`. + The VM checks read the console as text, hold the boot at fixed points and wait -on marker files instead of sleeping. To step through one by hand, run +on marker files instead of sleeping. The failures check starts each failing +unit by hand, so it knows which one is newest in the panel. To step through one by hand, run `nix run .#checks.x86_64-linux.vm-boot-and-shutdown.driverInteractive` and call `test_script()`. @@ -83,12 +105,13 @@ exports libraries that clash with [QEMU](https://www.qemu.org) or Chromium. ## Repository tasks -- `nix run .#record-screencasts` re-records the README animations from the - scripted demos and copies the passphrase screenshot from the VM check. +- `nix run .#record-screencasts` re-records the animations from the scripted + demos and copies the passphrase and failures screenshots from the VM checks + into `recordings/`, for the README and the [guide](guide.md). - `cargo xtask sync-default-config` rewrites the defaults block in [configuration](configuration.md#defaults) from `bootvine defaults`. -- `cargo run -- install --destdir --initrd dracut` stages an install - without touching the system; `cargo run -- uninstall --destdir ` removes +- `cargo run --bin bootvine -- install --destdir --initrd dracut` stages an install + without touching the system; `cargo run --bin bootvine -- uninstall --destdir ` removes it again. ## Code style and design diff --git a/docs/guide.md b/docs/guide.md new file mode 100644 index 0000000..505d67d --- /dev/null +++ b/docs/guide.md @@ -0,0 +1,157 @@ +# Guide + +What bootvine shows on the console, part by part, with the recordings that go +with each. The animations come from the [scripted demos](#scripted-demos); the +screenshots come from the [VM checks](development.md#testing) on real NixOS +machines. + +## Reading the screen + +A boot through the initrd into stage 2, with one failed unit + +From top to bottom: + +1. **The header**: the host name, the system and its generation on NixOS + (otherwise the `PRETTY_NAME` from `os-release`), and the kernel. Once a + stage has finished, the total time since the kernel started sits on the + right. +2. **The stages**: each finished stage, such as `Initrd`, with how long it + took, how many units finished in it and how many of them failed. The stage + still running is the last line, ending in `Running`. +3. **The tree** of units with a pending job. Each unit is nested under the unit + waiting on it, and the milestone targets (`sysinit.target`, `basic.target`, + `multi-user.target`, `graphical.target`) open the levels: + - `◆` is the stage being worked on, `◇` one still to come, `√` one reached + (`*`, `o` and `+` on console fonts without those glyphs); + - `[|]` with a spinner is running, `[ ]` is waiting, `[√]` finished in the + last couple of seconds (`keep-finished-units-for`); + - `[×]` in bold red broke or timed out; dimmed red failed only because a + unit it needed did; + - the `Running` column counts up while a unit runs, and after 5 seconds the + `Timeout` column shows the limit systemd will stop it at, red once it + is passed; + - a unit waiting on exactly one other, which waits on exactly one other, + folds into one row joined by `─`; mounts and devices show their + description instead of their unit name. + + When the tree is taller than the screen, finished and waiting rows give way + first, and a last line counts the units left out. +4. **The failure panel**, only once a unit broke or timed out; see + [failures](#failures). +5. **The timeline**, in the room left below the tree: the milestones reached, + with how many units they took, the slowest finished units (up to five, and + none shorter than `hide-runs-shorter-than`) and the units still running, + as bars against seconds since the kernel started. A `│` marks the switch + from the initrd to the real root. +6. **The footer**: the units done, running, waiting and failed so far. + +## Booting through the initrd + +The same recording shows a boot from the start. With a systemd initrd, +bootvine starts about a second after the kernel: + +- **The initrd stage**: udev settling, the root device and its file systems + under `initrd.target`. +- **The switch to the real root**: the initrd moves into the stages at the + top, with its time and unit count, and the tree starts again under the + default target. The timeline keeps the initrd's runs, left of the `│`. +- **Stage 2** up to `graphical.target`. +- **The handover**: bootvine hands the console back once the default target is + reached, or earlier, once the display manager has switched to its own + terminal. + +How bootvine gets in that early and survives the switch is in +[architecture](architecture.md#starting-in-the-initrd). + +## Shutdown + +A poweroff: units stopping under the targets waiting on them + +On a poweroff, reboot or halt the tree turns around: + +- its root is the target being reached, such as `poweroff.target`; +- each unit being stopped sits under the units that have to stop before it, + so the unit holding up the shutdown is at the bottom of its branch; +- `shutdown.target`, `umount.target` and `final.target` open the levels; +- the screen stays until systemd hands over to `systemd-shutdown`. + +## Passphrase prompts + +A LUKS passphrase prompt floating over the initrd tree + +When a unit asks for a passphrase, such as `systemd-cryptsetup@….service` +unlocking a [LUKS](https://gitlab.com/cryptsetup/cryptsetup) volume, the +question floats over a dimmed tree: + +- typing shows one mark per character; +- `Enter` sends it, and the popup says it is checking; +- a wrong passphrase brings the prompt back with "Wrong passphrase, try + again"; +- once unlocked, the popup closes and the boot carries on. + +If another password agent, such as Plymouth's, was already asking when +bootvine started, bootvine waits behind it. + +## Failures + +The failure panel listing failed units with their reason and last journal lines + +A unit that broke or timed out gets an entry in the failure panel, newest +first: + +- **The headline**: the unit and why it failed, such as `exited with status 2`, + `killed by SIGSEGV`, `timed out`, `restarted too often` or + `exited with status 203` (systemd could not run the command). A service that + reported a status with `sd_notify` adds it, as in + `exited with status 1: Cannot reach database`. +- **The journal lines**: up to six of the unit's last messages from this boot, + the same ones `journalctl -u` shows, without systemd's own "Starting…" and + "Failed with result…" lines. +- **Out of room**: the newest failure keeps its lines, older ones shrink to + their headline, and a last line counts the rest. +- **What stays out**: a unit that failed only because one it needed did stays + dimmed in the tree, so the panel points at the unit that actually broke. + The footer counts both. + +Failures from the initrd stay in the panel after the switch to the real root. +The screenshot comes from the `failures` sample machine, with every kind of +failure systemd reports. + +## Trying it + +Neither of these touches your system's boot. + +### Scripted demos + +Play a scripted boot, poweroff and passphrase prompt in your terminal, with +[Rust](https://www.rust-lang.org/tools/install) installed: + +```sh +git clone https://codeberg.org/wvhulle/bootvine.git +cd bootvine +cargo run --bin scripted-boot +cargo run --bin scripted-shutdown +cargo run --bin scripted-luks-unlock +``` + +`q` quits. The encrypted root unlocks with the passphrase `demo`; `Esc` quits +while it asks. These are the scenarios the animations above were recorded +from. + +### Sample machines + +Each sample machine boots in a [QEMU](https://www.qemu.org) window with +[Nix](https://nixos.org/download/), and logs in as root at the end. `poweroff` +shows the shutdown. + +| Run | Shows | +| --- | --- | +| `nix run .#sample-vm-slow-services` | slow units in the initrd and stage 2, one failing unit, and the handover to a greetd login | +| `nix run .#sample-vm-text-console` | the same units without a display manager, handing over at the default target | +| `nix run .#sample-vm-scripted-initrd` | the same units with NixOS's scripted initrd, so bootvine starts in stage 2 only | +| `nix run .#sample-vm-encrypted-volume` | a LUKS volume unlocked in the initrd; the passphrase is `sample passphrase` | +| `nix run .#sample-vm-failures` | a firmware failure in the initrd, then units that exit with an error, crash, time out, cannot be run, restart too often, report a status, die after starting, and one that fails only through its dependency | + +Each machine keeps its disk in `~/.cache/bootvine-.qcow2`; delete it +to start fresh. The machines live in `vm-tests/`, one folder each, next to the +checks that drive them. diff --git a/docs/install.md b/docs/install.md index 7acec09..8924086 100644 --- a/docs/install.md +++ b/docs/install.md @@ -48,7 +48,7 @@ as well, the initrd has to run systemd. 1. Install the build dependencies and Rust: ```sh - sudo apt install build-essential pkg-config libdbus-1-dev git curl + sudo apt install build-essential pkg-config libdbus-1-dev libsystemd-dev git curl curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` @@ -88,7 +88,7 @@ as well, the initrd has to run systemd. ## Other distributions Install as in the Ubuntu steps, with your distribution's packages for a C -toolchain, `pkg-config` and the libdbus headers. +toolchain, `pkg-config` and the libdbus and libsystemd headers. For the initrd, use the dracut module as above, or on [mkinitcpio](https://wiki.archlinux.org/title/Mkinitcpio): diff --git a/flake.nix b/flake.nix index 777efef..6d4c639 100644 --- a/flake.nix +++ b/flake.nix @@ -46,11 +46,11 @@ craneLib rustToolchain ; - builds = import ./crane-builds.nix { inherit lib pkgs craneLib; }; + builds = import ./nix/crane-builds.nix { inherit lib pkgs craneLib; }; } ); - sampleMachine = import ./tests/sample-machine { + vmTests = import ./vm-tests { inherit lib; bootvineModule = self.nixosModules.bootvine; }; @@ -60,19 +60,22 @@ { pkgs, builds, ... }: { default = builds.bootvine; - inherit (builds) bootvine scripted-boot-examples repo-tasks; - sample-unit-files = pkgs.callPackage ./tests/sample-machine/sample-unit-files.nix { - inherit (sampleMachine) sampleUnits; + inherit (builds) bootvine scripted-demos repo-tasks; + sample-unit-files = pkgs.callPackage ./vm-tests/slow-services/sample-unit-files.nix { + inherit (vmTests) sampleUnits; }; } ); nixosModules = { default = self.nixosModules.bootvine; - bootvine = import ./nixos-module.nix { inherit self; }; + bootvine = import ./nix/nixos-module.nix { inherit self; }; }; - nixosConfigurations.sample-machine = sampleMachine.nixosSystemFor "x86_64-linux"; + nixosConfigurations = lib.mapAttrs' ( + machineName: _: + lib.nameValuePair "sample-${machineName}" (vmTests.nixosSystemFor "x86_64-linux" machineName) + ) vmTests.machines; apps = forEachSystem ( { @@ -83,6 +86,11 @@ ... }: let + runSampleMachine = machineName: description: { + type = "app"; + meta.description = "Boot the ${machineName} sample machine in QEMU: ${description}"; + program = lib.getExe (vmTests.nixosSystemFor system machineName).config.system.build.vm; + }; runReleasePlzOnCodeberg = subcommand: description: { type = "app"; meta.description = description; @@ -107,25 +115,25 @@ { release-plz-release = runReleasePlzOnCodeberg "release" "Publish unpublished versions to crates.io and tag them with a Codeberg release"; release-plz-release-pr = runReleasePlzOnCodeberg "release-pr" "Open or update the Codeberg pull request that bumps the version and changelog"; - sample-machine-vm = { - type = "app"; - meta.description = "Boot a sample machine with bootvine and slow sample units in QEMU"; - program = lib.getExe (sampleMachine.nixosSystemFor system).config.system.build.vm; - }; + sample-vm-slow-services = runSampleMachine "slow-services" "slow and failing units, a held initrd probe and a greetd login"; + sample-vm-text-console = runSampleMachine "text-console" "the slow units without a display manager, handing over at the default target"; + sample-vm-scripted-initrd = runSampleMachine "scripted-initrd" "the slow units with the scripted initrd, so bootvine starts in stage 2"; + sample-vm-encrypted-volume = runSampleMachine "encrypted-volume" "a LUKS volume unlocked in the initrd with the passphrase 'sample passphrase'"; + sample-vm-failures = runSampleMachine "failures" "units that fail in every way systemd reports, each with journal messages"; record-screencasts = { type = "app"; - meta.description = "Re-record the README screencasts and passphrase screenshot into recordings/"; + meta.description = "Re-record the screencasts and the passphrase and failures screenshots into recordings/"; program = lib.getExe ( pkgs.writeShellApplication { name = "record-screencasts"; runtimeInputs = [ pkgs.vhs - builds.scripted-boot-examples + builds.scripted-demos builds.repo-tasks ]; text = "exec xtask record-screencasts --passphrase-png ${ self.checks.${system}.vm-luks-passphrase - }/passphrase.png"; + }/passphrase.png --failures-png ${self.checks.${system}.vm-failures}/failures.png"; } ); }; @@ -163,13 +171,8 @@ inherit (builds) rustfmt-formatting clippy-lints; bootvine-build = builds.bootvine; repo-tasks-build = builds.repo-tasks; - vm-boot-and-shutdown = pkgs.testers.runNixOSTest ( - import ./tests/vm-checks/boot.nix { sampleMachine = sampleMachine.machine; } - ); - vm-luks-passphrase = pkgs.testers.runNixOSTest ( - import ./tests/vm-checks/luks.nix { bootvineModule = self.nixosModules.bootvine; } - ); } + // vmTests.checksFor pkgs ); devShells = forEachSystem ( diff --git a/crane-builds.nix b/nix/crane-builds.nix similarity index 56% rename from crane-builds.nix rename to nix/crane-builds.nix index 71a0aba..e5e0923 100644 --- a/crane-builds.nix +++ b/nix/crane-builds.nix @@ -4,18 +4,24 @@ craneLib, }: let + bootvineCrate = craneLib.crateNameFromCargoToml { cargoToml = ../bootvine/Cargo.toml; }; + commonArgs = { + inherit (bootvineCrate) pname version; src = lib.fileset.toSource { - root = ./.; + root = ../.; fileset = lib.fileset.unions [ - (craneLib.fileset.commonCargoSources ./.) - ./system-files - ./.rustfmt.toml + (craneLib.fileset.commonCargoSources ../.) + ../bootvine/system-files + ../.rustfmt.toml ]; }; strictDeps = true; nativeBuildInputs = [ pkgs.pkg-config ]; - buildInputs = [ pkgs.dbus ]; + buildInputs = [ + pkgs.dbus + pkgs.systemd + ]; }; cargoArtifacts = craneLib.buildDepsOnly (commonArgs // { pname = "bootvine-workspace"; }); @@ -25,7 +31,7 @@ let in { rustfmt-formatting = craneLib.cargoFmt { - inherit (commonArgs) src; + inherit (commonArgs) src pname version; cargoExtraArgs = "--all"; }; @@ -41,35 +47,33 @@ in cargoExtraArgs = "-p bootvine"; postInstall = '' for unit in bootvine.service bootvine-shutdown.service; do - install -Dm644 system-files/systemd/$unit $out/lib/systemd/system/$unit + install -Dm644 bootvine/system-files/systemd/$unit $out/lib/systemd/system/$unit substituteInPlace $out/lib/systemd/system/$unit \ --replace-fail /usr/bin/bootvine $out/bin/bootvine done - install -Dm755 system-files/dracut/90bootvine/module-setup.sh \ + install -Dm755 bootvine/system-files/dracut/90bootvine/module-setup.sh \ $out/lib/dracut/modules.d/90bootvine/module-setup.sh - install -Dm644 system-files/mkinitcpio/install/sd-bootvine \ + install -Dm644 bootvine/system-files/mkinitcpio/install/sd-bootvine \ $out/lib/initcpio/install/sd-bootvine ''; meta = { mainProgram = "bootvine"; platforms = lib.platforms.linux; - license = lib.getLicenseFromSpdxId (lib.importTOML ./Cargo.toml).package.license; + license = lib.getLicenseFromSpdxId (lib.importTOML ../bootvine/Cargo.toml).package.license; }; }; - scripted-boot-examples = buildWorkspaceMember { - pname = "bootvine-scripted-boot-examples"; - cargoExtraArgs = "-p bootvine --examples"; + scripted-demos = buildWorkspaceMember { + inherit (craneLib.crateNameFromCargoToml { cargoToml = ../scripted-demos/Cargo.toml; }) + pname + version + ; + cargoExtraArgs = "-p scripted-demos"; doCheck = false; - installPhaseCommand = '' - for example in boot shutdown; do - install -Dm755 target/release/examples/$example $out/bin/bootvine-$example-example - done - ''; }; repo-tasks = buildWorkspaceMember { - pname = "xtask"; + inherit (craneLib.crateNameFromCargoToml { cargoToml = ../xtask/Cargo.toml; }) pname version; cargoExtraArgs = "-p xtask"; doCheck = false; meta.mainProgram = "xtask"; diff --git a/nixos-module.nix b/nix/nixos-module.nix similarity index 100% rename from nixos-module.nix rename to nix/nixos-module.nix diff --git a/recordings/failures.png b/recordings/failures.png new file mode 100644 index 0000000..99d8607 Binary files /dev/null and b/recordings/failures.png differ diff --git a/recordings/passphrase.png b/recordings/passphrase.png index 4824e57..4bfaa2b 100644 Binary files a/recordings/passphrase.png and b/recordings/passphrase.png differ diff --git a/scripted-demos/Cargo.toml b/scripted-demos/Cargo.toml new file mode 100644 index 0000000..675c488 --- /dev/null +++ b/scripted-demos/Cargo.toml @@ -0,0 +1,17 @@ +[package] +description = "Scripted boots, shutdowns and passphrase prompts that play bootvine's screen in a terminal" +edition = "2024" +license = "GPL-3.0-or-later" +name = "scripted-demos" +publish = false +version = "0.1.0" + +[lints] +workspace = true + +[dependencies] +bootvine = { path = "../bootvine" } +futures = { version = "0.3.34", default-features = false, features = ["std"] } +ratatui = "0.30.2" +tokio = { version = "1.54.0", default-features = false, features = ["rt", "time"] } +zeroize = "1.9.1" diff --git a/examples/boot.rs b/scripted-demos/src/bin/scripted-boot.rs similarity index 92% rename from examples/boot.rs rename to scripted-demos/src/bin/scripted-boot.rs index 5a812ef..619ff36 100644 --- a/examples/boot.rs +++ b/scripted-demos/src/bin/scripted-boot.rs @@ -1,9 +1,7 @@ use std::{io, time::Duration}; -use bootvine::{ - boot_model::Phase, - demo_script::scripted_boot::{Scenario, Step, failing, play, step}, -}; +use bootvine::boot_model::Phase; +use scripted_demos::scripted_boot::{Scenario, ScriptedFailure, Step, failing, play, step}; const INITRD: &[Step] = &[ step( @@ -127,6 +125,14 @@ const HOST: &[Step] = &[ "nvidia-powerd.service", &["basic.target"], Duration::from_millis(700), + ScriptedFailure { + reason: "exited with status 1", + log_lines: &[ + "nvidia-powerd version:1.0(build 1)", + "ERROR! Failed to get system power limits", + "ERROR! Client (presumably SBIOS) has not registered for D-Notify events", + ], + }, ), step( "sys-devices-virtual-misc-rfkill.device", diff --git a/examples/luks.rs b/scripted-demos/src/bin/scripted-luks-unlock.rs similarity index 98% rename from examples/luks.rs rename to scripted-demos/src/bin/scripted-luks-unlock.rs index 5eb859a..77bf15d 100644 --- a/examples/luks.rs +++ b/scripted-demos/src/bin/scripted-luks-unlock.rs @@ -1,9 +1,7 @@ use std::{io, time::Duration}; -use bootvine::{ - boot_model::Phase, - demo_script::scripted_boot::{Scenario, Step, play, step, unlocking}, -}; +use bootvine::boot_model::Phase; +use scripted_demos::scripted_boot::{Scenario, Step, play, step, unlocking}; const ROOT_PARTITION: &str = "dev-disk-by\\x2duuid-7c2e9f4a\\x2d1b3d\\x2d4e8f\\x2d9a6c\\x2d2d5b8e1f0c3a.device"; diff --git a/examples/shutdown.rs b/scripted-demos/src/bin/scripted-shutdown.rs similarity index 96% rename from examples/shutdown.rs rename to scripted-demos/src/bin/scripted-shutdown.rs index 0d0faf9..10465c9 100644 --- a/examples/shutdown.rs +++ b/scripted-demos/src/bin/scripted-shutdown.rs @@ -1,9 +1,7 @@ use std::{io, time::Duration}; -use bootvine::{ - boot_model::Phase, - demo_script::scripted_boot::{Scenario, Step, play, step, stopping}, -}; +use bootvine::boot_model::Phase; +use scripted_demos::scripted_boot::{Scenario, Step, play, step, stopping}; const SHUTDOWN: &[Step] = &[ step( diff --git a/scripted-demos/src/lib.rs b/scripted-demos/src/lib.rs new file mode 100644 index 0000000..02ce22d --- /dev/null +++ b/scripted-demos/src/lib.rs @@ -0,0 +1 @@ +pub mod scripted_boot; diff --git a/src/demo_script/scripted_boot.rs b/scripted-demos/src/scripted_boot.rs similarity index 91% rename from src/demo_script/scripted_boot.rs rename to scripted-demos/src/scripted_boot.rs index c0a97a2..9e161dc 100644 --- a/src/demo_script/scripted_boot.rs +++ b/scripted-demos/src/scripted_boot.rs @@ -7,15 +7,10 @@ use std::{ time::Duration, }; -use futures::stream; -use ratatui::{DefaultTerminal, crossterm::event::Event}; -use tokio::runtime::Builder; -use zeroize::Zeroizing; - -use crate::{ +use bootvine::{ boot_model::{ - ActiveState, Activity, BootChange, BootState, JobKind, JobResult, Phase, UnitFacts, - UnitTimeouts, boot_origin::BootOrigin, + ActiveState, Activity, BootChange, BootState, FailureCause, JobKind, JobResult, Phase, + UnitFacts, UnitTimeouts, boot_origin::BootOrigin, }, boot_view::PasswordPrompt, console_session::{ @@ -26,17 +21,33 @@ use crate::{ screen_settings::{color_scheme::Colors, config_file::Timing}, systemd_link::monotonic_clock::monotonic_now, }; +use futures::stream; +use ratatui::{DefaultTerminal, crossterm::event::Event}; +use tokio::runtime::Builder; +use zeroize::Zeroizing; const FINAL_FRAME: Duration = Duration::from_secs(3); pub const DEMO_PASSPHRASE: &str = "demo"; const DEMO_PASSPHRASE_HINT: &str = "This demo unlocks with the passphrase \"demo\""; +#[derive(Clone, Copy)] +pub struct ScriptedFailure { + pub reason: &'static str, + pub log_lines: &'static [&'static str], +} + +#[derive(Clone, Copy)] +enum StepEnding { + Succeeds, + Fails(ScriptedFailure), +} + pub struct Step { unit: &'static str, after: &'static [&'static str], work: Duration, - fails: bool, + ending: StepEnding, stops: bool, asks: Option<&'static str>, timeout: Option, @@ -58,7 +69,7 @@ pub const fn step(unit: &'static str, after: &'static [&'static str], work: Dura unit, after, work, - fails: false, + ending: StepEnding::Succeeds, stops: false, asks: None, timeout: None, @@ -66,9 +77,14 @@ pub const fn step(unit: &'static str, after: &'static [&'static str], work: Dura } #[must_use] -pub const fn failing(unit: &'static str, after: &'static [&'static str], work: Duration) -> Step { +pub const fn failing( + unit: &'static str, + after: &'static [&'static str], + work: Duration, + failure: ScriptedFailure, +) -> Step { Step { - fails: true, + ending: StepEnding::Fails(failure), ..step(unit, after, work) } } @@ -132,13 +148,26 @@ impl ScriptedJob { } fn outcome(&self) -> JobResult { - if self.step.fails { + if matches!(self.step.ending, StepEnding::Fails(_)) { JobResult::Failed } else { JobResult::Done } } + fn explanation(&self) -> Option { + let StepEnding::Fails(failure) = self.step.ending else { + return None; + }; + Some(BootChange::ExplainFailure { + unit: self.step.unit.to_owned(), + cause: FailureCause { + reason: failure.reason.to_owned(), + log_lines: failure.log_lines.iter().map(ToString::to_string).collect(), + }, + }) + } + fn initial_activity(&self) -> Activity { Activity { state: self.step.stops.then_some(ActiveState::Active), @@ -163,7 +192,7 @@ impl ScriptedJob { } fn settled_activity(&self, now: Duration) -> Activity { - let state = if self.step.fails { + let state = if matches!(self.step.ending, StepEnding::Fails(_)) { ActiveState::Failed } else if self.step.stops { ActiveState::Inactive @@ -329,6 +358,8 @@ impl Script { at: now, }, ] + .into_iter() + .chain(job.explanation()) }), ) .collect(); diff --git a/recordings/boot.tape b/scripted-demos/tapes/scripted-boot.tape similarity index 77% rename from recordings/boot.tape rename to scripted-demos/tapes/scripted-boot.tape index a50b3ff..71e1736 100644 --- a/recordings/boot.tape +++ b/scripted-demos/tapes/scripted-boot.tape @@ -1,6 +1,6 @@ Output recordings/boot.gif -Require bootvine-boot-example +Require scripted-boot Set Shell bash Set FontSize 22 @@ -12,7 +12,7 @@ Set TypingSpeed 0 Set Framerate 10 Hide -Type "clear && bootvine-boot-example" +Type "clear && scripted-boot" Enter Show diff --git a/recordings/shutdown.tape b/scripted-demos/tapes/scripted-shutdown.tape similarity index 76% rename from recordings/shutdown.tape rename to scripted-demos/tapes/scripted-shutdown.tape index 3b92cf3..2c09f67 100644 --- a/recordings/shutdown.tape +++ b/scripted-demos/tapes/scripted-shutdown.tape @@ -1,6 +1,6 @@ Output recordings/shutdown.gif -Require bootvine-shutdown-example +Require scripted-shutdown Set Shell bash Set FontSize 22 @@ -12,7 +12,7 @@ Set TypingSpeed 0 Set Framerate 10 Hide -Type "clear && bootvine-shutdown-example" +Type "clear && scripted-shutdown" Enter Show diff --git a/src/demo_script.rs b/src/demo_script.rs deleted file mode 100644 index c185991..0000000 --- a/src/demo_script.rs +++ /dev/null @@ -1,3 +0,0 @@ -#[cfg(test)] -pub(crate) mod boot_test_fixtures; -pub mod scripted_boot; diff --git a/tests/sample-machine/default.nix b/tests/sample-machine/default.nix deleted file mode 100644 index 67c8dbb..0000000 --- a/tests/sample-machine/default.nix +++ /dev/null @@ -1,25 +0,0 @@ -{ lib, bootvineModule }: -let - sampleUnits = import ./sample-units.nix; - - machine = { - imports = [ - bootvineModule - ./machine.nix - ]; - _module.args = { inherit sampleUnits; }; - }; -in -{ - inherit sampleUnits machine; - - nixosSystemFor = - system: - lib.nixosSystem { - inherit system; - modules = [ - machine - ./interactive-vm.nix - ]; - }; -} diff --git a/tests/sample-machine/machine.nix b/tests/sample-machine/machine.nix deleted file mode 100644 index 5f6eed2..0000000 --- a/tests/sample-machine/machine.nix +++ /dev/null @@ -1,91 +0,0 @@ -{ - sampleUnits, - lib, - pkgs, - ... -}: -let - sampleService = spec: { - inherit (spec) description; - after = map (dependency: "${dependency}.service") (spec.after or [ ]); - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Type = "oneshot"; - ExecStart = [ - "${pkgs.coreutils}/bin/sleep ${spec.seconds}" - ] - ++ lib.optional (spec.fails or false) "${pkgs.coreutils}/bin/false"; - } - // lib.optionalAttrs (spec ? stopSeconds) { - RemainAfterExit = true; - ExecStop = "${pkgs.coreutils}/bin/sleep ${spec.stopSeconds}"; - }; - }; -in -{ - networking.hostName = "bootvine-demo"; - system.stateVersion = "26.05"; - - boot.consoleLogLevel = lib.mkForce 3; - - boot.initrd.systemd = { - services.demo-initrd-probe = { - description = "Slow hardware probe"; - wantedBy = [ "initrd.target" ]; - before = [ "initrd.target" ]; - unitConfig.DefaultDependencies = false; - serviceConfig = { - Type = "oneshot"; - ExecStart = "/bin/sleep ${sampleUnits.initrdProbeSeconds}"; - }; - }; - }; - - services.bootvine = { - enable = true; - settings.colors = { - tree.current-stage = "light-yellow"; - tree.running-marker = "light-yellow"; - timeline.running = "light-yellow"; - footer.running = "light-yellow"; - password.frame = "light-yellow"; - }; - }; - - console = { - colors = [ - "1d2021" - "fb4934" - "b8bb26" - "fabd2f" - "83a598" - "d3869b" - "8ec07c" - "d5c4a1" - "665c54" - "fb4934" - "b8bb26" - "fabd2f" - "83a598" - "d3869b" - "8ec07c" - "fbf1c7" - ]; - earlySetup = true; - font = "${pkgs.terminus_font}/share/consolefonts/ter-v20n.psf.gz"; - }; - - systemd.services = lib.mapAttrs (_: sampleService) sampleUnits.services // { - greetd.after = [ "multi-user.target" ]; - }; - - services.greetd = { - enable = true; - settings.default_session.command = "${pkgs.greetd}/bin/agreety --cmd ${pkgs.bashInteractive}/bin/bash"; - }; - - users.users.demo = { - isNormalUser = true; - password = "demo"; - }; -} diff --git a/tests/vm-checks/luks.nix b/tests/vm-checks/luks.nix deleted file mode 100644 index 897428c..0000000 --- a/tests/vm-checks/luks.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ bootvineModule }: -let - passphrase = "quietly unlocked"; -in -{ - name = "bootvine-luks-passphrase"; - nodes.machine = - { lib, pkgs, ... }: - let - secretUuid = "9d1f2a8e-0b7c-4c55-9a8e-6b2f8f3c1d00"; - secretDisk = pkgs.runCommand "bootvine-luks-disk" { nativeBuildInputs = [ pkgs.cryptsetup ]; } '' - truncate -s 32M disk.img - echo -n '${passphrase}' | cryptsetup luksFormat -q --disable-locks --type luks2 \ - --pbkdf pbkdf2 --pbkdf-force-iterations 1000 --uuid ${secretUuid} disk.img - - mv disk.img $out - ''; - in - { - imports = [ bootvineModule ]; - - services.bootvine.enable = true; - - testing.initrdBackdoor = true; - boot.initrd.luks.devices = lib.mkVMOverride { - secret.device = "/dev/disk/by-uuid/${secretUuid}"; - }; - - virtualisation.qemu.options = [ - "-vga none" - "-device virtio-vga,xres=1280,yres=800" - ]; - virtualisation.qemu.drives = [ - { - name = "secret"; - file = "${secretDisk}"; - driveExtraOpts = { - format = "raw"; - snapshot = "on"; - }; - } - ]; - }; - - testScript = '' - PASSPHRASE = "${passphrase}" - '' - + builtins.readFile ./console.py - + builtins.readFile ./luks.py; -} diff --git a/vm-tests/default.nix b/vm-tests/default.nix new file mode 100644 index 0000000..e516868 --- /dev/null +++ b/vm-tests/default.nix @@ -0,0 +1,51 @@ +{ lib, bootvineModule }: +let + machineModule = machineName: { + imports = [ + bootvineModule + ./shared/common.nix + ./${machineName}/machine.nix + ]; + networking.hostName = "bootvine-${machineName}"; + }; + + machineNames = [ + "slow-services" + "text-console" + "scripted-initrd" + "encrypted-volume" + "failures" + ]; + + machines = lib.genAttrs machineNames machineModule; +in +{ + inherit machines; + + sampleUnits = import ./slow-services/sample-units.nix; + + nixosSystemFor = + system: machineName: + lib.nixosSystem { + inherit system; + modules = [ + machines.${machineName} + (import ./shared/interactive-vm.nix machineName) + ]; + }; + + checksFor = pkgs: { + vm-boot-and-shutdown = pkgs.testers.runNixOSTest ( + import ./slow-services/check.nix { machine = machines.slow-services; } + ); + vm-luks-passphrase = pkgs.testers.runNixOSTest ( + import ./encrypted-volume/check.nix { + machine = machines.encrypted-volume; + inherit pkgs; + } + ); + vm-failures = pkgs.testers.runNixOSTest ( + import ./failures/check.nix { machine = machines.failures; } + ); + }; +} diff --git a/vm-tests/encrypted-volume/check.nix b/vm-tests/encrypted-volume/check.nix new file mode 100644 index 0000000..086467b --- /dev/null +++ b/vm-tests/encrypted-volume/check.nix @@ -0,0 +1,24 @@ +{ machine, pkgs }: +let + secretDisk = import ./secret-disk.nix { inherit pkgs; }; +in +{ + name = "bootvine-luks-passphrase"; + nodes.machine = { + imports = [ machine ]; + + testing.initrdBackdoor = true; + + virtualisation.qemu.options = [ + "-vga none" + "-device virtio-vga,xres=1280,yres=800" + ]; + virtualisation.qemu.drives = [ secretDisk.qemuDrive ]; + }; + + testScript = '' + PASSPHRASE = "${secretDisk.passphrase}" + '' + + builtins.readFile ../shared/console.py + + builtins.readFile ./check.py; +} diff --git a/tests/vm-checks/luks.py b/vm-tests/encrypted-volume/check.py similarity index 91% rename from tests/vm-checks/luks.py rename to vm-tests/encrypted-volume/check.py index 6ec8f9b..795a697 100644 --- a/tests/vm-checks/luks.py +++ b/vm-tests/encrypted-volume/check.py @@ -11,13 +11,13 @@ machine.start() retry(lambda _: pending_prompt() != "", timeout_seconds=60) wait_for_screen("passphrase") -assert_text_visible() +assert_text_visible(font_has_512_glyphs=True) first_prompt = pending_prompt() machine.send_chars("not the passphrase\n") retry(lambda _: answered_prompts() == 1, timeout_seconds=30) retry(lambda _: pending_prompt() not in ("", first_prompt), timeout_seconds=30) wait_for_screen("passphrase") -assert_text_visible() +assert_text_visible(font_has_512_glyphs=True) machine.screenshot("passphrase") machine.send_chars(f"{PASSPHRASE}\n") machine.wait_for_unit("systemd-cryptsetup@secret.service") diff --git a/vm-tests/encrypted-volume/machine.nix b/vm-tests/encrypted-volume/machine.nix new file mode 100644 index 0000000..1d84ec7 --- /dev/null +++ b/vm-tests/encrypted-volume/machine.nix @@ -0,0 +1,11 @@ +{ lib, pkgs, ... }: +let + secretDisk = import ./secret-disk.nix { inherit pkgs; }; +in +{ + boot.initrd.luks.devices = lib.mkVMOverride { + secret.device = "/dev/disk/by-uuid/${secretDisk.uuid}"; + }; + + virtualisation.vmVariant.virtualisation.qemu.drives = [ secretDisk.qemuDrive ]; +} diff --git a/vm-tests/encrypted-volume/secret-disk.nix b/vm-tests/encrypted-volume/secret-disk.nix new file mode 100644 index 0000000..18c2d87 --- /dev/null +++ b/vm-tests/encrypted-volume/secret-disk.nix @@ -0,0 +1,22 @@ +{ pkgs }: +let + passphrase = "sample passphrase"; + uuid = "9d1f2a8e-0b7c-4c55-9a8e-6b2f8f3c1d00"; + image = pkgs.runCommand "bootvine-luks-disk" { nativeBuildInputs = [ pkgs.cryptsetup ]; } '' + truncate -s 32M disk.img + echo -n '${passphrase}' | cryptsetup luksFormat -q --disable-locks --type luks2 \ + --pbkdf pbkdf2 --pbkdf-force-iterations 1000 --uuid ${uuid} disk.img - + mv disk.img $out + ''; +in +{ + inherit passphrase uuid; + qemuDrive = { + name = "secret"; + file = "${image}"; + driveExtraOpts = { + format = "raw"; + snapshot = "on"; + }; + }; +} diff --git a/vm-tests/failures/check.nix b/vm-tests/failures/check.nix new file mode 100644 index 0000000..ac18a18 --- /dev/null +++ b/vm-tests/failures/check.nix @@ -0,0 +1,49 @@ +{ machine }: +{ + name = "bootvine-failures"; + + nodes.machine = + { pkgs, ... }: + let + holdUntilReleased = + marker: "${pkgs.bash}/bin/bash -c 'until [ -e /run/${marker} ]; do sleep 0.1; done'"; + in + { + imports = [ machine ]; + bootvineSample.startFailuresByHand = true; + testing.initrdBackdoor = true; + + boot.initrd.systemd = { + storePaths = [ "${pkgs.bash}/bin/bash" ]; + services.initrd-failures-hold = { + description = "Initrd held for the check"; + wantedBy = [ "initrd.target" ]; + before = [ "initrd.target" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + ExecStart = holdUntilReleased "release-initrd-failures-hold"; + }; + }; + }; + + systemd.services.boot-failures-hold = { + description = "Boot held for the check"; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = holdUntilReleased "release-boot-failures-hold"; + }; + }; + + virtualisation = { + memorySize = 2048; + qemu.options = [ + "-vga none" + "-device virtio-vga,xres=1280,yres=800" + ]; + }; + }; + + testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py; +} diff --git a/vm-tests/failures/check.py b/vm-tests/failures/check.py new file mode 100644 index 0000000..c61243c --- /dev/null +++ b/vm-tests/failures/check.py @@ -0,0 +1,59 @@ +def start_failing_unit(unit): + machine.succeed(f"systemctl start --no-block {unit}") + + +def unit_result(unit): + return machine.succeed(f"systemctl show -p Result --value {unit}").strip() + + +def failure_panel(): + lines = screen().splitlines() + start = next(index for index, line in enumerate(lines) if " Failures " in line) + return "\n".join(lines[start:]) + + +machine.start() + +wait_for_screen("Initrd", "Initrd held for the check") +start_failing_unit("initrd-nic-firmware-load") +wait_for_screen("Failures", "initrd-nic-firmware-load exited with status 1", "rtl8168h-2.fw not found") +assert_bootvine_in_front() +assert_text_visible(font_has_512_glyphs=True) +machine.succeed("touch /run/release-initrd-failures-hold") +machine.switch_root() + +wait_for_screen("Stage 2", "Boot held for the check") +wait_for_screen("initrd-nic-firmware-load exited with status 1") + +start_failing_unit("failure-report") +wait_for_screen("app-config-check exited with status 2", "missing key 'listen'") +retry(lambda _: unit_result("failure-report") != "", timeout_seconds=10) +assert "× failure-report" not in failure_panel(), f"a dependency failure reached the panel:\n{screen()}" +assert "failure-report" in screen(), f"the dependency failure left the tree:\n{screen()}" + +start_failing_unit("worker-pool") +wait_for_screen("worker-pool", "SIGSEGV", "starting worker pool with 4 threads") + +start_failing_unit("license-server-wait") +wait_for_screen("license-server-wait timed out", "waiting for license server at 10.0.0.9:27000") + +start_failing_unit("metrics-agent") +wait_for_screen("metrics-agent exited with status 203") + +start_failing_unit("http-listener") +wait_for_screen("http-listener restarted too often", "address already in use") + +start_failing_unit("database-client") +wait_for_screen("database-client exited with status 1: Cannot reach database", "connection refused") + +start_failing_unit("upstream-uplink") +wait_for_screen("upstream-uplink exited with status 1", "lost connection to upstream, giving up") + +wait_for_screen("× 9 failed", "connection refused") +assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}" +print(screen()) +machine.screenshot("failures") + +machine.succeed("touch /run/release-boot-failures-hold") +retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60) +print(bootvine_journal()) diff --git a/vm-tests/failures/machine.nix b/vm-tests/failures/machine.nix new file mode 100644 index 0000000..61c918f --- /dev/null +++ b/vm-tests/failures/machine.nix @@ -0,0 +1,148 @@ +{ + config, + lib, + pkgs, + ... +}: +let + startedByHand = config.bootvineSample.startFailuresByHand; + wantedAtBoot = target: lib.optional (!startedByHand) target; + + failingService = + { + description, + script, + serviceConfig ? { }, + unitConfig ? { }, + extraAttributes ? { }, + }: + { + inherit description unitConfig; + wantedBy = wantedAtBoot "multi-user.target"; + serviceConfig = { + Type = "oneshot"; + ExecStart = pkgs.writeShellScript "failing-service" script; + } + // serviceConfig; + } + // extraAttributes; + + initrdFirmwareScript = pkgs.writeShellScript "initrd-nic-firmware-load" '' + echo "loading firmware for enp1s0 (r8169)" + echo "firmware: rtl_nic/rtl8168h-2.fw not found" >&2 + exit 1 + ''; +in +{ + options.bootvineSample.startFailuresByHand = lib.mkEnableOption '' + leaving the failing sample units out of the boot targets, so a check can + start them one at a time + ''; + + config = { + boot.initrd.systemd = { + storePaths = [ + initrdFirmwareScript + "${pkgs.bash}/bin/bash" + ]; + services.initrd-nic-firmware-load = { + description = "Network card firmware"; + wantedBy = wantedAtBoot "initrd.target"; + before = [ "initrd.target" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${pkgs.bash}/bin/bash ${initrdFirmwareScript}"; + }; + }; + }; + + systemd.services = { + app-config-check = failingService { + description = "Application configuration check"; + script = '' + echo "reading /etc/app/app.toml" + echo "config: /etc/app/app.toml: missing key 'listen'" >&2 + exit 2 + ''; + }; + + worker-pool = failingService { + description = "Worker pool"; + script = '' + echo "starting worker pool with 4 threads" + kill -SEGV $$ + ''; + }; + + license-server-wait = failingService { + description = "License server check"; + script = '' + echo "waiting for license server at 10.0.0.9:27000" + sleep infinity + ''; + serviceConfig.TimeoutStartSec = 4; + }; + + metrics-agent = { + description = "Metrics agent"; + wantedBy = wantedAtBoot "multi-user.target"; + serviceConfig = { + Type = "oneshot"; + ExecStart = "/usr/bin/metrics-agent --listen :9100"; + }; + }; + + http-listener = failingService { + description = "HTTP listener"; + script = '' + echo "bind 0.0.0.0:8080: address already in use" >&2 + exit 1 + ''; + serviceConfig = { + Restart = "on-failure"; + RestartSec = "200ms"; + }; + unitConfig = { + StartLimitBurst = 3; + StartLimitIntervalSec = 30; + }; + }; + + database-client = failingService { + description = "Database client"; + script = '' + systemd-notify --status="Cannot reach database at db.internal:5432" + echo "connect db.internal:5432: connection refused" >&2 + exit 1 + ''; + serviceConfig = { + Type = "notify"; + NotifyAccess = "all"; + }; + }; + + upstream-uplink = failingService { + description = "Upstream uplink"; + script = '' + echo "connected to upstream" + sleep 3 + echo "lost connection to upstream, giving up" >&2 + exit 1 + ''; + serviceConfig.Type = "simple"; + }; + + failure-report = failingService { + description = "Failure report"; + script = '' + echo "all checks passed" + ''; + extraAttributes = { + requires = [ "app-config-check.service" ]; + after = [ "app-config-check.service" ]; + }; + }; + }; + }; +} diff --git a/vm-tests/scripted-initrd/machine.nix b/vm-tests/scripted-initrd/machine.nix new file mode 100644 index 0000000..d3c53d5 --- /dev/null +++ b/vm-tests/scripted-initrd/machine.nix @@ -0,0 +1,6 @@ +{ lib, ... }: +{ + imports = [ ../slow-services/machine.nix ]; + + boot.initrd.systemd.enable = lib.mkForce false; +} diff --git a/vm-tests/shared/common.nix b/vm-tests/shared/common.nix new file mode 100644 index 0000000..10fdfa0 --- /dev/null +++ b/vm-tests/shared/common.nix @@ -0,0 +1,45 @@ +{ lib, pkgs, ... }: +{ + system.stateVersion = "26.05"; + + boot.consoleLogLevel = lib.mkForce 3; + + services.bootvine = { + enable = true; + settings.colors = { + tree.current-stage = "light-yellow"; + tree.running-marker = "light-yellow"; + timeline.running = "light-yellow"; + footer.running = "light-yellow"; + password.frame = "light-yellow"; + }; + }; + + console = { + colors = [ + "1d2021" + "fb4934" + "b8bb26" + "fabd2f" + "83a598" + "d3869b" + "8ec07c" + "d5c4a1" + "665c54" + "fb4934" + "b8bb26" + "fabd2f" + "83a598" + "d3869b" + "8ec07c" + "fbf1c7" + ]; + earlySetup = true; + font = "${pkgs.terminus_font}/share/consolefonts/ter-v20n.psf.gz"; + }; + + users.users.sample-user = { + isNormalUser = true; + password = "sample-user"; + }; +} diff --git a/tests/vm-checks/console.py b/vm-tests/shared/console.py similarity index 98% rename from tests/vm-checks/console.py rename to vm-tests/shared/console.py index 75adb3b..f466410 100644 --- a/tests/vm-checks/console.py +++ b/vm-tests/shared/console.py @@ -51,6 +51,7 @@ def wait_for_screen(*fragments): text = screen() if last_try: print(text) + print(bootvine_journal()) return all(fragment in text for fragment in fragments) retry(shows_all, timeout_seconds=60) diff --git a/tests/sample-machine/interactive-vm.nix b/vm-tests/shared/interactive-vm.nix similarity index 79% rename from tests/sample-machine/interactive-vm.nix rename to vm-tests/shared/interactive-vm.nix index e876556..5f062a6 100644 --- a/tests/sample-machine/interactive-vm.nix +++ b/vm-tests/shared/interactive-vm.nix @@ -1,4 +1,4 @@ -{ +machineName: { boot.loader.grub.enable = false; services.getty.autologinUser = "root"; @@ -11,7 +11,7 @@ memorySize = 2048; cores = 2; graphics = true; - diskImage = "\${XDG_CACHE_HOME:-$HOME/.cache}/bootvine-sample-machine.qcow2"; + diskImage = "\${XDG_CACHE_HOME:-$HOME/.cache}/bootvine-${machineName}.qcow2"; qemu.options = [ "-vga none" "-device virtio-vga,xres=1920,yres=1080" diff --git a/tests/vm-checks/boot.nix b/vm-tests/slow-services/check.nix similarity index 54% rename from tests/vm-checks/boot.nix rename to vm-tests/slow-services/check.nix index 768351a..b8bdf04 100644 --- a/tests/vm-checks/boot.nix +++ b/vm-tests/slow-services/check.nix @@ -1,4 +1,4 @@ -{ sampleMachine }: +{ machine }: { name = "bootvine-boot-and-shutdown"; @@ -8,21 +8,21 @@ waitFor = flag: "/bin/sh -c 'until [ -e /run/${flag} ]; do sleep 0.1; done'"; in { - imports = [ sampleMachine ]; + imports = [ machine ]; testing.initrdBackdoor = true; - boot.initrd.systemd.services.demo-initrd-probe.serviceConfig.ExecStart = lib.mkForce ( - waitFor "release-initrd-probe" + boot.initrd.systemd.services.initrd-hardware-probe.serviceConfig.ExecStart = lib.mkForce ( + waitFor "release-initrd-hardware-probe" ); systemd = { settings.Manager.ShowStatus = lib.mkForce true; - services.demo-db.serviceConfig.ExecStop = lib.mkForce (waitFor "release-demo-db"); - services.backdoor.before = [ "demo-db.service" ]; - services.demo-late-sensor = { + services.sample-database.serviceConfig.ExecStop = lib.mkForce (waitFor "release-sample-database"); + services.backdoor.before = [ "sample-database.service" ]; + services.late-sensor-probe = { wantedBy = [ "graphical.target" ]; after = [ "greetd.service" ]; serviceConfig = { Type = "oneshot"; - ExecStart = waitFor "release-late-sensor"; + ExecStart = waitFor "release-late-sensor-probe"; }; }; }; @@ -35,5 +35,5 @@ }; }; - testScript = builtins.readFile ./console.py + builtins.readFile ./boot.py; + testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py; } diff --git a/tests/vm-checks/boot.py b/vm-tests/slow-services/check.py similarity index 81% rename from tests/vm-checks/boot.py rename to vm-tests/slow-services/check.py index e8bd0ee..f327ed6 100644 --- a/tests/vm-checks/boot.py +++ b/vm-tests/slow-services/check.py @@ -12,20 +12,20 @@ wait_for_screen("Initrd", "Slow hardware probe") assert_bootvine_in_front() assert_text_visible(font_has_512_glyphs=True) assert console_level() == "1", f"the kernel console was not quieted: {console_level()}" -machine.succeed("touch /run/release-initrd-probe") +machine.succeed("touch /run/release-initrd-hardware-probe") machine.switch_root() -retry(lambda _: unit_state("demo-db") == "activating", timeout_seconds=60) -wait_for_screen("Stage 2", "demo-db") +retry(lambda _: unit_state("sample-database") == "activating", timeout_seconds=60) +wait_for_screen("Stage 2", "sample-database") assert_bootvine_in_front() assert_text_visible(font_has_512_glyphs=True) assert console_level() == "1", f"stage 2 sysctl reopened the kernel console: {console_level()}" machine.wait_for_unit("greetd.service") retry(lambda _: unit_state("bootvine") == "inactive", timeout_seconds=60) -assert unit_state("demo-late-sensor") == "activating", "the boot finished before the display manager took over" +assert unit_state("late-sensor-probe") == "activating", "the boot finished before the display manager took over" assert foreground_terminal() != f"tty{BOOTVINE_TERMINAL}", "bootvine took the console back from the display manager" -machine.succeed("touch /run/release-late-sensor") +machine.succeed("touch /run/release-late-sensor-probe") machine.wait_for_unit("graphical.target") journal = bootvine_journal() @@ -50,9 +50,9 @@ assert "boot already finished" in bootvine_journal(), "a restart after boot did assert foreground_terminal() == foreground, "a restart after boot took the console" machine.succeed("systemctl poweroff --no-block") -retry(lambda _: unit_state("demo-db") == "deactivating", timeout_seconds=60) -wait_for_screen("Shutdown", "demo-db") +retry(lambda _: unit_state("sample-database") == "deactivating", timeout_seconds=60) +wait_for_screen("Shutdown", "sample-database") assert_bootvine_in_front() assert_text_visible(font_has_512_glyphs=True) -machine.succeed("touch /run/release-demo-db") +machine.succeed("touch /run/release-sample-database") machine.wait_for_shutdown() diff --git a/vm-tests/slow-services/machine.nix b/vm-tests/slow-services/machine.nix new file mode 100644 index 0000000..1a4d0ed --- /dev/null +++ b/vm-tests/slow-services/machine.nix @@ -0,0 +1,42 @@ +{ lib, pkgs, ... }: +let + sampleUnits = import ./sample-units.nix; + + sampleService = spec: { + inherit (spec) description; + after = map (dependency: "${dependency}.service") (spec.after or [ ]); + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = [ + "${pkgs.coreutils}/bin/sleep ${spec.seconds}" + ] + ++ lib.optional (spec.fails or false) "${pkgs.coreutils}/bin/false"; + } + // lib.optionalAttrs (spec ? stopSeconds) { + RemainAfterExit = true; + ExecStop = "${pkgs.coreutils}/bin/sleep ${spec.stopSeconds}"; + }; + }; +in +{ + boot.initrd.systemd.services.initrd-hardware-probe = { + description = "Slow hardware probe"; + wantedBy = [ "initrd.target" ]; + before = [ "initrd.target" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + ExecStart = "/bin/sleep ${sampleUnits.initrdProbeSeconds}"; + }; + }; + + systemd.services = lib.mapAttrs (_: sampleService) sampleUnits.services // { + greetd.after = [ "multi-user.target" ]; + }; + + services.greetd = { + enable = true; + settings.default_session.command = "${pkgs.greetd}/bin/agreety --cmd ${pkgs.bashInteractive}/bin/bash"; + }; +} diff --git a/tests/sample-machine/sample-unit-files.nix b/vm-tests/slow-services/sample-unit-files.nix similarity index 100% rename from tests/sample-machine/sample-unit-files.nix rename to vm-tests/slow-services/sample-unit-files.nix diff --git a/tests/sample-machine/sample-units.nix b/vm-tests/slow-services/sample-units.nix similarity index 63% rename from tests/sample-machine/sample-units.nix rename to vm-tests/slow-services/sample-units.nix index d3408a1..cec8124 100644 --- a/tests/sample-machine/sample-units.nix +++ b/vm-tests/slow-services/sample-units.nix @@ -1,51 +1,51 @@ { - target = "bootvine-demo"; + target = "bootvine-sample-services"; initrdProbeSeconds = "6"; services = { - demo-disk = { + scratch-disk-prepare = { description = "Scratch disk preparation"; seconds = "2"; stopSeconds = "2"; }; - demo-db = { - description = "Demo database"; + sample-database = { + description = "Sample database"; seconds = "3"; stopSeconds = "3"; - after = [ "demo-disk" ]; + after = [ "scratch-disk-prepare" ]; }; - demo-cache = { + memory-cache = { description = "In-memory cache"; seconds = "1.5"; stopSeconds = "1"; }; - demo-web = { + web-frontend = { description = "Web frontend"; seconds = "2"; stopSeconds = "2"; after = [ - "demo-db" - "demo-cache" + "sample-database" + "memory-cache" ]; }; - demo-fetch = { + remote-content-sync = { description = "Remote content sync"; seconds = "4"; }; - demo-flaky = { + flaky-health-check = { description = "Flaky health check"; seconds = "1"; fails = true; }; - demo-slow = { + slow-warm-up = { description = "Slow warm-up job"; seconds = "8"; }; - demo-report = { + startup-report = { description = "Startup report"; seconds = "1"; after = [ - "demo-web" - "demo-fetch" + "web-frontend" + "remote-content-sync" ]; }; }; diff --git a/vm-tests/text-console/machine.nix b/vm-tests/text-console/machine.nix new file mode 100644 index 0000000..9043e74 --- /dev/null +++ b/vm-tests/text-console/machine.nix @@ -0,0 +1,6 @@ +{ lib, ... }: +{ + imports = [ ../slow-services/machine.nix ]; + + services.greetd.enable = lib.mkForce false; +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 90d35b4..f61c226 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -16,8 +16,8 @@ struct RepoTasks { #[derive(Subcommand)] enum Task { #[command( - about = "Record the README screencasts with vhs and copy the passphrase screenshot into \ - recordings/" + about = "Record the screencasts with vhs and copy the passphrase and failures screenshots \ + into recordings/" )] RecordScreencasts(RecordScreencastsOptions), #[command( @@ -30,12 +30,17 @@ enum Task { struct RecordScreencastsOptions { #[arg(long, help = "passphrase.png produced by the vm-luks-passphrase check")] passphrase_png: PathBuf, + #[arg(long, help = "failures.png produced by the vm-failures check")] + failures_png: PathBuf, } fn main() -> ExitCode { let outcome = match RepoTasks::parse().task { Task::RecordScreencasts(options) => { - record_screencasts::record_screencasts(&options.passphrase_png) + record_screencasts::record_screencasts(&record_screencasts::CheckScreenshots { + passphrase_png: &options.passphrase_png, + failures_png: &options.failures_png, + }) } Task::SyncDefaultConfig => sync_default_config::sync_default_config(), }; diff --git a/xtask/src/record_screencasts.rs b/xtask/src/record_screencasts.rs index 3c77800..ec2ea50 100644 --- a/xtask/src/record_screencasts.rs +++ b/xtask/src/record_screencasts.rs @@ -8,23 +8,40 @@ use xshell::{Shell, cmd}; use crate::task_error::TaskError; -const TAPES: [&str; 2] = ["boot.tape", "shutdown.tape"]; +const TAPES: [&str; 2] = [ + "scripted-demos/tapes/scripted-boot.tape", + "scripted-demos/tapes/scripted-shutdown.tape", +]; -pub fn record_screencasts(passphrase_png: &Path) -> Result<(), TaskError> { - let recordings = Path::new("recordings"); - if !recordings.is_dir() { - return Err(TaskError::NotInRepositoryRoot); - } - let shell = Shell::new()?; - TAPES.iter().try_for_each(|tape| { - let tape_path = recordings.join(tape); - cmd!(shell, "vhs {tape_path}").run() - })?; - let copied_png = recordings.join("passphrase.png"); - fs::copy(passphrase_png, &copied_png) +pub struct CheckScreenshots<'a> { + pub passphrase_png: &'a Path, + pub failures_png: &'a Path, +} + +fn copy_screenshot( + screenshot: &Path, + recordings: &Path, + published_name: &str, +) -> Result<(), TaskError> { + let copied_png = recordings.join(published_name); + fs::copy(screenshot, &copied_png) .and_then(|_| fs::set_permissions(&copied_png, Permissions::from_mode(0o644))) .map_err(|source| TaskError::File { path: copied_png, source, }) } + +pub fn record_screencasts(screenshots: &CheckScreenshots) -> Result<(), TaskError> { + let recordings = Path::new("recordings"); + if !recordings.is_dir() { + return Err(TaskError::NotInRepositoryRoot); + } + let shell = Shell::new()?; + TAPES.iter().try_for_each(|tape| { + let tape_path = Path::new(tape); + cmd!(shell, "vhs {tape_path}").run() + })?; + copy_screenshot(screenshots.passphrase_png, recordings, "passphrase.png")?; + copy_screenshot(screenshots.failures_png, recordings, "failures.png") +}