feat: show why units failed, with their last journal lines

Failed jobs keep their kind: units that broke or timed out get a failure
panel entry with the reason systemd reports (exit status, signal, timeout,
start limit, status text) and their last journal lines read through
libsystemd. Units that failed only through a dependency are dimmed and left
out of the panel, and units that crash after starting count as failed.

Restructure the repository around this:
- move the library and binary into bootvine/ and the scripted demos into
  their own unpublished scripted-demos crate, with their VHS tapes
- move the crane builds and NixOS module into nix/
- group each sample VM with its check under vm-tests/, adding text-console,
  scripted-initrd, encrypted-volume and failures machines and a vm-failures
  check
- add docs/guide.md walking through the screen with every recording, keep
  one recording in the README, and re-render all recordings
This commit is contained in:
Willem Vanhulle committed 2026-10-30 14:59:10 +01:00
1 parent f2544a964a
commit 7e95cbd8e9
96 files changed
+2027 -462

No files matched your search

Generated
+88
View File
@@ -158,6 +158,7 @@ dependencies = [
"ser_nix",
"serde",
"serde_plain",
"systemd",
"thiserror 2.0.21",
"tokio",
"toml",
@@ -166,6 +167,12 @@ dependencies = [
"zeroize",
]
[[package]]
name = "build-env"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1522ac6ee801a11bf9ef3f80403f4ede6eb41291fac3dde3de09989679305f25"
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -365,6 +372,16 @@ dependencies = [
"phf",
]
[[package]]
name = "cstr-argument"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bd9c8e659a473bce955ae5c35b116af38af11a7acb0b480e01f3ed348aeb40"
dependencies = [
"cfg-if",
"memchr",
]
[[package]]
name = "darling"
version = "0.24.1"
@@ -588,6 +605,33 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "foreign-types"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
dependencies = [
"foreign-types-macros",
"foreign-types-shared",
]
[[package]]
name = "foreign-types-macros"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea5190182e6915eb873ddbc16e23b711b6eb1f9c00a0d0a3a91b5f6228475225"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.9",
]
[[package]]
name = "foreign-types-shared"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"
[[package]]
name = "futures"
version = "0.3.34"
@@ -880,6 +924,18 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libsystemd-sys"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "976306de183e6046819ef6505888d00996214766a3f4660a2ed5761c84a20aed"
dependencies = [
"build-env",
"cfg-if",
"libc",
"pkg-config",
]
[[package]]
name = "line-clipping"
version = "0.3.8"
@@ -1532,6 +1588,17 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "scripted-demos"
version = "0.1.0"
dependencies = [
"bootvine",
"futures",
"ratatui",
"tokio",
"zeroize",
]
[[package]]
name = "semver"
version = "1.0.28"
@@ -1750,6 +1817,21 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "systemd"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01e9d1976a15b86245def55d20d52b5818e1a1e81aa030b6a608d3ce57709423"
dependencies = [
"cstr-argument",
"foreign-types",
"libc",
"libsystemd-sys",
"log",
"memchr",
"utf8-cstr",
]
[[package]]
name = "termina"
version = "0.3.3"
@@ -1993,6 +2075,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]]
name = "utf8-cstr"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55bcbb425141152b10d5693095950b51c3745d019363fc2929ffd8f61449b628"
[[package]]
name = "utf8parse"
version = "0.2.2"
+2 -68
View File
@@ -1,18 +1,6 @@
[package]
categories = ["command-line-utilities", "os::linux-apis"]
description = "Boot and shutdown progress for systemd on the console, with each job nested under the units waiting on it"
edition = "2024"
include = ["/src", "/examples", "/system-files", "/README.md", "/LICENSE"]
keywords = ["systemd", "boot", "initrd", "tui", "console"]
license = "GPL-3.0-or-later"
name = "bootvine"
readme = "README.md"
repository = "https://codeberg.org/wvhulle/bootvine"
rust-version = "1.91"
version = "0.1.0"
[workspace]
members = [".", "xtask"]
members = ["bootvine", "scripted-demos", "xtask"]
resolver = "3"
[workspace.lints.clippy]
absolute_paths = "warn"
@@ -20,60 +8,6 @@ missing_errors_doc = "allow"
needless_for_each = "allow"
pedantic = { priority = -1, level = "warn" }
[lints]
workspace = true
[dependencies]
clap = { version = "4.6.7", default-features = false, features = [
"std",
"derive",
"help",
"usage",
"error-context",
] }
crossterm = "0.29.0"
dbus = "0.9.12"
futures = { version = "0.3.34", default-features = false, features = [
"std",
"async-await",
] }
heck = "0.5.0"
jiff = { version = "0.2.38", default-features = false, features = [
"serde",
"std",
] }
petgraph = { version = "0.8.3", default-features = false, features = [
"std",
"graphmap",
] }
ratatui = { version = "0.30.2", features = [
"serde",
"unstable-rendered-line-info",
] }
rustix = { version = "1.1.5", features = [
"event",
"fs",
"process",
"stdio",
"system",
"termios",
"time",
] }
ser_nix = "0.2.4"
serde = { version = "1.0.229", features = ["derive"] }
serde_plain = "1.0.2"
thiserror = "2.0.12"
tokio = { version = "1.54.0", default-features = false, features = [
"rt",
"time",
"net",
"signal",
] }
toml = { version = "1.1.8", features = ["preserve_order"] }
unicode-truncate = "2.0.1"
unicode-width = "0.2.2"
zeroize = "1.9.1"
[profile.release]
codegen-units = 1
lto = "fat"
+16 -31
View File
@@ -11,21 +11,16 @@ gave me any idea of what was going on.
<img src="https://codeberg.org/wvhulle/bootvine/raw/branch/main/recordings/boot.gif" width="691" alt="A boot through the initrd into stage 2, with one failed unit">
It follows shutdowns and reboots too: the units being stopped, nested under the
units that have to stop first.
<img src="https://codeberg.org/wvhulle/bootvine/raw/branch/main/recordings/shutdown.gif" width="691" alt="A poweroff: units stopping under the targets waiting on them">
Passphrase prompts, such as a
[LUKS](https://gitlab.com/cryptsetup/cryptsetup) unlock in the initrd, are
answered in place over the tree.
<img src="https://codeberg.org/wvhulle/bootvine/raw/branch/main/recordings/passphrase.png" width="640" alt="A LUKS passphrase prompt floating over the initrd tree">
It also follows shutdowns, answers passphrase prompts such as a
[LUKS](https://gitlab.com/cryptsetup/cryptsetup) unlock in place, and shows why
a unit failed with its last journal lines. The
[guide](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/guide.md)
walks through each part of the screen.
## Requirements
- systemd ≥ 246 as PID 1
- [libdbus](https://www.freedesktop.org/wiki/Software/dbus/)
- [libdbus](https://www.freedesktop.org/wiki/Software/dbus/) and libsystemd
- a virtual terminal
- a systemd initrd, to show the boot from the initrd as well as stage 2
@@ -34,33 +29,23 @@ for each distribution.
## Try it without installing
Neither of these touches your system's boot.
Play a scripted boot, poweroff and encrypted-root boot in your terminal, with
[Rust](https://www.rust-lang.org/tools/install) installed:
With [Nix](https://nixos.org/download/), boot a sample [NixOS](https://nixos.org) machine with slow and
failing units in a [QEMU](https://www.qemu.org) window, without touching your
system's boot:
```sh
git clone https://codeberg.org/wvhulle/bootvine.git
cd bootvine
cargo run --example boot
cargo run --example shutdown
cargo run --example luks
nix run git+https://codeberg.org/wvhulle/bootvine#sample-vm-slow-services
```
`q` quits. The encrypted root unlocks with the passphrase `demo`; `Esc` quits
while it asks.
Or, with [Nix](https://nixos.org/download/), boot a sample [NixOS](https://nixos.org) machine with slow and
failing units in a [QEMU](https://www.qemu.org) window:
```sh
nix run git+https://codeberg.org/wvhulle/bootvine#sample-machine-vm
```
It logs in as root; `poweroff` shows the shutdown.
It logs in as root; `poweroff` shows the shutdown. The
[guide](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/guide.md#trying-it)
lists the other sample machines and the scripted demos that play in a terminal.
## Documentation
- [Guide](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/guide.md)
to the screen: the tree, the timeline, failures, shutdown and passphrase
prompts.
- [Install](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/install.md)
on NixOS, Ubuntu and other distributions.
- [Configuration](https://codeberg.org/wvhulle/bootvine/src/branch/main/docs/configuration.md)
+69
View File
@@ -0,0 +1,69 @@
[package]
categories = ["command-line-utilities", "os::linux-apis"]
description = "Boot and shutdown progress for systemd on the console, with each job nested under the units waiting on it"
edition = "2024"
include = ["/src", "/system-files", "/LICENSE"]
keywords = ["systemd", "boot", "initrd", "tui", "console"]
license = "GPL-3.0-or-later"
name = "bootvine"
readme = "../README.md"
repository = "https://codeberg.org/wvhulle/bootvine"
rust-version = "1.91"
version = "0.1.0"
[lints]
workspace = true
[dependencies]
clap = { version = "4.6.7", default-features = false, features = [
"std",
"derive",
"help",
"usage",
"error-context",
] }
crossterm = "0.29.0"
dbus = "0.9.12"
futures = { version = "0.3.34", default-features = false, features = [
"std",
"async-await",
] }
heck = "0.5.0"
jiff = { version = "0.2.38", default-features = false, features = [
"serde",
"std",
] }
petgraph = { version = "0.8.3", default-features = false, features = [
"std",
"graphmap",
] }
ratatui = { version = "0.30.2", features = [
"serde",
"unstable-rendered-line-info",
] }
rustix = { version = "1.1.5", features = [
"event",
"fs",
"process",
"stdio",
"system",
"termios",
"time",
] }
ser_nix = "0.2.4"
serde = { version = "1.0.229", features = ["derive"] }
serde_plain = "1.0.2"
systemd = { version = "0.10.1", default-features = false, features = [
"journal",
] }
thiserror = "2.0.12"
tokio = { version = "1.54.0", default-features = false, features = [
"rt",
"time",
"net",
"signal",
] }
toml = { version = "1.1.8", features = ["preserve_order"] }
unicode-truncate = "2.0.1"
unicode-width = "0.2.2"
zeroize = "1.9.1"
+1
View File
@@ -0,0 +1 @@
../LICENSE
@@ -41,10 +41,44 @@ struct PendingJob {
kind: JobKind,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FailureKind {
Broke,
TimedOut,
DependencyFailed,
}
impl FailureKind {
#[must_use]
pub fn has_own_cause(self) -> bool {
self != Self::DependencyFailed
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FailureCause {
pub reason: String,
pub log_lines: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum FailureExplanation {
Unexplained,
Explained(FailureCause),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FailedUnit {
pub unit: String,
pub label: String,
pub kind: FailureKind,
pub explanation: FailureExplanation,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Outcome {
Succeeded { at: Duration },
Failed,
Failed(FailureKind),
Abandoned,
}
@@ -109,7 +143,7 @@ pub struct BootState {
pub phase: Phase,
pub origin: Duration,
pub root: Root,
pub failed: Vec<String>,
pub failed: Vec<FailedUnit>,
pub finished_stages: Vec<FinishedStage>,
pub finished_runs: Vec<FinishedRun>,
pub handoff_unit: Option<String>,
@@ -1,6 +1,6 @@
use std::time::Duration;
use super::{Activity, BootState, JobKind, JobResult, UnitFacts};
use super::{Activity, BootState, FailureCause, JobKind, JobResult, UnitFacts};
pub enum BootChange {
TrackUnit {
@@ -22,6 +22,10 @@ pub enum BootChange {
result: JobResult,
at: Duration,
},
ExplainFailure {
unit: String,
cause: FailureCause,
},
SetRoot(String),
SetHandoff(String),
SetOrigin(Duration),
@@ -42,6 +46,7 @@ impl BootState {
BootChange::UpdateActivity { unit, activity } => self.update_activity(&unit, activity),
BootChange::QueueJob { id, unit, kind, at } => self.queue_job(id, &unit, kind, at),
BootChange::FinishJob { id, result, at } => self.finish_job(id, result, at),
BootChange::ExplainFailure { unit, cause } => self.explain_failure(&unit, cause),
BootChange::SetRoot(root) => self.set_root(root),
BootChange::SetHandoff(unit) => {
if let Some(handoff) = &mut self.handoff_unit {
@@ -62,7 +67,7 @@ mod tests {
use super::*;
use crate::{
boot_model::Phase,
demo_script::boot_test_fixtures::{MS, facts, waiting_rows},
boot_test_fixtures::{MS, facts, waiting_rows},
};
#[test]
File renamed without changes.
@@ -1,9 +1,10 @@
use std::{collections::HashMap, time::Duration};
use super::{
ActiveState, Activity, BootState, FinishedRun, FinishedStage, HANDOFF_UNIT, HOST_MILESTONES,
JobKind, JobResult, Outcome, PendingJob, Phase, Progress, Root, SHUTDOWN_MILESTONES, Tally,
UnitFacts, UnitKind, UnitRecord, unit_facts::display_label,
ActiveState, Activity, BootState, FailedUnit, FailureCause, FailureExplanation, FailureKind,
FinishedRun, FinishedStage, HANDOFF_UNIT, HOST_MILESTONES, JobKind, JobResult, Outcome,
PendingJob, Phase, Progress, Root, SHUTDOWN_MILESTONES, Tally, UnitFacts, UnitKind, UnitRecord,
unit_facts::display_label,
};
impl UnitRecord {
@@ -116,11 +117,40 @@ impl BootState {
}
pub fn update_activity(&mut self, unit: &str, activity: Activity) {
if let Some(record) = self.units.get_mut(unit) {
record.activity = record.activity.merge(activity);
let Some(record) = self.units.get_mut(unit) else {
return;
};
record.activity = record.activity.merge(activity);
if activity.state == Some(ActiveState::Failed) {
let label = display_label(&record.facts);
self.record_failure(unit, label, FailureKind::Broke);
}
}
fn record_failure(&mut self, unit: &str, label: String, kind: FailureKind) {
match self.failed.iter_mut().find(|failed| failed.unit == unit) {
Some(known) if kind != FailureKind::Broke => known.kind = kind,
Some(_) => {}
None => self.failed.push(FailedUnit {
unit: unit.to_owned(),
label,
kind,
explanation: FailureExplanation::Unexplained,
}),
}
}
pub fn explain_failure(&mut self, unit: &str, cause: FailureCause) {
if let Some(failed) = self.failed.iter_mut().find(|failed| failed.unit == unit) {
failed.explanation = FailureExplanation::Explained(cause);
}
}
#[must_use]
pub fn has_failed(&self, unit: &str) -> bool {
self.failed.iter().any(|failed| failed.unit == unit)
}
pub fn queue_job(&mut self, id: u32, unit: &str, kind: JobKind, at: Duration) {
if self.enqueue(id, unit, kind, at)
&& kind == JobKind::BringsUp
@@ -150,18 +180,26 @@ impl BootState {
let Some(unit) = self.jobs.remove(&id) else {
return;
};
let Some(record) = self.units.get_mut(&unit) else {
let Some(label) = self
.units
.get(&unit)
.map(|record| display_label(&record.facts))
else {
return;
};
let outcome = match result {
JobResult::Done => Outcome::Succeeded { at },
JobResult::Skipped | JobResult::Canceled => Outcome::Abandoned,
JobResult::Failed | JobResult::Timeout | JobResult::Dependency => Outcome::Failed,
JobResult::Failed => Outcome::Failed(FailureKind::Broke),
JobResult::Timeout => Outcome::Failed(FailureKind::TimedOut),
JobResult::Dependency => Outcome::Failed(FailureKind::DependencyFailed),
};
let label = display_label(&record.facts);
if outcome == Outcome::Failed && !self.failed.contains(&label) {
self.failed.push(label.clone());
if let Outcome::Failed(kind) = outcome {
self.record_failure(&unit, label.clone(), kind);
}
let Some(record) = self.units.get_mut(&unit) else {
return;
};
let job = match record.progress {
Progress::Pending(job) if job.id == id => job,
_ => return,
@@ -176,7 +214,7 @@ impl BootState {
label,
started_at,
finished_at: at,
failed: outcome == Outcome::Failed,
failed: matches!(outcome, Outcome::Failed(_)),
});
}
}
@@ -193,7 +231,7 @@ impl BootState {
failed_units: self
.units
.values()
.filter(|record| record.progress == Progress::Finished(Outcome::Failed))
.filter(|record| matches!(record.progress, Progress::Finished(Outcome::Failed(_))))
.count(),
started_at: self.origin,
reached_at: at,
@@ -254,7 +292,7 @@ impl BootState {
Progress::Finished(Outcome::Succeeded { .. } | Outcome::Abandoned) => {
tally.done += 1;
}
Progress::Finished(Outcome::Failed) | Progress::Untouched => {}
Progress::Finished(Outcome::Failed(_)) | Progress::Untouched => {}
}
tally
})
@@ -266,17 +304,82 @@ mod tests {
use super::*;
use crate::{
boot_model::{BootChange, RowStatus, UnitTimeouts},
demo_script::boot_test_fixtures::{
boot_test_fixtures::{
MS, activating, facts, nested_boot_sample as sample, queue_unit, queued_boot, running,
waiting, waiting_rows,
},
};
fn failure_kinds(state: &BootState) -> Vec<(&str, FailureKind)> {
state
.failed
.iter()
.map(|failed| (failed.unit.as_str(), failed.kind))
.collect()
}
#[test]
fn job_results_keep_their_failure_kind() {
let mut state = sample();
state.finish_job(4, JobResult::Timeout, 2000 * MS);
state.finish_job(3, JobResult::Dependency, 2000 * MS);
assert_eq!(
failure_kinds(&state),
[
("nm.service", FailureKind::TimedOut),
("nm-wait.service", FailureKind::DependencyFailed),
]
);
}
#[test]
fn a_unit_entering_the_failed_state_counts_without_a_failed_job() {
let mut state = sample();
state.update_activity(
"nm.service",
Activity {
state: Some(ActiveState::Failed),
..Activity::default()
},
);
assert_eq!(failure_kinds(&state), [("nm.service", FailureKind::Broke)]);
state.finish_job(4, JobResult::Timeout, 2000 * MS);
assert_eq!(
failure_kinds(&state),
[("nm.service", FailureKind::TimedOut)]
);
assert_eq!(state.tally().failed, 1);
}
#[test]
fn an_explanation_attaches_to_its_failed_unit() {
let mut state = sample();
let cause = FailureCause {
reason: "exited with status 1".to_owned(),
log_lines: vec!["no carrier".to_owned()],
};
state.explain_failure("nm.service", cause.clone());
assert!(state.failed.is_empty());
state.finish_job(4, JobResult::Failed, 2000 * MS);
state.explain_failure("nm.service", cause.clone());
assert_eq!(
state.failed[0].explanation,
FailureExplanation::Explained(cause)
);
}
#[test]
fn failures_are_listed_and_not_counted_as_done() {
let mut state = sample();
state.finish_job(4, JobResult::Failed, 2000 * MS);
assert_eq!(state.failed, ["nm"]);
assert_eq!(
state
.failed
.iter()
.map(|failed| failed.label.as_str())
.collect::<Vec<_>>(),
["nm"]
);
let tally = state.tally();
assert_eq!(tally.failed, 1);
assert_eq!(tally.done, 0);
@@ -23,7 +23,9 @@ impl BootState {
Progress::Untouched => record.activity.active_enter.filter(|entered| {
record.activity.state == Some(ActiveState::Active) && *entered >= self.origin
}),
Progress::Pending(_) | Progress::Finished(Outcome::Failed | Outcome::Abandoned) => None,
Progress::Pending(_) | Progress::Finished(Outcome::Failed(_) | Outcome::Abandoned) => {
None
}
}
}
@@ -81,9 +83,7 @@ mod tests {
use super::*;
use crate::{
boot_model::{Activity, JobResult, Phase},
demo_script::boot_test_fixtures::{
MS, nested_boot_sample, queued_boot, waiting, waiting_rows,
},
boot_test_fixtures::{MS, nested_boot_sample, queued_boot, waiting, waiting_rows},
};
#[test]
@@ -152,7 +152,7 @@ mod tests {
use serde_plain::from_str;
use super::*;
use crate::demo_script::boot_test_fixtures::{facts, waiting};
use crate::boot_test_fixtures::{facts, waiting};
#[test]
fn unknown_systemd_strings_fall_back_instead_of_failing() {
@@ -7,7 +7,7 @@ use petgraph::{
};
use super::{
BootState, Outcome, Phase, Progress, UnitKind, UnitRecord,
BootState, FailureKind, Outcome, Phase, Progress, UnitKind, UnitRecord,
unit_facts::{display_label, summary},
};
@@ -31,7 +31,7 @@ pub enum RowStatus {
since: Duration,
deadline: Option<Duration>,
},
Failed,
Failed(FailureKind),
Done,
}
@@ -87,7 +87,7 @@ impl UnitRecord {
deadline: self.deadline(),
})
}
Progress::Finished(Outcome::Failed) => RowStatus::Failed,
Progress::Finished(Outcome::Failed(kind)) => RowStatus::Failed(kind),
Progress::Finished(_) | Progress::Untouched => RowStatus::Done,
}
}
@@ -100,7 +100,7 @@ impl BootState {
self.units
.iter()
.filter(|(name, record)| match record.progress {
Progress::Pending(_) | Progress::Finished(Outcome::Failed) => true,
Progress::Pending(_) | Progress::Finished(Outcome::Failed(_)) => true,
Progress::Finished(Outcome::Succeeded { at }) => {
at >= finished_since || self.is_milestone(name)
}
@@ -240,7 +240,7 @@ impl BootState {
Progress::Finished(Outcome::Succeeded { at }) => {
SiblingRank::RunningSince(record.activity.inactive_exit.unwrap_or(at))
}
Progress::Finished(Outcome::Failed) => SiblingRank::Failed,
Progress::Finished(Outcome::Failed(_)) => SiblingRank::Failed,
Progress::Finished(Outcome::Abandoned) | Progress::Untouched => {
SiblingRank::Waiting
}
@@ -253,7 +253,7 @@ impl BootState {
#[cfg(test)]
mod tests {
use super::*;
use crate::demo_script::boot_test_fixtures::{
use crate::boot_test_fixtures::{
nested_boot_sample, queued_boot, tree_outline, waiting, waiting_rows,
};
File renamed without changes.
+65 -7
View File
@@ -1,3 +1,4 @@
mod failure_panel;
#[cfg(test)]
mod frame_cost_bench;
#[cfg(test)]
@@ -27,6 +28,7 @@ use unicode_width::UnicodeWidthStr;
pub use self::screen_chrome::PasswordPrompt;
use self::{
failure_panel::FailurePanel,
run_timeline::{RunningRun, Timeline},
screen_chrome::{FOOTER_HEIGHT, Footer, Header, PasswordPopup},
stage_summary::Stages,
@@ -46,6 +48,9 @@ const MIN_BAR_WIDTH: u16 = 12;
const MIN_HEIGHT_FOR_TIMELINE: u16 = 12;
const MIN_TIMELINE_HEIGHT: u16 = run_timeline::CHROME_HEIGHT + 4;
const HEADER_HEIGHT: u16 = 1;
const MIN_FAILURE_PANEL_HEIGHT: u16 = failure_panel::RULE_HEIGHT + 1;
const RESERVED_FAILURE_PANEL_HEIGHT: u16 = failure_panel::RULE_HEIGHT + 3;
const MIN_TREE_HEIGHT_BESIDE_FAILURES: u16 = 4;
const BRANCH: &str = "├─ ";
const LAST_BRANCH: &str = "└─ ";
const PASSING_TRUNK: &str = "│ ";
@@ -75,6 +80,20 @@ pub struct BootView<'a> {
}
impl BootView<'_> {
fn failure_panel(&self) -> FailurePanel<'_> {
FailurePanel {
newest_first: self
.state
.failed
.iter()
.rev()
.filter(|failed| failed.kind.has_own_cause())
.collect(),
colors: &self.colors.failures,
glyphs: self.glyphs,
}
}
fn timeline<'a>(&'a self, rows: &'a [TreeRow]) -> Timeline<'a> {
let mut running: Vec<RunningRun> = rows
.iter()
@@ -135,6 +154,7 @@ impl Widget for BootView<'_> {
glyphs: self.glyphs,
};
let timeline = self.timeline(&all_rows);
let failure_panel = self.failure_panel();
let footer = Footer {
tally: self.state.tally(),
colors: &self.colors.footer,
@@ -148,6 +168,7 @@ impl Widget for BootView<'_> {
let heights = Heights::share(
free,
UnitTree::height_of(pending_count, 0),
failure_panel.height(),
if fits_timeline { timeline.height() } else { 0 },
);
let budget = usize::from(heights.tree.saturating_sub(1));
@@ -168,6 +189,7 @@ impl Widget for BootView<'_> {
stage_area,
tree_area,
_,
failure_area,
timeline_area,
footer_area,
] = Layout::vertical([
@@ -175,6 +197,7 @@ impl Widget for BootView<'_> {
Constraint::Length(stages.height()),
Constraint::Length(tree_height),
Constraint::Fill(1),
Constraint::Length(heights.failures),
Constraint::Length(heights.timeline),
Constraint::Length(FOOTER_HEIGHT),
])
@@ -183,6 +206,9 @@ impl Widget for BootView<'_> {
header.render(header_area, buf);
stages.render(stage_area, buf);
tree.render(tree_area, buf);
if heights.failures > 0 {
failure_panel.render(failure_area, buf);
}
if heights.timeline > 0 {
timeline.render(timeline_area, buf);
}
@@ -200,19 +226,32 @@ impl Widget for BootView<'_> {
struct Heights {
tree: u16,
failures: u16,
timeline: u16,
}
impl Heights {
fn share(free: u16, tree_need: u16, timeline_need: u16) -> Self {
let leftover = free.saturating_sub(tree_need);
fn share(free: u16, tree_need: u16, failures_need: u16, timeline_need: u16) -> Self {
let reserved = RESERVED_FAILURE_PANEL_HEIGHT
.min(failures_need)
.min(free.saturating_sub(MIN_TREE_HEIGHT_BESIDE_FAILURES));
let failures = failures_need
.min(free.saturating_sub(tree_need))
.max(reserved);
let failures = if failures >= MIN_FAILURE_PANEL_HEIGHT {
failures
} else {
0
};
let leftover = free.saturating_sub(tree_need).saturating_sub(failures);
let timeline = if leftover >= MIN_TIMELINE_HEIGHT {
timeline_need.min(leftover)
} else {
0
};
Self {
tree: free - timeline,
tree: free - failures - timeline,
failures,
timeline,
}
}
@@ -264,13 +303,32 @@ mod tests {
#[test]
fn the_tree_takes_the_room_and_the_timeline_only_what_is_left() {
let heights = Heights::share(30, 40, 50);
let heights = Heights::share(30, 40, 0, 50);
assert_eq!((heights.tree, heights.timeline), (30, 0));
let heights = Heights::share(30, 12, 50);
let heights = Heights::share(30, 12, 0, 50);
assert_eq!((heights.tree, heights.timeline), (12, 18));
let heights = Heights::share(30, 25, 50);
let heights = Heights::share(30, 25, 0, 50);
assert_eq!((heights.tree, heights.timeline), (30, 0));
let heights = Heights::share(30, 12, 8);
let heights = Heights::share(30, 12, 0, 8);
assert_eq!((heights.tree, heights.timeline), (22, 8));
}
#[test]
fn failures_come_before_the_timeline_and_keep_a_few_rows_beside_a_full_tree() {
let heights = Heights::share(30, 12, 6, 50);
assert_eq!(
(heights.tree, heights.failures, heights.timeline),
(12, 6, 12)
);
let heights = Heights::share(30, 40, 9, 50);
assert_eq!(
(heights.tree, heights.failures, heights.timeline),
(26, 4, 0)
);
let heights = Heights::share(5, 40, 9, 50);
assert_eq!(
(heights.tree, heights.failures, heights.timeline),
(5, 0, 0)
);
}
}
+227
View File
@@ -0,0 +1,227 @@
use std::iter::once;
use ratatui::{
buffer::Buffer,
layout::Rect,
style::Stylize,
text::{Line, Span},
widgets::{Paragraph, Widget},
};
use unicode_width::UnicodeWidthStr;
use super::{fit_width, rule};
use crate::{
boot_model::{FailedUnit, FailureExplanation},
screen_settings::{color_scheme::FailureColors, font_glyphs::Glyphs},
};
pub const RULE_HEIGHT: u16 = 1;
const LOG_INDENT: &str = " ";
const REASON_GAP: &str = " ";
pub struct FailurePanel<'a> {
pub newest_first: Vec<&'a FailedUnit>,
pub colors: &'a FailureColors,
pub glyphs: &'a Glyphs,
}
fn log_lines(failed: &FailedUnit) -> &[String] {
match &failed.explanation {
FailureExplanation::Explained(cause) => &cause.log_lines,
FailureExplanation::Unexplained => &[],
}
}
impl FailurePanel<'_> {
pub fn height(&self) -> u16 {
if self.newest_first.is_empty() {
return 0;
}
let rows: usize = self
.newest_first
.iter()
.map(|failed| 1 + log_lines(failed).len())
.sum();
RULE_HEIGHT.saturating_add(u16::try_from(rows).unwrap_or(u16::MAX))
}
fn headline(&self, failed: &FailedUnit, width: usize) -> Line<'static> {
let marker = format!("{} ", self.glyphs.failed);
let label_room = width.saturating_sub(marker.width());
let label = fit_width(&failed.label, label_room, self.glyphs.ellipsis);
let reason_room = label_room.saturating_sub(label.width() + REASON_GAP.width());
let reason = match &failed.explanation {
FailureExplanation::Explained(cause) if reason_room > 0 => {
fit_width(&cause.reason, reason_room, self.glyphs.ellipsis)
}
_ => String::new(),
};
let reason_span =
(!reason.is_empty()).then(|| format!("{REASON_GAP}{reason}").fg(self.colors.reason));
Line::from(
[
marker.fg(self.colors.unit),
label.fg(self.colors.unit).bold(),
]
.into_iter()
.chain(reason_span)
.collect::<Vec<Span>>(),
)
}
fn block(&self, failed: &FailedUnit, width: usize) -> Vec<Line<'static>> {
let log_room = width.saturating_sub(LOG_INDENT.width());
once(self.headline(failed, width))
.chain(log_lines(failed).iter().map(|line| {
format!(
"{LOG_INDENT}{}",
fit_width(line, log_room, self.glyphs.ellipsis)
)
.fg(self.colors.log)
.into()
}))
.collect()
}
fn lines_within(&self, rows: usize, width: usize) -> Vec<Line<'static>> {
if self.newest_first.len() > rows {
return self.headlines_with_overflow(rows, width);
}
let mut spare_rows = rows - self.newest_first.len();
self.newest_first
.iter()
.flat_map(|failed| {
let mut block = self.block(failed, width);
let kept_log_lines = (block.len() - 1).min(spare_rows);
spare_rows -= kept_log_lines;
block.truncate(1 + kept_log_lines);
block
})
.collect()
}
fn headlines_with_overflow(&self, rows: usize, width: usize) -> Vec<Line<'static>> {
let Some(headline_rows) = rows.checked_sub(1) else {
return Vec::new();
};
let hidden = self.newest_first.len() - headline_rows;
self.newest_first
.iter()
.take(headline_rows)
.map(|failed| self.headline(failed, width))
.chain(once(
Line::from(format!("{} {hidden} more failed", self.glyphs.ellipsis))
.fg(self.colors.overflow),
))
.collect()
}
}
impl Widget for FailurePanel<'_> {
fn render(self, area: Rect, buf: &mut Buffer) {
let block = rule(
self.colors.rule,
Line::from(" Failures ".fg(self.colors.title)),
);
let inner = block.inner(area);
block.render(area, buf);
Paragraph::new(self.lines_within(usize::from(inner.height), usize::from(inner.width)))
.render(inner, buf);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::boot_model::{FailureCause, FailureKind};
fn explained(label: &str, log_lines: &[&str]) -> FailedUnit {
FailedUnit {
unit: format!("{label}.service"),
label: label.to_owned(),
kind: FailureKind::Broke,
explanation: FailureExplanation::Explained(FailureCause {
reason: "exited with status 1".to_owned(),
log_lines: log_lines.iter().map(ToString::to_string).collect(),
}),
}
}
fn panel_text(failures: &[FailedUnit], rows: usize) -> Vec<String> {
FailurePanel {
newest_first: failures.iter().collect(),
colors: &FailureColors::default(),
glyphs: &Glyphs::fitting(|_| true),
}
.lines_within(rows, 60)
.iter()
.map(ToString::to_string)
.collect()
}
#[test]
fn the_newest_failure_keeps_its_lines_and_older_ones_shrink_to_a_headline() {
let failures = [
explained("web", &["bind: address in use", "giving up"]),
explained("db", &["disk full"]),
explained("cache", &["no memory"]),
];
assert_eq!(
panel_text(&failures, 5),
[
"× web exited with status 1",
" bind: address in use",
" giving up",
"× db exited with status 1",
"× cache exited with status 1",
]
);
}
#[test]
fn room_left_by_the_newest_failure_goes_to_older_ones() {
let failures = [
explained("web", &["giving up"]),
explained("db", &["disk full", "read-only file system"]),
explained("cache", &["no memory"]),
];
assert_eq!(
panel_text(&failures, 6),
[
"× web exited with status 1",
" giving up",
"× db exited with status 1",
" disk full",
" read-only file system",
"× cache exited with status 1",
]
);
}
#[test]
fn failures_beyond_the_room_are_counted() {
let failures = [
explained("web", &["giving up"]),
explained("db", &[]),
explained("cache", &[]),
explained("queue", &[]),
];
assert_eq!(
panel_text(&failures, 3),
[
"× web exited with status 1",
"× db exited with status 1",
"… 2 more failed"
]
);
}
#[test]
fn an_unexplained_failure_shows_only_its_name() {
let failures = [FailedUnit {
explanation: FailureExplanation::Unexplained,
..explained("web", &[])
}];
assert_eq!(panel_text(&failures, 4), ["× web"]);
}
}
@@ -9,7 +9,7 @@ use ratatui::{Terminal, backend::TestBackend};
use super::{BootProjection, BootView, project_boot, screen_render_fixtures::ryzen_origin};
use crate::{
boot_model::{BootState, JobResult, Phase, flatten_forest},
demo_script::boot_test_fixtures::{MS, QueuedUnit, queued_boot, running, waiting},
boot_test_fixtures::{MS, QueuedUnit, queued_boot, running, waiting},
screen_settings::{color_scheme::Colors, config_file::Timing, font_glyphs::Glyphs},
};
@@ -5,8 +5,8 @@ use super::{
screen_render_fixtures::{boot_in_progress, render, render_prompted, screen_lines},
};
use crate::{
boot_model::{ActiveState, Activity, BootState, HANDOFF_UNIT, JobResult, Phase},
demo_script::boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting},
boot_model::{ActiveState, Activity, BootState, FailureCause, HANDOFF_UNIT, JobResult, Phase},
boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting},
screen_settings::color_scheme::Colors,
};
@@ -391,3 +391,82 @@ fn the_timeline_keeps_only_the_slowest_finished_runs() {
"{screen}"
);
}
fn explained_nvidia_failure() -> BootState {
let mut state = boot_in_progress();
state.explain_failure(
"nvidia-powerd.service",
FailureCause {
reason: "exited with status 1".to_owned(),
log_lines: vec![
"ERROR! Failed to get system power limits".to_owned(),
"giving up".to_owned(),
],
},
);
state
}
#[test]
fn failures_show_their_reason_and_journal_lines_above_the_timeline() {
let lines = render(&explained_nvidia_failure(), 12_400 * MS, 110, 30);
let screen = lines.join("\n");
let failures = lines
.iter()
.position(|line| line.contains(" Failures "))
.expect("failure panel");
let timeline = lines
.iter()
.position(|line| line.contains(" Timeline "))
.expect("timeline panel");
assert!(failures < timeline, "{screen}");
assert_eq!(
lines[failures + 1],
"× nvidia-powerd exited with status 1",
"{screen}"
);
assert_eq!(
lines[failures + 2],
" ERROR! Failed to get system power limits",
"{screen}"
);
assert_eq!(lines[failures + 3], " giving up", "{screen}");
}
#[test]
fn failure_lines_are_cut_to_a_narrow_screen() {
let lines = render(&explained_nvidia_failure(), 12_400 * MS, 30, 30);
let screen = lines.join("\n");
let failures = lines
.iter()
.position(|line| line.contains(" Failures "))
.expect("failure panel");
assert_eq!(
lines[failures + 1],
"× nvidia-powerd exited with …",
"{screen}"
);
assert_eq!(
lines[failures + 2],
" ERROR! Failed to get syst…",
"{screen}"
);
}
#[test]
fn dependency_failures_stay_out_of_the_failure_panel() {
let mut state = queued_boot(
Phase::Host,
1000 * MS,
[
waiting("multi-user.target", &["web.service"]),
waiting("web.service", &[]),
],
);
state.finish_job(2, JobResult::Dependency, 2000 * MS);
let lines = render(&state, 3000 * MS, 110, 24);
let screen = lines.join("\n");
assert!(!screen.contains(" Failures "), "{screen}");
assert!(screen.contains("[×] web"), "{screen}");
assert!(screen.contains("× 1 failed"), "{screen}");
}
File renamed without changes.
File renamed without changes.
@@ -5,7 +5,7 @@ use ratatui::{Terminal, backend::TestBackend, buffer::Buffer};
use super::{BootView, PasswordPrompt, project_boot};
use crate::{
boot_model::{BootState, JobResult, Phase, boot_origin::BootOrigin},
demo_script::boot_test_fixtures::{MS, queued_boot, running, waiting},
boot_test_fixtures::{MS, queued_boot, running, waiting},
screen_settings::{color_scheme::Colors, config_file::Timing, font_glyphs::Glyphs},
};
File renamed without changes.
@@ -34,7 +34,7 @@ impl TreeNode {
fn leads_to_running(&self) -> bool {
matches!(
self.row.status,
RowStatus::Running { .. } | RowStatus::Failed
RowStatus::Running { .. } | RowStatus::Failed(_)
) || self.children.iter().any(Self::leads_to_running)
}
@@ -95,7 +95,7 @@ mod tests {
use super::*;
use crate::{
boot_model::{BootState, Phase},
demo_script::boot_test_fixtures::{
boot_test_fixtures::{
activating, nested_boot_sample, queued_boot, running, tree_outline, waiting,
},
};
@@ -5,7 +5,7 @@ use unicode_width::UnicodeWidthStr;
use super::screen_render_fixtures::{boot_in_progress, chain_boot, described_boot, render};
use crate::{
boot_model::{BootState, JobResult, Phase, UnitTimeouts},
demo_script::boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting},
boot_test_fixtures::{MS, queue_unit, queued_boot, running, waiting},
};
#[test]
@@ -166,7 +166,10 @@ impl UnitTree<'_> {
fn unit_label_style(&self, row: &TreeRow) -> Style {
match row.status {
RowStatus::Failed => Style::from(self.colors.failed_unit).bold(),
RowStatus::Failed(kind) if kind.has_own_cause() => {
Style::from(self.colors.failed_unit).bold()
}
RowStatus::Failed(_) => Style::from(self.colors.failed_unit).dim(),
_ if row.kind == UnitKind::Target => self.colors.target.into(),
RowStatus::Running { .. } => self.colors.running_unit.into(),
RowStatus::Waiting => self.colors.waiting_unit.into(),
@@ -187,7 +190,9 @@ impl UnitTree<'_> {
fn unit_marker(&self, row: &TreeRow) -> Span<'static> {
match row.status {
RowStatus::Failed => format!("[{}] ", self.glyphs.failed).fg(self.colors.failed_unit),
RowStatus::Failed(_) => {
format!("[{}] ", self.glyphs.failed).fg(self.colors.failed_unit)
}
_ if row.kind == UnitKind::Target => Span::default(),
RowStatus::Running { .. } => {
format!("[{}] ", spinner_at(self.now)).fg(self.colors.running_marker)
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -273,7 +273,7 @@ mod tests {
use super::*;
use crate::{
boot_model::{ActiveState, Activity, HANDOFF_UNIT},
demo_script::boot_test_fixtures::{MS, nested_boot_sample, queue_unit, waiting},
boot_test_fixtures::{MS, nested_boot_sample, queue_unit, waiting},
};
#[derive(Default)]
+2 -1
View File
@@ -1,7 +1,8 @@
pub mod boot_model;
#[cfg(test)]
mod boot_test_fixtures;
pub mod boot_view;
pub mod console_session;
pub mod demo_script;
pub mod screen_settings;
pub mod system_install;
pub mod systemd_link;
File renamed without changes.
File renamed without changes.
@@ -8,6 +8,7 @@ pub struct Colors {
pub stages: StageColors,
pub tree: TreeColors,
pub timeline: TimelineColors,
pub failures: FailureColors,
pub footer: FooterColors,
pub password: PasswordColors,
}
@@ -122,6 +123,30 @@ impl Default for TimelineColors {
}
}
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[serde(default, deny_unknown_fields, rename_all = "kebab-case")]
pub struct FailureColors {
pub rule: Color,
pub title: Color,
pub unit: Color,
pub reason: Color,
pub log: Color,
pub overflow: Color,
}
impl Default for FailureColors {
fn default() -> Self {
Self {
rule: Color::DarkGray,
title: Color::LightRed,
unit: Color::LightRed,
reason: Color::White,
log: Color::Gray,
overflow: Color::DarkGray,
}
}
}
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[serde(default, deny_unknown_fields, rename_all = "kebab-case")]
pub struct FooterColors {
File renamed without changes.
@@ -1,4 +1,5 @@
pub mod boot_feed;
mod journal_tail;
pub mod link_error;
pub mod monotonic_clock;
pub mod switch_root;
@@ -1,5 +1,5 @@
use std::{
collections::HashMap,
collections::{HashMap, HashSet},
iter::once,
process,
time::{Duration, Instant},
@@ -10,12 +10,16 @@ use futures::{Stream, StreamExt, stream};
use tokio::time::sleep;
use crate::{
boot_model::{Activity, BootChange, BootState, HANDOFF_UNIT, JobKind, Phase, UnitFacts},
boot_model::{
ActiveState, Activity, BootChange, BootState, FailureCause, HANDOFF_UNIT, JobKind,
JobResult, Phase, UnitFacts,
},
console_session::{
stream_sources::{BACKGROUND_CHECK_INTERVAL, CarriedItem, unfold_carrying},
watch_screen::ScreenInput,
},
systemd_link::{
journal_tail::recent_unit_messages,
link_error::Error,
monotonic_clock::monotonic_now,
switch_root::Sysroot,
@@ -27,6 +31,12 @@ use crate::{
};
const RECONNECT_PATIENCE: Duration = Duration::from_secs(60);
const JOURNAL_SETTLING_WAITS: [Duration; 3] = [
Duration::from_millis(200),
Duration::from_millis(300),
Duration::from_millis(500),
];
const LOG_LINES_KEPT: usize = 6;
const SWITCH_ROOT_UNIT: &str = "initrd-switch-root.service";
const SHUTDOWN_ROOTS: [&str; 5] = [
"poweroff.target",
@@ -108,6 +118,8 @@ struct BootTracker {
target: RootRequest,
phase: Phase,
unit_paths: HashMap<Path<'static>, String>,
explained_failures: HashSet<String>,
units_in_failed_state: HashSet<String>,
root_switch: RootSwitch,
}
@@ -131,6 +143,8 @@ impl BusFeed {
target,
phase,
unit_paths: HashMap::new(),
explained_failures: HashSet::new(),
units_in_failed_state: HashSet::new(),
root_switch: RootSwitch::NotQueued,
},
}
@@ -196,6 +210,8 @@ impl BusFeed {
}
tracker.root_switch = RootSwitch::NotQueued;
tracker.unit_paths.clear();
tracker.explained_failures.clear();
tracker.units_in_failed_state.clear();
tracker.phase = Phase::Host;
self.link = Link::connecting(ConnectAttempt::Again);
vec![ScreenInput::Boot(vec![BootChange::SwitchRoot { at }])]
@@ -395,20 +411,68 @@ impl BootTracker {
Signal::JobNew { id, job, unit } => {
self.queue_new_job(connection, id, &job, unit, at).await
}
Signal::JobRemoved { id, result } => vec![BootChange::FinishJob { id, result, at }],
Signal::UnitChanged { path, activity } => self
.unit_paths
.get(&path)
.map(|unit| BootChange::UpdateActivity {
unit: unit.clone(),
activity,
})
.into_iter()
.collect(),
Signal::JobRemoved { id, unit, result } => {
let explanation = match result {
JobResult::Failed => self.explain_failure(connection, unit, "failed").await,
JobResult::Timeout => self.explain_failure(connection, unit, "timed out").await,
_ => None,
};
once(BootChange::FinishJob { id, result, at })
.chain(explanation)
.collect()
}
Signal::UnitChanged { path, activity } => {
let Some(unit) = self.unit_paths.get(&path).cloned() else {
return Vec::new();
};
let explanation = if self.enters_failed_state(&unit, activity) {
self.explain_failure(connection, unit.clone(), "failed")
.await
} else {
None
};
once(BootChange::UpdateActivity { unit, activity })
.chain(explanation)
.collect()
}
Signal::BootFinished => self.boot_finished(),
}
}
fn enters_failed_state(&mut self, unit: &str, activity: Activity) -> bool {
match activity.state {
Some(ActiveState::Failed) => self.units_in_failed_state.insert(unit.to_owned()),
Some(_) => {
self.units_in_failed_state.remove(unit);
false
}
None => false,
}
}
async fn explain_failure(
&mut self,
connection: &mut BusConnection,
unit: String,
fallback_reason: &str,
) -> Option<BootChange> {
if !self.unit_paths.values().any(|known| *known == unit) {
return None;
}
self.explained_failures.insert(unit.clone());
let reason = connection
.failure_reason(&unit)
.await
.ok()
.flatten()
.unwrap_or_else(|| fallback_reason.to_owned());
let log_lines = settled_unit_messages(&unit).await;
Some(BootChange::ExplainFailure {
unit,
cause: FailureCause { reason, log_lines },
})
}
fn boot_finished(&self) -> Vec<BootChange> {
(self.phase == Phase::Host)
.then_some(BootChange::BootFinished)
@@ -440,6 +504,17 @@ impl BootTracker {
}
}
async fn settled_unit_messages(unit: &str) -> Vec<String> {
for settling in JOURNAL_SETTLING_WAITS {
sleep(settling).await;
let log_lines = recent_unit_messages(unit.to_owned(), LOG_LINES_KEPT).await;
if !log_lines.is_empty() {
return log_lines;
}
}
Vec::new()
}
pub fn bus_inputs(
phase: Phase,
scope: Scope,
@@ -454,3 +529,38 @@ pub fn bus_inputs(
})
.flat_map(stream::iter)
}
#[cfg(test)]
mod tests {
use super::*;
fn host_tracker() -> BootTracker {
BootTracker {
scope: Scope::System,
target: RootRequest::ManagerDefault,
phase: Phase::Host,
unit_paths: HashMap::new(),
explained_failures: HashSet::new(),
units_in_failed_state: HashSet::new(),
root_switch: RootSwitch::NotQueued,
}
}
fn in_state(state: ActiveState) -> Activity {
Activity {
state: Some(state),
..Activity::default()
}
}
#[test]
fn only_entering_the_failed_state_asks_for_a_fresh_explanation() {
let mut tracker = host_tracker();
let unit = "http-listener.service";
assert!(tracker.enters_failed_state(unit, in_state(ActiveState::Failed)));
assert!(!tracker.enters_failed_state(unit, in_state(ActiveState::Failed)));
assert!(!tracker.enters_failed_state(unit, Activity::default()));
assert!(!tracker.enters_failed_state(unit, in_state(ActiveState::Activating)));
assert!(tracker.enters_failed_state(unit, in_state(ActiveState::Failed)));
}
}
+132
View File
@@ -0,0 +1,132 @@
use std::{io, str::Chars};
use systemd::{
id128::Id128,
journal::{Journal, OpenOptions},
};
use tokio::task::spawn_blocking;
const UNIT_FIELDS: [&str; 4] = ["_SYSTEMD_UNIT", "UNIT", "_SYSTEMD_USER_UNIT", "USER_UNIT"];
const MANAGER_SUMMARY_MESSAGE_IDS: [&str; 10] = [
"7d4958e842da4a758f6c1cdc7b36dcc5",
"39f53479d3a045ac8e11786248231fbf",
"7ad2d189f7e94e70a38c781354912448",
"5eb03494b6584870a536b337290809b3",
"be02cf6855d2428ba40df7e9d022f03d",
"de5b426a63be47a7b6ac3eaac82e2f6f",
"9d1aaa27d60140bd96365438aad20286",
"ae8f7b866b0347b9af31fe1c80b127c0",
"d9b373ed55a64feb8242e02dbe79a49c",
"98e322203f7a4ed290d09fe03c09fe15",
];
const ENTRIES_SCANNED: usize = 200;
const ESCAPE: char = '\u{1b}';
pub async fn recent_unit_messages(unit: String, wanted: usize) -> Vec<String> {
spawn_blocking(move || {
read_recent_messages(&unit, wanted).unwrap_or_else(|error| {
eprintln!("bootvine: reading the journal of {unit}: {error}");
Vec::new()
})
})
.await
.unwrap_or_default()
}
fn read_recent_messages(unit: &str, wanted: usize) -> io::Result<Vec<String>> {
let mut journal = OpenOptions::default().open()?;
match_unit_in_this_boot(&mut journal, unit)?;
journal.seek_tail()?;
let mut newest_first = Vec::new();
let mut scanned = 0;
while newest_first.len() < wanted
&& scanned < ENTRIES_SCANNED
&& journal.previous().is_ok_and(|moved| moved > 0)
{
scanned += 1;
if is_manager_summary(&mut journal).unwrap_or(true) {
continue;
}
if let Ok(Some(message)) = entry_message(&mut journal) {
newest_first.push(message);
}
}
newest_first.reverse();
Ok(newest_first)
}
fn match_unit_in_this_boot(journal: &mut Journal, unit: &str) -> io::Result<()> {
UNIT_FIELDS.iter().try_for_each(|field| {
journal.match_add(field, unit)?.match_or()?;
Ok::<_, io::Error>(())
})?;
journal
.match_and()?
.match_add("_BOOT_ID", Id128::from_boot()?.to_string())?;
Ok(())
}
fn field_text(journal: &mut Journal, field: &str) -> io::Result<Option<String>> {
Ok(journal.get_data(field)?.and_then(|entry_field| {
entry_field
.value()
.map(|value| String::from_utf8_lossy(value).into_owned())
}))
}
fn is_manager_summary(journal: &mut Journal) -> io::Result<bool> {
Ok(field_text(journal, "MESSAGE_ID")?
.is_some_and(|message_id| MANAGER_SUMMARY_MESSAGE_IDS.contains(&message_id.as_str())))
}
fn entry_message(journal: &mut Journal) -> io::Result<Option<String>> {
Ok(field_text(journal, "MESSAGE")?
.map(|message| printable_line(&message))
.filter(|line| !line.is_empty()))
}
fn printable_line(message: &str) -> String {
let mut characters = message.lines().next().unwrap_or_default().chars();
let mut printable = String::new();
while let Some(character) = characters.next() {
if character == ESCAPE {
skip_escape_sequence(&mut characters);
} else if !character.is_control() {
printable.push(character);
}
}
printable.trim_end().to_owned()
}
fn skip_escape_sequence(characters: &mut Chars<'_>) {
if characters.clone().next() == Some('[') {
characters.next();
characters.find(|character| ('@'..='~').contains(character));
}
}
#[cfg(test)]
mod tests {
use std::env;
use super::*;
#[test]
#[ignore = "reads this machine's journal: cargo test journal_of_this_boot -- --ignored \
--nocapture"]
fn journal_of_this_boot_has_lines_for_journald() {
let unit = env::var("BOOTVINE_JOURNAL_UNIT")
.unwrap_or_else(|_| "systemd-journald.service".to_owned());
let lines = read_recent_messages(&unit, 6).expect("readable journal");
lines.iter().for_each(|line| println!("{line}"));
assert!(!lines.is_empty());
}
#[test]
fn printable_line_keeps_the_first_line_without_colours() {
assert_eq!(
printable_line("\u{1b}[31mno such file\u{1b}[0m\nsecond line"),
"no such file"
);
}
}
File renamed without changes.
File renamed without changes.
@@ -75,6 +75,7 @@ pub enum Signal {
},
JobRemoved {
id: u32,
unit: String,
result: JobResult,
},
UnitChanged {
@@ -342,6 +343,18 @@ impl BusConnection {
})
}
pub async fn failure_reason(&mut self, name: &str) -> Result<Option<String>, Error> {
let Some(interface) = kind_interface(UnitKind::of(name)) else {
return Ok(None);
};
let path: Path<'static> = self
.call(manager_call("GetUnit", (name,))?)
.await?
.read1()?;
let properties = self.all_properties(&path, interface).await?;
Ok(failure_reason_from(&properties))
}
async fn all_properties(&mut self, path: &Path<'_>, interface: &str) -> Result<PropMap, Error> {
Ok(self
.call(method_call(
@@ -376,6 +389,52 @@ fn text_property(properties: &PropMap, property: &str) -> Option<String> {
.map(ToOwned::to_owned)
}
fn signal_name(number: i64) -> String {
let name = match number {
1 => "SIGHUP",
2 => "SIGINT",
3 => "SIGQUIT",
4 => "SIGILL",
5 => "SIGTRAP",
6 => "SIGABRT",
7 => "SIGBUS",
8 => "SIGFPE",
9 => "SIGKILL",
11 => "SIGSEGV",
13 => "SIGPIPE",
14 => "SIGALRM",
15 => "SIGTERM",
other => return format!("signal {other}"),
};
name.to_owned()
}
fn failure_reason_from(properties: &PropMap) -> Option<String> {
let result = text_property(properties, "Result").filter(|result| result != "success")?;
let status = properties
.get("ExecMainStatus")
.and_then(|value| value.0.as_i64());
let reason = match (result.as_str(), status) {
("exit-code", Some(status)) if status != 0 => format!("exited with status {status}"),
("exit-code", _) => "exited with an error".to_owned(),
("signal", Some(signal)) => format!("killed by {}", signal_name(signal)),
("core-dump", Some(signal)) => format!("dumped core on {}", signal_name(signal)),
("timeout", _) => "timed out".to_owned(),
("watchdog", _) => "stopped answering the watchdog".to_owned(),
("start-limit-hit", _) => "restarted too often".to_owned(),
("oom-kill", _) => "killed for running out of memory".to_owned(),
("resources", _) => "could not set up its resources".to_owned(),
("protocol", _) => "broke the start-up protocol".to_owned(),
(other, _) => other.to_owned(),
};
Some(
match text_property(properties, "StatusText").filter(|text| !text.is_empty()) {
Some(status_text) => format!("{reason}: {status_text}"),
None => reason,
},
)
}
fn unit_id(name: &str, properties: &PropMap) -> String {
text_property(properties, "Id").unwrap_or_else(|| name.to_owned())
}
@@ -474,10 +533,11 @@ fn parse_job_signal(message: &Message, member: &str) -> Option<Signal> {
Some(Signal::JobNew { id, job, unit })
}
"JobRemoved" => {
let (id, _job, _unit, result): (u32, Path<'static>, String, String) =
let (id, _job, unit, result): (u32, Path<'static>, String, String) =
message.read4().ok()?;
Some(Signal::JobRemoved {
id,
unit,
result: serde_plain::from_str(&result).unwrap_or(JobResult::Failed),
})
}
@@ -567,6 +627,46 @@ mod tests {
);
}
fn reason_properties(result: &str, status: i32, status_text: &str) -> PropMap {
[
("Result", Box::new(result.to_owned()) as Box<dyn RefArg>),
("ExecMainStatus", Box::new(status)),
("StatusText", Box::new(status_text.to_owned())),
]
.into_iter()
.map(|(name, value)| (name.to_owned(), Variant(value)))
.collect()
}
#[test]
fn exit_codes_and_signals_become_a_reason() {
assert_eq!(
failure_reason_from(&reason_properties("exit-code", 1, "")).as_deref(),
Some("exited with status 1")
);
assert_eq!(
failure_reason_from(&reason_properties("signal", 11, "")).as_deref(),
Some("killed by SIGSEGV")
);
}
#[test]
fn status_text_follows_the_reason() {
assert_eq!(
failure_reason_from(&reason_properties("timeout", 0, "waiting for the disk"))
.as_deref(),
Some("timed out: waiting for the disk")
);
}
#[test]
fn success_has_no_reason() {
assert_eq!(
failure_reason_from(&reason_properties("success", 0, "")),
None
);
}
#[test]
fn mounts_use_one_timeout_for_both_directions() {
let mount = limit_properties(&[("TimeoutUSec", Duration::from_secs(20))]);
File renamed without changes.
+44 -1
View File
@@ -11,6 +11,8 @@ while it does. None of this is needed to install or configure it.
PID 1 over systemd's private socket, which
[zbus](https://github.com/dbus2/zbus) cannot use; see
[systemd's private socket](#systemds-private-socket).
- **libsystemd.** It reads the last journal lines of a failed unit; see
[failures](#failures).
- **A virtual terminal.** It draws on `tty12` and switches the console to it.
## Overview
@@ -116,7 +118,7 @@ From then on it follows the signals:
| Signal | Carries | Used for |
| --- | --- | --- |
| `JobNew` | job id, job path, unit name | a row in the tree |
| `JobRemoved` | job id, result | the row finishes as done, failed or skipped |
| `JobRemoved` | job id, unit name, result | the row finishes as done, failed or skipped |
| `PropertiesChanged` on a unit | `ActiveState` and its timestamps | running since, and the timeline |
| `StartupFinished` | | the boot reached its default target |
@@ -151,6 +153,47 @@ systemd's `…TimestampMonotonic` properties count microseconds of
clock, so the times it shows are measured from the kernel start, even for
units that ran before bootvine did.
### Failures
`JobRemoved` says how a job ended: `failed`, `timeout` or `dependency`. Like
systemd's own `[FAILED]`, `[ TIME ]` and `[DEPEND]`, bootvine tells them
apart: a unit that failed only because a dependency did is shown dimmed in the
tree and left out of the failure panel, so the panel points at the unit that
actually broke. A unit whose `ActiveState` turns `failed` without a failed job,
such as a service crashing after it started, counts as failed too.
For a unit that broke or timed out, bootvine collects the cause once:
```mermaid
sequenceDiagram
participant PID1 as systemd (PID 1)
participant Bootvine as bootvine
participant Journal as journal files
PID1-->>Bootvine: ActiveState=failed, or JobRemoved failed/timeout
Bootvine->>PID1: GetUnit, Properties.GetAll on the unit type
PID1->>Bootvine: Result, ExecMainCode, ExecMainStatus, StatusText
Note over Bootvine: wait 200 ms for journald, longer while nothing is there yet
Bootvine->>Journal: sd_journal, matching the unit in this boot
Journal->>Bootvine: last messages, newest last
Note over Bootvine: reason and lines in the failure panel
```
- **The reason** comes from the unit type's `Result` (`exit-code`, `signal`,
`timeout`, `start-limit-hit`, `oom-kill`, …), with the exit status or signal
of the main process, and the service's `StatusText` when it set one. These
are read when the unit fails, since they change at that moment.
- **The journal lines** match the unit the same way `journalctl -u` does
(`_SYSTEMD_UNIT=` or `UNIT=`, in the current boot), without PID 1's own
"Starting…", "Failed with result…" and "Failed to start…" lines, which the
reason already covers. In the initrd they come from `/run/log/journal`. The
journal is opened without `SD_JOURNAL_LOCAL_ONLY`: libsystemd caches the
initrd's machine ID, which would hide the real root's files after the
switch, and the boot ID match already keeps other machines out.
- **libsystemd** is linked rather than calling `journalctl`, since dracut,
mkinitcpio and the NixOS initrd builder copy the libraries a binary links
into the initramfs along with it, while `journalctl` may not be there.
## Surviving the switch to the real root
At the end of the initrd, systemd mounts the real root at `/sysroot`, isolates
+9
View File
@@ -81,6 +81,14 @@ queued-bar = "gray"
running = "yellow"
running-label = "white"
[colors.failures]
rule = "dark-gray"
title = "light-red"
unit = "light-red"
reason = "white"
log = "gray"
overflow = "dark-gray"
[colors.footer]
rule = "dark-gray"
done = "green"
@@ -114,6 +122,7 @@ Each `[colors.*]` table covers one part of the screen:
| `stages` | the finished stages, such as the initrd |
| `tree` | the unit tree |
| `timeline` | the timeline of the slowest units |
| `failures` | the failed units with their reason and last journal lines |
| `footer` | the tallies of done, running, waiting and failed units |
| `password` | the passphrase popup |
+32 -9
View File
@@ -1,5 +1,19 @@
# Development
## Layout
| Folder | Holds |
| --- | --- |
| `bootvine/` | the published crate: the library, the `bootvine` binary and the unit and initrd files it installs |
| `scripted-demos/` | the scripted boot, shutdown and passphrase demos, and the VHS tapes that record them |
| `xtask/` | repository tasks run with `cargo xtask` |
| `vm-tests/` | the sample NixOS machines, one folder each with its VM check |
| `nix/` | the crane builds and the NixOS module |
| `recordings/` | the animations and screenshots the README and the [guide](guide.md) show |
`cargo build` and `cargo test` act on every crate. `cargo run` needs the binary:
`--bin bootvine`, `--bin scripted-boot` and so on.
## Setup
Work inside the dev shell (`direnv allow` once, with
@@ -57,24 +71,32 @@ locally from the dev shell with `release-plz update`.
| What | Command |
| --- | --- |
| Model and render tests | `cargo test --workspace` |
| Scripted demos | `cargo run --example boot` (also `shutdown`, `luks`) |
| Sample machine in a window | `nix run .#sample-machine-vm` |
| Scripted demos | `cargo run --bin scripted-boot` (also `scripted-shutdown`, `scripted-luks-unlock`) |
| Sample machines in a window | `nix run .#sample-vm-<machine>`, see the [guide](guide.md#sample-machines) |
| Boot and shutdown in a VM | `nix build .#checks.x86_64-linux.vm-boot-and-shutdown` |
| Passphrase prompt in a VM | `nix build .#checks.x86_64-linux.vm-luks-passphrase` |
| Failure panel in a VM | `nix build .#checks.x86_64-linux.vm-failures` |
To watch your own user manager start the sample units:
```sh
units=$(nix build --no-link --print-out-paths .#sample-unit-files)
systemctl --user link --runtime $units/*
systemctl --user start --no-block bootvine-demo.target
cargo run -- --user --target bootvine-demo.target --hold 3s
systemctl --user start --no-block bootvine-sample-services.target
cargo run --bin bootvine -- --user --target bootvine-sample-services.target --hold 3s
systemctl --user stop $(ls $units)
systemctl --user disable --runtime $(ls $units)
```
Everything a sample machine needs sits in its own folder under `vm-tests/`:
`machine.nix`, and for the machines with a check, `check.nix` and `check.py`.
`vm-tests/shared/` holds the settings every machine shares, the window setup
for `nix run` and the console helpers for the checks. A new machine is a new
folder plus its name in `vm-tests/default.nix`.
The VM checks read the console as text, hold the boot at fixed points and wait
on marker files instead of sleeping. To step through one by hand, run
on marker files instead of sleeping. The failures check starts each failing
unit by hand, so it knows which one is newest in the panel. To step through one by hand, run
`nix run .#checks.x86_64-linux.vm-boot-and-shutdown.driverInteractive` and
call `test_script()`.
@@ -83,12 +105,13 @@ exports libraries that clash with [QEMU](https://www.qemu.org) or Chromium.
## Repository tasks
- `nix run .#record-screencasts` re-records the README animations from the
scripted demos and copies the passphrase screenshot from the VM check.
- `nix run .#record-screencasts` re-records the animations from the scripted
demos and copies the passphrase and failures screenshots from the VM checks
into `recordings/`, for the README and the [guide](guide.md).
- `cargo xtask sync-default-config` rewrites the defaults block in
[configuration](configuration.md#defaults) from `bootvine defaults`.
- `cargo run -- install --destdir <dir> --initrd dracut` stages an install
without touching the system; `cargo run -- uninstall --destdir <dir>` removes
- `cargo run --bin bootvine -- install --destdir <dir> --initrd dracut` stages an install
without touching the system; `cargo run --bin bootvine -- uninstall --destdir <dir>` removes
it again.
## Code style and design
+157
View File
@@ -0,0 +1,157 @@
# Guide
What bootvine shows on the console, part by part, with the recordings that go
with each. The animations come from the [scripted demos](#scripted-demos); the
screenshots come from the [VM checks](development.md#testing) on real NixOS
machines.
## Reading the screen
<img src="../recordings/boot.gif" width="691" alt="A boot through the initrd into stage 2, with one failed unit">
From top to bottom:
1. **The header**: the host name, the system and its generation on NixOS
(otherwise the `PRETTY_NAME` from `os-release`), and the kernel. Once a
stage has finished, the total time since the kernel started sits on the
right.
2. **The stages**: each finished stage, such as `Initrd`, with how long it
took, how many units finished in it and how many of them failed. The stage
still running is the last line, ending in `Running`.
3. **The tree** of units with a pending job. Each unit is nested under the unit
waiting on it, and the milestone targets (`sysinit.target`, `basic.target`,
`multi-user.target`, `graphical.target`) open the levels:
- `◆` is the stage being worked on, `◇` one still to come, `√` one reached
(`*`, `o` and `+` on console fonts without those glyphs);
- `[|]` with a spinner is running, `[ ]` is waiting, `[√]` finished in the
last couple of seconds (`keep-finished-units-for`);
- `[×]` in bold red broke or timed out; dimmed red failed only because a
unit it needed did;
- the `Running` column counts up while a unit runs, and after 5 seconds the
`Timeout` column shows the limit systemd will stop it at, red once it
is passed;
- a unit waiting on exactly one other, which waits on exactly one other,
folds into one row joined by `─`; mounts and devices show their
description instead of their unit name.
When the tree is taller than the screen, finished and waiting rows give way
first, and a last line counts the units left out.
4. **The failure panel**, only once a unit broke or timed out; see
[failures](#failures).
5. **The timeline**, in the room left below the tree: the milestones reached,
with how many units they took, the slowest finished units (up to five, and
none shorter than `hide-runs-shorter-than`) and the units still running,
as bars against seconds since the kernel started. A `│` marks the switch
from the initrd to the real root.
6. **The footer**: the units done, running, waiting and failed so far.
## Booting through the initrd
The same recording shows a boot from the start. With a systemd initrd,
bootvine starts about a second after the kernel:
- **The initrd stage**: udev settling, the root device and its file systems
under `initrd.target`.
- **The switch to the real root**: the initrd moves into the stages at the
top, with its time and unit count, and the tree starts again under the
default target. The timeline keeps the initrd's runs, left of the `│`.
- **Stage 2** up to `graphical.target`.
- **The handover**: bootvine hands the console back once the default target is
reached, or earlier, once the display manager has switched to its own
terminal.
How bootvine gets in that early and survives the switch is in
[architecture](architecture.md#starting-in-the-initrd).
## Shutdown
<img src="../recordings/shutdown.gif" width="691" alt="A poweroff: units stopping under the targets waiting on them">
On a poweroff, reboot or halt the tree turns around:
- its root is the target being reached, such as `poweroff.target`;
- each unit being stopped sits under the units that have to stop before it,
so the unit holding up the shutdown is at the bottom of its branch;
- `shutdown.target`, `umount.target` and `final.target` open the levels;
- the screen stays until systemd hands over to `systemd-shutdown`.
## Passphrase prompts
<img src="../recordings/passphrase.png" width="640" alt="A LUKS passphrase prompt floating over the initrd tree">
When a unit asks for a passphrase, such as `systemd-cryptsetup@….service`
unlocking a [LUKS](https://gitlab.com/cryptsetup/cryptsetup) volume, the
question floats over a dimmed tree:
- typing shows one mark per character;
- `Enter` sends it, and the popup says it is checking;
- a wrong passphrase brings the prompt back with "Wrong passphrase, try
again";
- once unlocked, the popup closes and the boot carries on.
If another password agent, such as Plymouth's, was already asking when
bootvine started, bootvine waits behind it.
## Failures
<img src="../recordings/failures.png" width="640" alt="The failure panel listing failed units with their reason and last journal lines">
A unit that broke or timed out gets an entry in the failure panel, newest
first:
- **The headline**: the unit and why it failed, such as `exited with status 2`,
`killed by SIGSEGV`, `timed out`, `restarted too often` or
`exited with status 203` (systemd could not run the command). A service that
reported a status with `sd_notify` adds it, as in
`exited with status 1: Cannot reach database`.
- **The journal lines**: up to six of the unit's last messages from this boot,
the same ones `journalctl -u` shows, without systemd's own "Starting…" and
"Failed with result…" lines.
- **Out of room**: the newest failure keeps its lines, older ones shrink to
their headline, and a last line counts the rest.
- **What stays out**: a unit that failed only because one it needed did stays
dimmed in the tree, so the panel points at the unit that actually broke.
The footer counts both.
Failures from the initrd stay in the panel after the switch to the real root.
The screenshot comes from the `failures` sample machine, with every kind of
failure systemd reports.
## Trying it
Neither of these touches your system's boot.
### Scripted demos
Play a scripted boot, poweroff and passphrase prompt in your terminal, with
[Rust](https://www.rust-lang.org/tools/install) installed:
```sh
git clone https://codeberg.org/wvhulle/bootvine.git
cd bootvine
cargo run --bin scripted-boot
cargo run --bin scripted-shutdown
cargo run --bin scripted-luks-unlock
```
`q` quits. The encrypted root unlocks with the passphrase `demo`; `Esc` quits
while it asks. These are the scenarios the animations above were recorded
from.
### Sample machines
Each sample machine boots in a [QEMU](https://www.qemu.org) window with
[Nix](https://nixos.org/download/), and logs in as root at the end. `poweroff`
shows the shutdown.
| Run | Shows |
| --- | --- |
| `nix run .#sample-vm-slow-services` | slow units in the initrd and stage 2, one failing unit, and the handover to a greetd login |
| `nix run .#sample-vm-text-console` | the same units without a display manager, handing over at the default target |
| `nix run .#sample-vm-scripted-initrd` | the same units with NixOS's scripted initrd, so bootvine starts in stage 2 only |
| `nix run .#sample-vm-encrypted-volume` | a LUKS volume unlocked in the initrd; the passphrase is `sample passphrase` |
| `nix run .#sample-vm-failures` | a firmware failure in the initrd, then units that exit with an error, crash, time out, cannot be run, restart too often, report a status, die after starting, and one that fails only through its dependency |
Each machine keeps its disk in `~/.cache/bootvine-<machine>.qcow2`; delete it
to start fresh. The machines live in `vm-tests/`, one folder each, next to the
checks that drive them.
+2 -2
View File
@@ -48,7 +48,7 @@ as well, the initrd has to run systemd.
1. Install the build dependencies and Rust:
```sh
sudo apt install build-essential pkg-config libdbus-1-dev git curl
sudo apt install build-essential pkg-config libdbus-1-dev libsystemd-dev git curl
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
```
@@ -88,7 +88,7 @@ as well, the initrd has to run systemd.
## Other distributions
Install as in the Ubuntu steps, with your distribution's packages for a C
toolchain, `pkg-config` and the libdbus headers.
toolchain, `pkg-config` and the libdbus and libsystemd headers.
For the initrd, use the dracut module as above, or on
[mkinitcpio](https://wiki.archlinux.org/title/Mkinitcpio):
+24 -21
View File
@@ -46,11 +46,11 @@
craneLib
rustToolchain
;
builds = import ./crane-builds.nix { inherit lib pkgs craneLib; };
builds = import ./nix/crane-builds.nix { inherit lib pkgs craneLib; };
}
);
sampleMachine = import ./tests/sample-machine {
vmTests = import ./vm-tests {
inherit lib;
bootvineModule = self.nixosModules.bootvine;
};
@@ -60,19 +60,22 @@
{ pkgs, builds, ... }:
{
default = builds.bootvine;
inherit (builds) bootvine scripted-boot-examples repo-tasks;
sample-unit-files = pkgs.callPackage ./tests/sample-machine/sample-unit-files.nix {
inherit (sampleMachine) sampleUnits;
inherit (builds) bootvine scripted-demos repo-tasks;
sample-unit-files = pkgs.callPackage ./vm-tests/slow-services/sample-unit-files.nix {
inherit (vmTests) sampleUnits;
};
}
);
nixosModules = {
default = self.nixosModules.bootvine;
bootvine = import ./nixos-module.nix { inherit self; };
bootvine = import ./nix/nixos-module.nix { inherit self; };
};
nixosConfigurations.sample-machine = sampleMachine.nixosSystemFor "x86_64-linux";
nixosConfigurations = lib.mapAttrs' (
machineName: _:
lib.nameValuePair "sample-${machineName}" (vmTests.nixosSystemFor "x86_64-linux" machineName)
) vmTests.machines;
apps = forEachSystem (
{
@@ -83,6 +86,11 @@
...
}:
let
runSampleMachine = machineName: description: {
type = "app";
meta.description = "Boot the ${machineName} sample machine in QEMU: ${description}";
program = lib.getExe (vmTests.nixosSystemFor system machineName).config.system.build.vm;
};
runReleasePlzOnCodeberg = subcommand: description: {
type = "app";
meta.description = description;
@@ -107,25 +115,25 @@
{
release-plz-release = runReleasePlzOnCodeberg "release" "Publish unpublished versions to crates.io and tag them with a Codeberg release";
release-plz-release-pr = runReleasePlzOnCodeberg "release-pr" "Open or update the Codeberg pull request that bumps the version and changelog";
sample-machine-vm = {
type = "app";
meta.description = "Boot a sample machine with bootvine and slow sample units in QEMU";
program = lib.getExe (sampleMachine.nixosSystemFor system).config.system.build.vm;
};
sample-vm-slow-services = runSampleMachine "slow-services" "slow and failing units, a held initrd probe and a greetd login";
sample-vm-text-console = runSampleMachine "text-console" "the slow units without a display manager, handing over at the default target";
sample-vm-scripted-initrd = runSampleMachine "scripted-initrd" "the slow units with the scripted initrd, so bootvine starts in stage 2";
sample-vm-encrypted-volume = runSampleMachine "encrypted-volume" "a LUKS volume unlocked in the initrd with the passphrase 'sample passphrase'";
sample-vm-failures = runSampleMachine "failures" "units that fail in every way systemd reports, each with journal messages";
record-screencasts = {
type = "app";
meta.description = "Re-record the README screencasts and passphrase screenshot into recordings/";
meta.description = "Re-record the screencasts and the passphrase and failures screenshots into recordings/";
program = lib.getExe (
pkgs.writeShellApplication {
name = "record-screencasts";
runtimeInputs = [
pkgs.vhs
builds.scripted-boot-examples
builds.scripted-demos
builds.repo-tasks
];
text = "exec xtask record-screencasts --passphrase-png ${
self.checks.${system}.vm-luks-passphrase
}/passphrase.png";
}/passphrase.png --failures-png ${self.checks.${system}.vm-failures}/failures.png";
}
);
};
@@ -163,13 +171,8 @@
inherit (builds) rustfmt-formatting clippy-lints;
bootvine-build = builds.bootvine;
repo-tasks-build = builds.repo-tasks;
vm-boot-and-shutdown = pkgs.testers.runNixOSTest (
import ./tests/vm-checks/boot.nix { sampleMachine = sampleMachine.machine; }
);
vm-luks-passphrase = pkgs.testers.runNixOSTest (
import ./tests/vm-checks/luks.nix { bootvineModule = self.nixosModules.bootvine; }
);
}
// vmTests.checksFor pkgs
);
devShells = forEachSystem (
+23 -19
View File
@@ -4,18 +4,24 @@
craneLib,
}:
let
bootvineCrate = craneLib.crateNameFromCargoToml { cargoToml = ../bootvine/Cargo.toml; };
commonArgs = {
inherit (bootvineCrate) pname version;
src = lib.fileset.toSource {
root = ./.;
root = ../.;
fileset = lib.fileset.unions [
(craneLib.fileset.commonCargoSources ./.)
./system-files
./.rustfmt.toml
(craneLib.fileset.commonCargoSources ../.)
../bootvine/system-files
../.rustfmt.toml
];
};
strictDeps = true;
nativeBuildInputs = [ pkgs.pkg-config ];
buildInputs = [ pkgs.dbus ];
buildInputs = [
pkgs.dbus
pkgs.systemd
];
};
cargoArtifacts = craneLib.buildDepsOnly (commonArgs // { pname = "bootvine-workspace"; });
@@ -25,7 +31,7 @@ let
in
{
rustfmt-formatting = craneLib.cargoFmt {
inherit (commonArgs) src;
inherit (commonArgs) src pname version;
cargoExtraArgs = "--all";
};
@@ -41,35 +47,33 @@ in
cargoExtraArgs = "-p bootvine";
postInstall = ''
for unit in bootvine.service bootvine-shutdown.service; do
install -Dm644 system-files/systemd/$unit $out/lib/systemd/system/$unit
install -Dm644 bootvine/system-files/systemd/$unit $out/lib/systemd/system/$unit
substituteInPlace $out/lib/systemd/system/$unit \
--replace-fail /usr/bin/bootvine $out/bin/bootvine
done
install -Dm755 system-files/dracut/90bootvine/module-setup.sh \
install -Dm755 bootvine/system-files/dracut/90bootvine/module-setup.sh \
$out/lib/dracut/modules.d/90bootvine/module-setup.sh
install -Dm644 system-files/mkinitcpio/install/sd-bootvine \
install -Dm644 bootvine/system-files/mkinitcpio/install/sd-bootvine \
$out/lib/initcpio/install/sd-bootvine
'';
meta = {
mainProgram = "bootvine";
platforms = lib.platforms.linux;
license = lib.getLicenseFromSpdxId (lib.importTOML ./Cargo.toml).package.license;
license = lib.getLicenseFromSpdxId (lib.importTOML ../bootvine/Cargo.toml).package.license;
};
};
scripted-boot-examples = buildWorkspaceMember {
pname = "bootvine-scripted-boot-examples";
cargoExtraArgs = "-p bootvine --examples";
scripted-demos = buildWorkspaceMember {
inherit (craneLib.crateNameFromCargoToml { cargoToml = ../scripted-demos/Cargo.toml; })
pname
version
;
cargoExtraArgs = "-p scripted-demos";
doCheck = false;
installPhaseCommand = ''
for example in boot shutdown; do
install -Dm755 target/release/examples/$example $out/bin/bootvine-$example-example
done
'';
};
repo-tasks = buildWorkspaceMember {
pname = "xtask";
inherit (craneLib.crateNameFromCargoToml { cargoToml = ../xtask/Cargo.toml; }) pname version;
cargoExtraArgs = "-p xtask";
doCheck = false;
meta.mainProgram = "xtask";
File renamed without changes.
Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.7 KiB

After

Width:  |  Height:  |  Size: 11 KiB

+17
View File
@@ -0,0 +1,17 @@
[package]
description = "Scripted boots, shutdowns and passphrase prompts that play bootvine's screen in a terminal"
edition = "2024"
license = "GPL-3.0-or-later"
name = "scripted-demos"
publish = false
version = "0.1.0"
[lints]
workspace = true
[dependencies]
bootvine = { path = "../bootvine" }
futures = { version = "0.3.34", default-features = false, features = ["std"] }
ratatui = "0.30.2"
tokio = { version = "1.54.0", default-features = false, features = ["rt", "time"] }
zeroize = "1.9.1"
@@ -1,9 +1,7 @@
use std::{io, time::Duration};
use bootvine::{
boot_model::Phase,
demo_script::scripted_boot::{Scenario, Step, failing, play, step},
};
use bootvine::boot_model::Phase;
use scripted_demos::scripted_boot::{Scenario, ScriptedFailure, Step, failing, play, step};
const INITRD: &[Step] = &[
step(
@@ -127,6 +125,14 @@ const HOST: &[Step] = &[
"nvidia-powerd.service",
&["basic.target"],
Duration::from_millis(700),
ScriptedFailure {
reason: "exited with status 1",
log_lines: &[
"nvidia-powerd version:1.0(build 1)",
"ERROR! Failed to get system power limits",
"ERROR! Client (presumably SBIOS) has not registered for D-Notify events",
],
},
),
step(
"sys-devices-virtual-misc-rfkill.device",
@@ -1,9 +1,7 @@
use std::{io, time::Duration};
use bootvine::{
boot_model::Phase,
demo_script::scripted_boot::{Scenario, Step, play, step, unlocking},
};
use bootvine::boot_model::Phase;
use scripted_demos::scripted_boot::{Scenario, Step, play, step, unlocking};
const ROOT_PARTITION: &str =
"dev-disk-by\\x2duuid-7c2e9f4a\\x2d1b3d\\x2d4e8f\\x2d9a6c\\x2d2d5b8e1f0c3a.device";
@@ -1,9 +1,7 @@
use std::{io, time::Duration};
use bootvine::{
boot_model::Phase,
demo_script::scripted_boot::{Scenario, Step, play, step, stopping},
};
use bootvine::boot_model::Phase;
use scripted_demos::scripted_boot::{Scenario, Step, play, step, stopping};
const SHUTDOWN: &[Step] = &[
step(
+1
View File
@@ -0,0 +1 @@
pub mod scripted_boot;
@@ -7,15 +7,10 @@ use std::{
time::Duration,
};
use futures::stream;
use ratatui::{DefaultTerminal, crossterm::event::Event};
use tokio::runtime::Builder;
use zeroize::Zeroizing;
use crate::{
use bootvine::{
boot_model::{
ActiveState, Activity, BootChange, BootState, JobKind, JobResult, Phase, UnitFacts,
UnitTimeouts, boot_origin::BootOrigin,
ActiveState, Activity, BootChange, BootState, FailureCause, JobKind, JobResult, Phase,
UnitFacts, UnitTimeouts, boot_origin::BootOrigin,
},
boot_view::PasswordPrompt,
console_session::{
@@ -26,17 +21,33 @@ use crate::{
screen_settings::{color_scheme::Colors, config_file::Timing},
systemd_link::monotonic_clock::monotonic_now,
};
use futures::stream;
use ratatui::{DefaultTerminal, crossterm::event::Event};
use tokio::runtime::Builder;
use zeroize::Zeroizing;
const FINAL_FRAME: Duration = Duration::from_secs(3);
pub const DEMO_PASSPHRASE: &str = "demo";
const DEMO_PASSPHRASE_HINT: &str = "This demo unlocks with the passphrase \"demo\"";
#[derive(Clone, Copy)]
pub struct ScriptedFailure {
pub reason: &'static str,
pub log_lines: &'static [&'static str],
}
#[derive(Clone, Copy)]
enum StepEnding {
Succeeds,
Fails(ScriptedFailure),
}
pub struct Step {
unit: &'static str,
after: &'static [&'static str],
work: Duration,
fails: bool,
ending: StepEnding,
stops: bool,
asks: Option<&'static str>,
timeout: Option<Duration>,
@@ -58,7 +69,7 @@ pub const fn step(unit: &'static str, after: &'static [&'static str], work: Dura
unit,
after,
work,
fails: false,
ending: StepEnding::Succeeds,
stops: false,
asks: None,
timeout: None,
@@ -66,9 +77,14 @@ pub const fn step(unit: &'static str, after: &'static [&'static str], work: Dura
}
#[must_use]
pub const fn failing(unit: &'static str, after: &'static [&'static str], work: Duration) -> Step {
pub const fn failing(
unit: &'static str,
after: &'static [&'static str],
work: Duration,
failure: ScriptedFailure,
) -> Step {
Step {
fails: true,
ending: StepEnding::Fails(failure),
..step(unit, after, work)
}
}
@@ -132,13 +148,26 @@ impl ScriptedJob {
}
fn outcome(&self) -> JobResult {
if self.step.fails {
if matches!(self.step.ending, StepEnding::Fails(_)) {
JobResult::Failed
} else {
JobResult::Done
}
}
fn explanation(&self) -> Option<BootChange> {
let StepEnding::Fails(failure) = self.step.ending else {
return None;
};
Some(BootChange::ExplainFailure {
unit: self.step.unit.to_owned(),
cause: FailureCause {
reason: failure.reason.to_owned(),
log_lines: failure.log_lines.iter().map(ToString::to_string).collect(),
},
})
}
fn initial_activity(&self) -> Activity {
Activity {
state: self.step.stops.then_some(ActiveState::Active),
@@ -163,7 +192,7 @@ impl ScriptedJob {
}
fn settled_activity(&self, now: Duration) -> Activity {
let state = if self.step.fails {
let state = if matches!(self.step.ending, StepEnding::Fails(_)) {
ActiveState::Failed
} else if self.step.stops {
ActiveState::Inactive
@@ -329,6 +358,8 @@ impl Script {
at: now,
},
]
.into_iter()
.chain(job.explanation())
}),
)
.collect();
@@ -1,6 +1,6 @@
Output recordings/boot.gif
Require bootvine-boot-example
Require scripted-boot
Set Shell bash
Set FontSize 22
@@ -12,7 +12,7 @@ Set TypingSpeed 0
Set Framerate 10
Hide
Type "clear && bootvine-boot-example"
Type "clear && scripted-boot"
Enter
Show
@@ -1,6 +1,6 @@
Output recordings/shutdown.gif
Require bootvine-shutdown-example
Require scripted-shutdown
Set Shell bash
Set FontSize 22
@@ -12,7 +12,7 @@ Set TypingSpeed 0
Set Framerate 10
Hide
Type "clear && bootvine-shutdown-example"
Type "clear && scripted-shutdown"
Enter
Show
-3
View File
@@ -1,3 +0,0 @@
#[cfg(test)]
pub(crate) mod boot_test_fixtures;
pub mod scripted_boot;
-25
View File
@@ -1,25 +0,0 @@
{ lib, bootvineModule }:
let
sampleUnits = import ./sample-units.nix;
machine = {
imports = [
bootvineModule
./machine.nix
];
_module.args = { inherit sampleUnits; };
};
in
{
inherit sampleUnits machine;
nixosSystemFor =
system:
lib.nixosSystem {
inherit system;
modules = [
machine
./interactive-vm.nix
];
};
}
-91
View File
@@ -1,91 +0,0 @@
{
sampleUnits,
lib,
pkgs,
...
}:
let
sampleService = spec: {
inherit (spec) description;
after = map (dependency: "${dependency}.service") (spec.after or [ ]);
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = [
"${pkgs.coreutils}/bin/sleep ${spec.seconds}"
]
++ lib.optional (spec.fails or false) "${pkgs.coreutils}/bin/false";
}
// lib.optionalAttrs (spec ? stopSeconds) {
RemainAfterExit = true;
ExecStop = "${pkgs.coreutils}/bin/sleep ${spec.stopSeconds}";
};
};
in
{
networking.hostName = "bootvine-demo";
system.stateVersion = "26.05";
boot.consoleLogLevel = lib.mkForce 3;
boot.initrd.systemd = {
services.demo-initrd-probe = {
description = "Slow hardware probe";
wantedBy = [ "initrd.target" ];
before = [ "initrd.target" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
ExecStart = "/bin/sleep ${sampleUnits.initrdProbeSeconds}";
};
};
};
services.bootvine = {
enable = true;
settings.colors = {
tree.current-stage = "light-yellow";
tree.running-marker = "light-yellow";
timeline.running = "light-yellow";
footer.running = "light-yellow";
password.frame = "light-yellow";
};
};
console = {
colors = [
"1d2021"
"fb4934"
"b8bb26"
"fabd2f"
"83a598"
"d3869b"
"8ec07c"
"d5c4a1"
"665c54"
"fb4934"
"b8bb26"
"fabd2f"
"83a598"
"d3869b"
"8ec07c"
"fbf1c7"
];
earlySetup = true;
font = "${pkgs.terminus_font}/share/consolefonts/ter-v20n.psf.gz";
};
systemd.services = lib.mapAttrs (_: sampleService) sampleUnits.services // {
greetd.after = [ "multi-user.target" ];
};
services.greetd = {
enable = true;
settings.default_session.command = "${pkgs.greetd}/bin/agreety --cmd ${pkgs.bashInteractive}/bin/bash";
};
users.users.demo = {
isNormalUser = true;
password = "demo";
};
}
-49
View File
@@ -1,49 +0,0 @@
{ bootvineModule }:
let
passphrase = "quietly unlocked";
in
{
name = "bootvine-luks-passphrase";
nodes.machine =
{ lib, pkgs, ... }:
let
secretUuid = "9d1f2a8e-0b7c-4c55-9a8e-6b2f8f3c1d00";
secretDisk = pkgs.runCommand "bootvine-luks-disk" { nativeBuildInputs = [ pkgs.cryptsetup ]; } ''
truncate -s 32M disk.img
echo -n '${passphrase}' | cryptsetup luksFormat -q --disable-locks --type luks2 \
--pbkdf pbkdf2 --pbkdf-force-iterations 1000 --uuid ${secretUuid} disk.img -
mv disk.img $out
'';
in
{
imports = [ bootvineModule ];
services.bootvine.enable = true;
testing.initrdBackdoor = true;
boot.initrd.luks.devices = lib.mkVMOverride {
secret.device = "/dev/disk/by-uuid/${secretUuid}";
};
virtualisation.qemu.options = [
"-vga none"
"-device virtio-vga,xres=1280,yres=800"
];
virtualisation.qemu.drives = [
{
name = "secret";
file = "${secretDisk}";
driveExtraOpts = {
format = "raw";
snapshot = "on";
};
}
];
};
testScript = ''
PASSPHRASE = "${passphrase}"
''
+ builtins.readFile ./console.py
+ builtins.readFile ./luks.py;
}
+51
View File
@@ -0,0 +1,51 @@
{ lib, bootvineModule }:
let
machineModule = machineName: {
imports = [
bootvineModule
./shared/common.nix
./${machineName}/machine.nix
];
networking.hostName = "bootvine-${machineName}";
};
machineNames = [
"slow-services"
"text-console"
"scripted-initrd"
"encrypted-volume"
"failures"
];
machines = lib.genAttrs machineNames machineModule;
in
{
inherit machines;
sampleUnits = import ./slow-services/sample-units.nix;
nixosSystemFor =
system: machineName:
lib.nixosSystem {
inherit system;
modules = [
machines.${machineName}
(import ./shared/interactive-vm.nix machineName)
];
};
checksFor = pkgs: {
vm-boot-and-shutdown = pkgs.testers.runNixOSTest (
import ./slow-services/check.nix { machine = machines.slow-services; }
);
vm-luks-passphrase = pkgs.testers.runNixOSTest (
import ./encrypted-volume/check.nix {
machine = machines.encrypted-volume;
inherit pkgs;
}
);
vm-failures = pkgs.testers.runNixOSTest (
import ./failures/check.nix { machine = machines.failures; }
);
};
}
+24
View File
@@ -0,0 +1,24 @@
{ machine, pkgs }:
let
secretDisk = import ./secret-disk.nix { inherit pkgs; };
in
{
name = "bootvine-luks-passphrase";
nodes.machine = {
imports = [ machine ];
testing.initrdBackdoor = true;
virtualisation.qemu.options = [
"-vga none"
"-device virtio-vga,xres=1280,yres=800"
];
virtualisation.qemu.drives = [ secretDisk.qemuDrive ];
};
testScript = ''
PASSPHRASE = "${secretDisk.passphrase}"
''
+ builtins.readFile ../shared/console.py
+ builtins.readFile ./check.py;
}
@@ -11,13 +11,13 @@ machine.start()
retry(lambda _: pending_prompt() != "", timeout_seconds=60)
wait_for_screen("passphrase")
assert_text_visible()
assert_text_visible(font_has_512_glyphs=True)
first_prompt = pending_prompt()
machine.send_chars("not the passphrase\n")
retry(lambda _: answered_prompts() == 1, timeout_seconds=30)
retry(lambda _: pending_prompt() not in ("", first_prompt), timeout_seconds=30)
wait_for_screen("passphrase")
assert_text_visible()
assert_text_visible(font_has_512_glyphs=True)
machine.screenshot("passphrase")
machine.send_chars(f"{PASSPHRASE}\n")
machine.wait_for_unit("systemd-cryptsetup@secret.service")
+11
View File
@@ -0,0 +1,11 @@
{ lib, pkgs, ... }:
let
secretDisk = import ./secret-disk.nix { inherit pkgs; };
in
{
boot.initrd.luks.devices = lib.mkVMOverride {
secret.device = "/dev/disk/by-uuid/${secretDisk.uuid}";
};
virtualisation.vmVariant.virtualisation.qemu.drives = [ secretDisk.qemuDrive ];
}
+22
View File
@@ -0,0 +1,22 @@
{ pkgs }:
let
passphrase = "sample passphrase";
uuid = "9d1f2a8e-0b7c-4c55-9a8e-6b2f8f3c1d00";
image = pkgs.runCommand "bootvine-luks-disk" { nativeBuildInputs = [ pkgs.cryptsetup ]; } ''
truncate -s 32M disk.img
echo -n '${passphrase}' | cryptsetup luksFormat -q --disable-locks --type luks2 \
--pbkdf pbkdf2 --pbkdf-force-iterations 1000 --uuid ${uuid} disk.img -
mv disk.img $out
'';
in
{
inherit passphrase uuid;
qemuDrive = {
name = "secret";
file = "${image}";
driveExtraOpts = {
format = "raw";
snapshot = "on";
};
};
}
+49
View File
@@ -0,0 +1,49 @@
{ machine }:
{
name = "bootvine-failures";
nodes.machine =
{ pkgs, ... }:
let
holdUntilReleased =
marker: "${pkgs.bash}/bin/bash -c 'until [ -e /run/${marker} ]; do sleep 0.1; done'";
in
{
imports = [ machine ];
bootvineSample.startFailuresByHand = true;
testing.initrdBackdoor = true;
boot.initrd.systemd = {
storePaths = [ "${pkgs.bash}/bin/bash" ];
services.initrd-failures-hold = {
description = "Initrd held for the check";
wantedBy = [ "initrd.target" ];
before = [ "initrd.target" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
ExecStart = holdUntilReleased "release-initrd-failures-hold";
};
};
};
systemd.services.boot-failures-hold = {
description = "Boot held for the check";
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = holdUntilReleased "release-boot-failures-hold";
};
};
virtualisation = {
memorySize = 2048;
qemu.options = [
"-vga none"
"-device virtio-vga,xres=1280,yres=800"
];
};
};
testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py;
}
+59
View File
@@ -0,0 +1,59 @@
def start_failing_unit(unit):
machine.succeed(f"systemctl start --no-block {unit}")
def unit_result(unit):
return machine.succeed(f"systemctl show -p Result --value {unit}").strip()
def failure_panel():
lines = screen().splitlines()
start = next(index for index, line in enumerate(lines) if " Failures " in line)
return "\n".join(lines[start:])
machine.start()
wait_for_screen("Initrd", "Initrd held for the check")
start_failing_unit("initrd-nic-firmware-load")
wait_for_screen("Failures", "initrd-nic-firmware-load exited with status 1", "rtl8168h-2.fw not found")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
machine.succeed("touch /run/release-initrd-failures-hold")
machine.switch_root()
wait_for_screen("Stage 2", "Boot held for the check")
wait_for_screen("initrd-nic-firmware-load exited with status 1")
start_failing_unit("failure-report")
wait_for_screen("app-config-check exited with status 2", "missing key 'listen'")
retry(lambda _: unit_result("failure-report") != "", timeout_seconds=10)
assert "× failure-report" not in failure_panel(), f"a dependency failure reached the panel:\n{screen()}"
assert "failure-report" in screen(), f"the dependency failure left the tree:\n{screen()}"
start_failing_unit("worker-pool")
wait_for_screen("worker-pool", "SIGSEGV", "starting worker pool with 4 threads")
start_failing_unit("license-server-wait")
wait_for_screen("license-server-wait timed out", "waiting for license server at 10.0.0.9:27000")
start_failing_unit("metrics-agent")
wait_for_screen("metrics-agent exited with status 203")
start_failing_unit("http-listener")
wait_for_screen("http-listener restarted too often", "address already in use")
start_failing_unit("database-client")
wait_for_screen("database-client exited with status 1: Cannot reach database", "connection refused")
start_failing_unit("upstream-uplink")
wait_for_screen("upstream-uplink exited with status 1", "lost connection to upstream, giving up")
wait_for_screen("× 9 failed", "connection refused")
assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}"
print(screen())
machine.screenshot("failures")
machine.succeed("touch /run/release-boot-failures-hold")
retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60)
print(bootvine_journal())
+148
View File
@@ -0,0 +1,148 @@
{
config,
lib,
pkgs,
...
}:
let
startedByHand = config.bootvineSample.startFailuresByHand;
wantedAtBoot = target: lib.optional (!startedByHand) target;
failingService =
{
description,
script,
serviceConfig ? { },
unitConfig ? { },
extraAttributes ? { },
}:
{
inherit description unitConfig;
wantedBy = wantedAtBoot "multi-user.target";
serviceConfig = {
Type = "oneshot";
ExecStart = pkgs.writeShellScript "failing-service" script;
}
// serviceConfig;
}
// extraAttributes;
initrdFirmwareScript = pkgs.writeShellScript "initrd-nic-firmware-load" ''
echo "loading firmware for enp1s0 (r8169)"
echo "firmware: rtl_nic/rtl8168h-2.fw not found" >&2
exit 1
'';
in
{
options.bootvineSample.startFailuresByHand = lib.mkEnableOption ''
leaving the failing sample units out of the boot targets, so a check can
start them one at a time
'';
config = {
boot.initrd.systemd = {
storePaths = [
initrdFirmwareScript
"${pkgs.bash}/bin/bash"
];
services.initrd-nic-firmware-load = {
description = "Network card firmware";
wantedBy = wantedAtBoot "initrd.target";
before = [ "initrd.target" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.bash}/bin/bash ${initrdFirmwareScript}";
};
};
};
systemd.services = {
app-config-check = failingService {
description = "Application configuration check";
script = ''
echo "reading /etc/app/app.toml"
echo "config: /etc/app/app.toml: missing key 'listen'" >&2
exit 2
'';
};
worker-pool = failingService {
description = "Worker pool";
script = ''
echo "starting worker pool with 4 threads"
kill -SEGV $$
'';
};
license-server-wait = failingService {
description = "License server check";
script = ''
echo "waiting for license server at 10.0.0.9:27000"
sleep infinity
'';
serviceConfig.TimeoutStartSec = 4;
};
metrics-agent = {
description = "Metrics agent";
wantedBy = wantedAtBoot "multi-user.target";
serviceConfig = {
Type = "oneshot";
ExecStart = "/usr/bin/metrics-agent --listen :9100";
};
};
http-listener = failingService {
description = "HTTP listener";
script = ''
echo "bind 0.0.0.0:8080: address already in use" >&2
exit 1
'';
serviceConfig = {
Restart = "on-failure";
RestartSec = "200ms";
};
unitConfig = {
StartLimitBurst = 3;
StartLimitIntervalSec = 30;
};
};
database-client = failingService {
description = "Database client";
script = ''
systemd-notify --status="Cannot reach database at db.internal:5432"
echo "connect db.internal:5432: connection refused" >&2
exit 1
'';
serviceConfig = {
Type = "notify";
NotifyAccess = "all";
};
};
upstream-uplink = failingService {
description = "Upstream uplink";
script = ''
echo "connected to upstream"
sleep 3
echo "lost connection to upstream, giving up" >&2
exit 1
'';
serviceConfig.Type = "simple";
};
failure-report = failingService {
description = "Failure report";
script = ''
echo "all checks passed"
'';
extraAttributes = {
requires = [ "app-config-check.service" ];
after = [ "app-config-check.service" ];
};
};
};
};
}
+6
View File
@@ -0,0 +1,6 @@
{ lib, ... }:
{
imports = [ ../slow-services/machine.nix ];
boot.initrd.systemd.enable = lib.mkForce false;
}
+45
View File
@@ -0,0 +1,45 @@
{ lib, pkgs, ... }:
{
system.stateVersion = "26.05";
boot.consoleLogLevel = lib.mkForce 3;
services.bootvine = {
enable = true;
settings.colors = {
tree.current-stage = "light-yellow";
tree.running-marker = "light-yellow";
timeline.running = "light-yellow";
footer.running = "light-yellow";
password.frame = "light-yellow";
};
};
console = {
colors = [
"1d2021"
"fb4934"
"b8bb26"
"fabd2f"
"83a598"
"d3869b"
"8ec07c"
"d5c4a1"
"665c54"
"fb4934"
"b8bb26"
"fabd2f"
"83a598"
"d3869b"
"8ec07c"
"fbf1c7"
];
earlySetup = true;
font = "${pkgs.terminus_font}/share/consolefonts/ter-v20n.psf.gz";
};
users.users.sample-user = {
isNormalUser = true;
password = "sample-user";
};
}
@@ -51,6 +51,7 @@ def wait_for_screen(*fragments):
text = screen()
if last_try:
print(text)
print(bootvine_journal())
return all(fragment in text for fragment in fragments)
retry(shows_all, timeout_seconds=60)
@@ -1,4 +1,4 @@
{
machineName: {
boot.loader.grub.enable = false;
services.getty.autologinUser = "root";
@@ -11,7 +11,7 @@
memorySize = 2048;
cores = 2;
graphics = true;
diskImage = "\${XDG_CACHE_HOME:-$HOME/.cache}/bootvine-sample-machine.qcow2";
diskImage = "\${XDG_CACHE_HOME:-$HOME/.cache}/bootvine-${machineName}.qcow2";
qemu.options = [
"-vga none"
"-device virtio-vga,xres=1920,yres=1080"
@@ -1,4 +1,4 @@
{ sampleMachine }:
{ machine }:
{
name = "bootvine-boot-and-shutdown";
@@ -8,21 +8,21 @@
waitFor = flag: "/bin/sh -c 'until [ -e /run/${flag} ]; do sleep 0.1; done'";
in
{
imports = [ sampleMachine ];
imports = [ machine ];
testing.initrdBackdoor = true;
boot.initrd.systemd.services.demo-initrd-probe.serviceConfig.ExecStart = lib.mkForce (
waitFor "release-initrd-probe"
boot.initrd.systemd.services.initrd-hardware-probe.serviceConfig.ExecStart = lib.mkForce (
waitFor "release-initrd-hardware-probe"
);
systemd = {
settings.Manager.ShowStatus = lib.mkForce true;
services.demo-db.serviceConfig.ExecStop = lib.mkForce (waitFor "release-demo-db");
services.backdoor.before = [ "demo-db.service" ];
services.demo-late-sensor = {
services.sample-database.serviceConfig.ExecStop = lib.mkForce (waitFor "release-sample-database");
services.backdoor.before = [ "sample-database.service" ];
services.late-sensor-probe = {
wantedBy = [ "graphical.target" ];
after = [ "greetd.service" ];
serviceConfig = {
Type = "oneshot";
ExecStart = waitFor "release-late-sensor";
ExecStart = waitFor "release-late-sensor-probe";
};
};
};
@@ -35,5 +35,5 @@
};
};
testScript = builtins.readFile ./console.py + builtins.readFile ./boot.py;
testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py;
}
@@ -12,20 +12,20 @@ wait_for_screen("Initrd", "Slow hardware probe")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
assert console_level() == "1", f"the kernel console was not quieted: {console_level()}"
machine.succeed("touch /run/release-initrd-probe")
machine.succeed("touch /run/release-initrd-hardware-probe")
machine.switch_root()
retry(lambda _: unit_state("demo-db") == "activating", timeout_seconds=60)
wait_for_screen("Stage 2", "demo-db")
retry(lambda _: unit_state("sample-database") == "activating", timeout_seconds=60)
wait_for_screen("Stage 2", "sample-database")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
assert console_level() == "1", f"stage 2 sysctl reopened the kernel console: {console_level()}"
machine.wait_for_unit("greetd.service")
retry(lambda _: unit_state("bootvine") == "inactive", timeout_seconds=60)
assert unit_state("demo-late-sensor") == "activating", "the boot finished before the display manager took over"
assert unit_state("late-sensor-probe") == "activating", "the boot finished before the display manager took over"
assert foreground_terminal() != f"tty{BOOTVINE_TERMINAL}", "bootvine took the console back from the display manager"
machine.succeed("touch /run/release-late-sensor")
machine.succeed("touch /run/release-late-sensor-probe")
machine.wait_for_unit("graphical.target")
journal = bootvine_journal()
@@ -50,9 +50,9 @@ assert "boot already finished" in bootvine_journal(), "a restart after boot did
assert foreground_terminal() == foreground, "a restart after boot took the console"
machine.succeed("systemctl poweroff --no-block")
retry(lambda _: unit_state("demo-db") == "deactivating", timeout_seconds=60)
wait_for_screen("Shutdown", "demo-db")
retry(lambda _: unit_state("sample-database") == "deactivating", timeout_seconds=60)
wait_for_screen("Shutdown", "sample-database")
assert_bootvine_in_front()
assert_text_visible(font_has_512_glyphs=True)
machine.succeed("touch /run/release-demo-db")
machine.succeed("touch /run/release-sample-database")
machine.wait_for_shutdown()
+42
View File
@@ -0,0 +1,42 @@
{ lib, pkgs, ... }:
let
sampleUnits = import ./sample-units.nix;
sampleService = spec: {
inherit (spec) description;
after = map (dependency: "${dependency}.service") (spec.after or [ ]);
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = [
"${pkgs.coreutils}/bin/sleep ${spec.seconds}"
]
++ lib.optional (spec.fails or false) "${pkgs.coreutils}/bin/false";
}
// lib.optionalAttrs (spec ? stopSeconds) {
RemainAfterExit = true;
ExecStop = "${pkgs.coreutils}/bin/sleep ${spec.stopSeconds}";
};
};
in
{
boot.initrd.systemd.services.initrd-hardware-probe = {
description = "Slow hardware probe";
wantedBy = [ "initrd.target" ];
before = [ "initrd.target" ];
unitConfig.DefaultDependencies = false;
serviceConfig = {
Type = "oneshot";
ExecStart = "/bin/sleep ${sampleUnits.initrdProbeSeconds}";
};
};
systemd.services = lib.mapAttrs (_: sampleService) sampleUnits.services // {
greetd.after = [ "multi-user.target" ];
};
services.greetd = {
enable = true;
settings.default_session.command = "${pkgs.greetd}/bin/agreety --cmd ${pkgs.bashInteractive}/bin/bash";
};
}
@@ -1,51 +1,51 @@
{
target = "bootvine-demo";
target = "bootvine-sample-services";
initrdProbeSeconds = "6";
services = {
demo-disk = {
scratch-disk-prepare = {
description = "Scratch disk preparation";
seconds = "2";
stopSeconds = "2";
};
demo-db = {
description = "Demo database";
sample-database = {
description = "Sample database";
seconds = "3";
stopSeconds = "3";
after = [ "demo-disk" ];
after = [ "scratch-disk-prepare" ];
};
demo-cache = {
memory-cache = {
description = "In-memory cache";
seconds = "1.5";
stopSeconds = "1";
};
demo-web = {
web-frontend = {
description = "Web frontend";
seconds = "2";
stopSeconds = "2";
after = [
"demo-db"
"demo-cache"
"sample-database"
"memory-cache"
];
};
demo-fetch = {
remote-content-sync = {
description = "Remote content sync";
seconds = "4";
};
demo-flaky = {
flaky-health-check = {
description = "Flaky health check";
seconds = "1";
fails = true;
};
demo-slow = {
slow-warm-up = {
description = "Slow warm-up job";
seconds = "8";
};
demo-report = {
startup-report = {
description = "Startup report";
seconds = "1";
after = [
"demo-web"
"demo-fetch"
"web-frontend"
"remote-content-sync"
];
};
};
+6
View File
@@ -0,0 +1,6 @@
{ lib, ... }:
{
imports = [ ../slow-services/machine.nix ];
services.greetd.enable = lib.mkForce false;
}
+8 -3
View File
@@ -16,8 +16,8 @@ struct RepoTasks {
#[derive(Subcommand)]
enum Task {
#[command(
about = "Record the README screencasts with vhs and copy the passphrase screenshot into \
recordings/"
about = "Record the screencasts with vhs and copy the passphrase and failures screenshots \
into recordings/"
)]
RecordScreencasts(RecordScreencastsOptions),
#[command(
@@ -30,12 +30,17 @@ enum Task {
struct RecordScreencastsOptions {
#[arg(long, help = "passphrase.png produced by the vm-luks-passphrase check")]
passphrase_png: PathBuf,
#[arg(long, help = "failures.png produced by the vm-failures check")]
failures_png: PathBuf,
}
fn main() -> ExitCode {
let outcome = match RepoTasks::parse().task {
Task::RecordScreencasts(options) => {
record_screencasts::record_screencasts(&options.passphrase_png)
record_screencasts::record_screencasts(&record_screencasts::CheckScreenshots {
passphrase_png: &options.passphrase_png,
failures_png: &options.failures_png,
})
}
Task::SyncDefaultConfig => sync_default_config::sync_default_config(),
};
+30 -13
View File
@@ -8,23 +8,40 @@ use xshell::{Shell, cmd};
use crate::task_error::TaskError;
const TAPES: [&str; 2] = ["boot.tape", "shutdown.tape"];
const TAPES: [&str; 2] = [
"scripted-demos/tapes/scripted-boot.tape",
"scripted-demos/tapes/scripted-shutdown.tape",
];
pub fn record_screencasts(passphrase_png: &Path) -> Result<(), TaskError> {
let recordings = Path::new("recordings");
if !recordings.is_dir() {
return Err(TaskError::NotInRepositoryRoot);
}
let shell = Shell::new()?;
TAPES.iter().try_for_each(|tape| {
let tape_path = recordings.join(tape);
cmd!(shell, "vhs {tape_path}").run()
})?;
let copied_png = recordings.join("passphrase.png");
fs::copy(passphrase_png, &copied_png)
pub struct CheckScreenshots<'a> {
pub passphrase_png: &'a Path,
pub failures_png: &'a Path,
}
fn copy_screenshot(
screenshot: &Path,
recordings: &Path,
published_name: &str,
) -> Result<(), TaskError> {
let copied_png = recordings.join(published_name);
fs::copy(screenshot, &copied_png)
.and_then(|_| fs::set_permissions(&copied_png, Permissions::from_mode(0o644)))
.map_err(|source| TaskError::File {
path: copied_png,
source,
})
}
pub fn record_screencasts(screenshots: &CheckScreenshots) -> Result<(), TaskError> {
let recordings = Path::new("recordings");
if !recordings.is_dir() {
return Err(TaskError::NotInRepositoryRoot);
}
let shell = Shell::new()?;
TAPES.iter().try_for_each(|tape| {
let tape_path = Path::new(tape);
cmd!(shell, "vhs {tape_path}").run()
})?;
copy_screenshot(screenshots.passphrase_png, recordings, "passphrase.png")?;
copy_screenshot(screenshots.failures_png, recordings, "failures.png")
}