From c5e2e16b67193b55b104cfb9f20ef2d0ffe3f2e6 Mon Sep 17 00:00:00 2001 From: Willem Vanhulle Date: Fri, 30 Oct 2026 14:53:38 +0100 Subject: [PATCH] feat: record every sample machine's boot as an animated WebP Each sample machine now has a VM check, including new ones for text-console and scripted-initrd. While a check runs, a recorder takes a frame every half second through a second QEMU monitor socket, and record-screencasts encodes the frames into recordings/.webp with img2webp, dropping the BIOS frames at other resolutions. The guide shows each machine's recording in the section it illustrates, replacing the PNG screenshots. The checks are now named vm-. --- docs/development.md | 25 +++--- docs/guide.md | 58 ++++++++++---- flake.nix | 15 ++-- recordings/failures.png | Bin 15468 -> 0 bytes recordings/passphrase.png | Bin 11612 -> 0 bytes vm-tests/default.nix | 20 ++--- vm-tests/encrypted-volume/check.nix | 12 +-- vm-tests/encrypted-volume/check.py | 4 +- vm-tests/failures/check.nix | 17 ++-- vm-tests/failures/check.py | 4 +- vm-tests/scripted-initrd/check.nix | 25 ++++++ vm-tests/scripted-initrd/check.py | 28 +++++++ vm-tests/shared/recorded-display.nix | 11 +++ vm-tests/shared/screen_recording.py | 74 ++++++++++++++++++ vm-tests/slow-services/check.nix | 19 ++--- vm-tests/slow-services/check.py | 3 + vm-tests/text-console/check.nix | 29 +++++++ vm-tests/text-console/check.py | 31 ++++++++ xtask/src/main.rs | 19 +++-- xtask/src/record_screencasts.rs | 112 ++++++++++++++++++++++----- xtask/src/task_error.rs | 2 + 21 files changed, 402 insertions(+), 106 deletions(-) delete mode 100644 recordings/failures.png delete mode 100644 recordings/passphrase.png create mode 100644 vm-tests/scripted-initrd/check.nix create mode 100644 vm-tests/scripted-initrd/check.py create mode 100644 vm-tests/shared/recorded-display.nix create mode 100644 vm-tests/shared/screen_recording.py create mode 100644 vm-tests/text-console/check.nix create mode 100644 vm-tests/text-console/check.py diff --git a/docs/development.md b/docs/development.md index a2e3100..50a45f1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -9,7 +9,7 @@ | `xtask/` | repository tasks run with `cargo xtask` | | `vm-tests/` | the sample NixOS machines, one folder each with its VM check | | `nix/` | the crane builds and the NixOS module | -| `recordings/` | the animations and screenshots the README and the [guide](guide.md) show | +| `recordings/` | the scripted demo GIFs and the sample machine WebP recordings the README and the [guide](guide.md) show | `cargo build` and `cargo test` act on every crate. `cargo run` needs the binary: `--bin bootvine`, `--bin scripted-boot` and so on. @@ -73,9 +73,7 @@ locally from the dev shell with `release-plz update`. | Model and render tests | `cargo test --workspace` | | Scripted demos | `cargo run --bin scripted-boot` (also `scripted-shutdown`, `scripted-luks-unlock`) | | Sample machines in a window | `nix run .#sample-vm-`, see the [guide](guide.md#sample-machines) | -| Boot and shutdown in a VM | `nix build .#checks.x86_64-linux.vm-boot-and-shutdown` | -| Passphrase prompt in a VM | `nix build .#checks.x86_64-linux.vm-luks-passphrase` | -| Failure panel in a VM | `nix build .#checks.x86_64-linux.vm-failures` | +| A sample machine's check | `nix build .#checks.x86_64-linux.vm-` | To watch your own user manager start the sample units: @@ -89,15 +87,18 @@ systemctl --user disable --runtime $(ls $units) ``` Everything a sample machine needs sits in its own folder under `vm-tests/`: -`machine.nix`, and for the machines with a check, `check.nix` and `check.py`. -`vm-tests/shared/` holds the settings every machine shares, the window setup -for `nix run` and the console helpers for the checks. A new machine is a new -folder plus its name in `vm-tests/default.nix`. +`machine.nix`, `check.nix` and `check.py`. `vm-tests/shared/` holds the +settings every machine shares, the window setup for `nix run`, the console +helpers for the checks and the screen recorder. A new machine is a new folder +plus its name in `vm-tests/default.nix`. The VM checks read the console as text, hold the boot at fixed points and wait on marker files instead of sleeping. The failures check starts each failing -unit by hand, so it knows which one is newest in the panel. To step through one by hand, run -`nix run .#checks.x86_64-linux.vm-boot-and-shutdown.driverInteractive` and +unit by hand, so it knows which one is newest in the panel. While a check runs, +`ScreenRecording` takes a frame every half second through a second QEMU +monitor socket, so it never competes with the driver's own monitor; the frames +land in the check's output under `recording/`. To step through a check by hand, run +`nix run .#checks.x86_64-linux.vm-slow-services.driverInteractive` and call `test_script()`. Prefix VM and recording commands with `env -u LD_LIBRARY_PATH` when your shell @@ -106,8 +107,8 @@ exports libraries that clash with [QEMU](https://www.qemu.org) or Chromium. ## Repository tasks - `nix run .#record-screencasts` re-records the animations from the scripted - demos and copies the passphrase and failures screenshots from the VM checks - into `recordings/`, for the README and the [guide](guide.md). + demos and turns the frames each VM check captured into + `recordings/.webp`, for the README and the [guide](guide.md). - `cargo xtask sync-default-config` rewrites the defaults block in [configuration](configuration.md#defaults) from `bootvine defaults`. - `cargo run --bin bootvine -- install --destdir --initrd dracut` stages an install diff --git a/docs/guide.md b/docs/guide.md index 505d67d..e91351b 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -1,9 +1,9 @@ # Guide What bootvine shows on the console, part by part, with the recordings that go -with each. The animations come from the [scripted demos](#scripted-demos); the -screenshots come from the [VM checks](development.md#testing) on real NixOS -machines. +with each. The GIFs come from the [scripted demos](#scripted-demos); the WebP +animations are real boots of the [sample machines](#sample-machines), recorded +by their [VM checks](development.md#testing). ## Reading the screen @@ -47,8 +47,12 @@ From top to bottom: ## Booting through the initrd -The same recording shows a boot from the start. With a systemd initrd, -bootvine starts about a second after the kernel: +A NixOS machine booting through the initrd into stage 2, handing over to a greetd login, then powering off + +The `slow-services` machine boots with a held hardware probe in the initrd, +slow units in stage 2 and one failing unit, hands over to its greetd login +and powers off. With a systemd initrd, bootvine starts about a second after +the kernel: - **The initrd stage**: udev settling, the root device and its file systems under `initrd.target`. @@ -63,6 +67,22 @@ bootvine starts about a second after the kernel: How bootvine gets in that early and survives the switch is in [architecture](architecture.md#starting-in-the-initrd). +### Without a display manager + +The same boot on a machine without a display manager, ending at a text login + +The `text-console` machine boots the same units without greetd. With no +display manager to wait for, bootvine hands the console back as soon as the +default target is reached, and the text login on `tty1` comes forward. + +### Without a systemd initrd + +The same boot with NixOS's scripted initrd: the kernel and initrd lines scroll past before bootvine starts in stage 2 + +The `scripted-initrd` machine uses NixOS's older scripted initrd, which has no +systemd. The initrd's lines scroll past as usual, and bootvine starts in +stage 2, from `sysinit.target` on. + ## Shutdown A poweroff: units stopping under the targets waiting on them @@ -77,7 +97,11 @@ On a poweroff, reboot or halt the tree turns around: ## Passphrase prompts -A LUKS passphrase prompt floating over the initrd tree +A LUKS passphrase prompt floating over the initrd tree, a wrong answer, then the unlock + +The `encrypted-volume` machine asks for a LUKS passphrase in the initrd; its +recording shows a wrong answer, then the right one. + When a unit asks for a passphrase, such as `systemd-cryptsetup@….service` unlocking a [LUKS](https://gitlab.com/cryptsetup/cryptsetup) volume, the @@ -94,7 +118,7 @@ bootvine started, bootvine waits behind it. ## Failures -The failure panel listing failed units with their reason and last journal lines +Units failing one after another, each landing in the failure panel with its reason and last journal lines A unit that broke or timed out gets an entry in the failure panel, newest first: @@ -114,8 +138,8 @@ first: The footer counts both. Failures from the initrd stay in the panel after the switch to the real root. -The screenshot comes from the `failures` sample machine, with every kind of -failure systemd reports. +The recording is the `failures` machine, where every kind of failure systemd +reports happens one after another. ## Trying it @@ -144,14 +168,14 @@ Each sample machine boots in a [QEMU](https://www.qemu.org) window with [Nix](https://nixos.org/download/), and logs in as root at the end. `poweroff` shows the shutdown. -| Run | Shows | -| --- | --- | -| `nix run .#sample-vm-slow-services` | slow units in the initrd and stage 2, one failing unit, and the handover to a greetd login | -| `nix run .#sample-vm-text-console` | the same units without a display manager, handing over at the default target | -| `nix run .#sample-vm-scripted-initrd` | the same units with NixOS's scripted initrd, so bootvine starts in stage 2 only | -| `nix run .#sample-vm-encrypted-volume` | a LUKS volume unlocked in the initrd; the passphrase is `sample passphrase` | -| `nix run .#sample-vm-failures` | a firmware failure in the initrd, then units that exit with an error, crash, time out, cannot be run, restart too often, report a status, die after starting, and one that fails only through its dependency | +| Run | Shows | Recording | +| --- | --- | --- | +| `nix run .#sample-vm-slow-services` | slow units in the initrd and stage 2, one failing unit, and the handover to a greetd login | [booting through the initrd](#booting-through-the-initrd) | +| `nix run .#sample-vm-text-console` | the same units without a display manager, handing over at the default target | [without a display manager](#without-a-display-manager) | +| `nix run .#sample-vm-scripted-initrd` | the same units with NixOS's scripted initrd, so bootvine starts in stage 2 only | [without a systemd initrd](#without-a-systemd-initrd) | +| `nix run .#sample-vm-encrypted-volume` | a LUKS volume unlocked in the initrd; the passphrase is `sample passphrase` | [passphrase prompts](#passphrase-prompts) | +| `nix run .#sample-vm-failures` | a firmware failure in the initrd, then units that exit with an error, crash, time out, cannot be run, restart too often, report a status, die after starting, and one that fails only through its dependency | [failures](#failures) | Each machine keeps its disk in `~/.cache/bootvine-.qcow2`; delete it to start fresh. The machines live in `vm-tests/`, one folder each, next to the -checks that drive them. +check that drives and records it. diff --git a/flake.nix b/flake.nix index 6d4c639..66e83f5 100644 --- a/flake.nix +++ b/flake.nix @@ -122,18 +122,21 @@ sample-vm-failures = runSampleMachine "failures" "units that fail in every way systemd reports, each with journal messages"; record-screencasts = { type = "app"; - meta.description = "Re-record the screencasts and the passphrase and failures screenshots into recordings/"; + meta.description = "Re-record the scripted demo animations and the sample machine recordings into recordings/"; program = lib.getExe ( pkgs.writeShellApplication { name = "record-screencasts"; runtimeInputs = [ pkgs.vhs + pkgs.libwebp builds.scripted-demos builds.repo-tasks ]; - text = "exec xtask record-screencasts --passphrase-png ${ - self.checks.${system}.vm-luks-passphrase - }/passphrase.png --failures-png ${self.checks.${system}.vm-failures}/failures.png"; + text = "exec xtask record-screencasts --vm-check-outputs ${ + pkgs.linkFarm "vm-check-outputs" ( + lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines + ) + }"; } ); }; @@ -172,7 +175,9 @@ bootvine-build = builds.bootvine; repo-tasks-build = builds.repo-tasks; } - // vmTests.checksFor pkgs + // lib.mapAttrs' ( + machineName: check: lib.nameValuePair "vm-${machineName}" check + ) (vmTests.checksFor pkgs) ); devShells = forEachSystem ( diff --git a/recordings/failures.png b/recordings/failures.png deleted file mode 100644 index 99d860775cd43c41b48811c242ead3dd9f29c60c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15468 zcmeAS@N?(olHy`uVBq!ia0y~yU}a!nU{+vZW?*3G+ji*$0|SFlfKQ0)j@@dq3W{kl zA-_FMe(lx2dSs!buHO;{1_cIB7srr_IdA9YZdtOp2eue#-dmX?2tNQ%-0-yJJ7!zfj=<&)2!fx2|PcwKr^vjOv3eH{I99 zi5s-`|5pk+XKQVd(e>-xQr$9zbp9NmROKYgU!d3t=t8BU|4cJiKem;nLSP8~7(o zZ&0~@W- z_Hzxo3tq)3{Vw`=<>37Ndjo17{`9{3=U-QSs!bWww-=3j+*dE~PQA6@+O7>|KPGK* z=ByB_Tec|oP15>g}}?ejQvdwjqM) zoc;FZ`z;O6ANewEdS<7Zu!>3SRl~~7Ik&FVre?oxxM-pF;Ds`a4R5|Sqp~XF-o=r! z20V>XLKiHiKZ@NT7U0h2o2j*7GVq_|HM) z`s;U1lFad^SzTm5xXd`;bXlS=QzA`U=kB-o@V72!y~XIyT6MT#88lj%7#)KZ6f(G2Sin&~vteoYmydg&o;W>s z*?#_eO>)j5Q;L3ToiyE)Ja@)En=R@KSkJ7Vu`xXF)6J}Dw^r?+D8+a>ZDNeJtF=D2 zb2Sq?hlgfSzfi!W+o_y8>aJ#(9@AX;Bd4uFdCGAUqcb_G92ze->oYKUZkI*Pv8ih&30XauG-E|-mAsO-TVL?L!(b85}LT5d8_{B!J@(1a(79jvV<6dvVY zSP^#M!gpV_-Rl~P;#M~`L~4ou+H3!rKS!iNV8fYfLKik3m1R+xF4}yRsq6mxh+;*_ z=@k;I&o<1giKISVYs9jps5GM02 z)AN{`>S^6c;hR{H)HnPn?5e7APvear}6Gzi_75 zX_3VtOa8JYS#SP4H{sHuovmi}wbdSJYc6kI&C=4q{wX_FCe*p;p%}}VvI&xJzwFUS zU==8sxzB%bmWJM=d->uY;!g{6R4Dvroiai2CDR(tAJ6?Ge!kydfA{^d9k#2C^M2i0 zP+WIh#O(O*_xAhUTmPEt*GNeCR{1sWbc!|T{XgUTvFEQdH+0;e_;&A&j)|RD=Ra6- zV}JdQhD{Q4$~GL8pWDdq9T>n=wWYyLc7pD&DL!wW*805g>h~{VJSs5J{(tMDH5v~@ zR69gPl8am!|Cb6}NSJh7Xv6cp9!;?#mp?ufQ#`86qOx=6T9zl-I?V4iSWGmyB))yy zwB@npeuo1g>@ng6>t4CuwE6$6>fBK+mN_c~46eMLx2$2M^RoUTZN|s31-%V-jHfv- ztZ8WwHf8#hG-0_YN-aO5|B=6}Deo=knDdJT*sV>m|CZIo{cVSh-N7gMaw^w0pRJmB&o{C5SC)RK zp-T%d`^0#M`L;g)_B5-o_{dhjnJ#?s(y~~lw7W(wQTNU!_g(K!(VG57{><6HV95yC z#7{;o@t(42F~J)ZzGr&Ov1Mgkz9E$HsgBZAreEeOgE|(ls7yO^awC7dM#4+^g5HKH zpA=^v+8J@-M5KaLOnA|!HNPwier^63c;}nYgq0qCF`R{l3J>%cBkfdzcT8${Vt20M zgu1lyOl;uG2h_vifidnxZ@*8Mr6jF0UfWHHYFdEueSydznTHP%!7+$91-*{8^8 zBzWl+H#TIo7e{*^nBrJ=boSJSm$T>nzW0ll{X~Pa~xk%9q zJ5-LWPm!Az3-*MD-pTn9K?$oHj%`l3+oC$3VfjJ6x_R9OJ3>vXeE*;AZBx8g92B1u+G(j@ z^W@;!O}?UzqLMG#w_bDLvptvgt8ZD?0#l(UQ6Zc+N(-+ZWh#u~u~TU+tuc0+_lVc; zjcngc6hLdQkO!jEc2lY}eFEr#w!fMZhj=7t^I_fC9KS)%bk{2uG%8=K~_{w_Pv&Oc{W>AuYOiQAdW6yL=- zf3bgQczAtUotVw_PtSJR&D)syK65+sCq=Wsduqyc@$7pO9gyopXmiNH;lh;IWT$^W z_iy;NzwmYa?EYQ0M{8eRwT>5C9=!49yhk=~UQPOMzmttIzVUnAvK=>$d=q*8(3NrR zSuTYHrPp=-vp3%WDUuRqV&rFG((=fW{L6Uv|KI!n|2&%iLHN)4ANTF|H`X2Zxb;VU zSwNtJKtZL|RLi&*ka^nB%ycQ_!V&6M`>&UelkYn=`-gVS3zi;HXl z_iCoPO4$olVlw$o%i0fJJ{10SP6j*A>^-st_eAB@n9GEN=J>T9pZvDr^SW$4_L)CC z@@LdvTij5!DP29BjVX?wV}~dxas?U|M~F_?#K9oL4sEGOvu;_=8=2l^`{ZEv?D(cv z=fdZ)xU@!o+34MTG)V97qi4TwXf4>e>zsi2(l=)5mEu)@bAw}a+9P$2N8kF*%k<3Q zz@!E)?K4T1_b%Pz=A2R5w=!Wvv!Vja8K;cQ%dg}lmj8QkC7SW-9EG}h6WaZbDb+7- zE?(dCrKfz?qe3BpZ2`i~J`u*H*wm~B!N-~V%={KMY-?#(&Rsr-C$} zc{RJ5aqO@Z-|w0K`$&^5)3jy0H?nUuK_ko~S~a0mFEekBf2rXe#Ra># z>tY?YS?>7|sCQ~5OUmpo+x&HI1T&qw>dgA*nAHPMR+~cEfOTRlM`8<9y_xjdt}b0G z{NYIX!AY04C_XJapt4}T`jdl;&QxFIJ8_%wsft1llc23(M{(tD*`-_jEjOj;POtpj z^0uLL$^47ERya(m``vf;(;tTe!3>PTThwP~FfXp*$0w*547Cc!)|?V0v97cD=2IEZd=LMkfp$$9X)SRFSquoKU>W0xY>-gnWPpw zf3Z=Vo_0L?Q@3o)rVSg8iB$+JU%7&LnZ~a3r(EZ4*WHjjPr&;?-n*xV4js9-MdIE^ zMLpm3yH0Oz&oI=<%VInkJ9qokhMM&e44qebuP#1x;?=J5(r;H7ud{xsTO@UokZoa?tW;b`7=!ENxo9BIR2v$~D&z!dR`TDc_${aWF0~1q>GJERd2lT6^lDbiBd? zs{*b)4A8vrVgeVq^{1c!Zphy>j^7~ntB@!Eh)EV_s-vKA=Gkowpns`COc#7y_^2~Vy<`YRe3z|gx2W? zS6CmMR$ZX0;61B~vCsa=s^Em@ZF+mQm3LUyi(9jN;q)ly+G4!GzozT%n{~`;jPjeB zrk`c|{($LJqvGTSSucm1(;PqrkbGj{mT30FFNjjw`rZ6SsC_aow0q(bg4y(4;c5v9+1Am`1RET@vG$r z4QGb3w=^hkd8lXM{jf)4`d)6ejj49rbuF#49xlGaXJNP_C!wSN6UWW^hdG7Cy8Zq< zLQ?Nfy=@SFvZ+x~SN+coPTYO1>RXJG%^SD*@v} zFd}-H*}dx*f?ox=8cyk%AqJ92WV*J=_UPG{)$UKOXDZG+`%OeWIXr*Hkqti^ZU^6y z;aH)tG4%`kmCr3c$~ITtC2*Y;VmDTngyHZPQ)o_0O@ldJeN zy%O`QdlTo}^i8w)!5lNs^MKvGo-Ye`uYAY*_)MPBl?Ofisn=~3gAORVB;IAN-JZd* z2fte=G%q%*KS?4%wiW{80@S&9HRizV~o#`YY zgR7rrROhhVDR}YnenaE|i&&^o=8Y5O94p>6IdfVVJz<(w<>KrxJAkP#KrdmX`xkT0 z6=8}G7CJbvcCKY*Tx*q3=)>`nZj6la%%B2i=3mCx?PmYm=b5EcoIZKK>5I+|i~sIsxg1-qZa%(w zRoSYn;3Mnf17@WnUm7D7zqWCHuKzKQ?dVVYJiUaYGbUtQe=sTK<{x>14P0spJ5^@R z`v16_Wy%cI2MX4+`1*3&LIgHUGSNC$Zn88brsvaBrl@46Ne%qZKmD{D=%5)5G zbi~{$v3pa^YBG=Y^iTT=>x3@t+l+<+OL+3xOA@!tp7Y;%J9Au}0!Om%f!YvTS;nn* z)`;(8V2Wda*U9FLsG|pnmXZjxKMF08pe`)3zeJ_dEn*Ox_Ji^ELF#=co7J$cLAo8R9{4^^ob@`cKlwt}5w(?qiOlQfKPzR$!2U>qbRvfbqd8YWN zHp1Z#OZ=Sq@wIK0rSFO-%=_KPZK}4Uo4+OUiN&NHvx7a@=le^|ef4+!(;bpx=PlcA z-S)3ej@mjydAWwbhevP5uAhR+`Rr6FEIrGo9OiorN*qJ&gU!rrtznfeE6OubJ#z{?&%G zYnru%a=chwb%fjV3S&OoZf-Hz`OdNiYp!$8nA&hogE4k$gO~L_@zMjc58Ptx7Yew} zI5&{Jr6J*%$cED_Z_GG${MFkL!@h;}NbHB&2mVgWR>?{Kzqry+@sC=IYu{oI7eT3n zw++F&@9&uMw&8Qyo!M6zpNshKTQ-a9VClvFM^?sRjlL5Z4H_CCtqIv6lo74nEK9aT z-Bj4KN4!QKDSbvCFkw;dy|h8xe{NFJJEH~e#WwS;(m#Bi!L&?I{o0}%+OA)dFKSjz zU-V@mKhKXMFD5ZvnT5VgZF`?6C|sxsII!dJ+1O8t3OAG%$e%ED^6P)C!KHDm^h1h( zKu?Y)<6Ps@#&XlRPh|HRrE_p7C}f;rf8X?#kAvgdZN|{lzWz%J_xNTUs$uiL$M@oR zmBln3rlyJJ-`X1E6J((o=Yparcn}tvU6Qz0sM{-OPhy;^q@TV)^S7_&ft#9B8n*0+ zZ1I{n-T20ozsp1(yxHS;z}MO=mH!>6ksK)y(EESJ3JIM}HV=<`yomSss_eF`{ao_7 z&&4k@&(6qYVfNK?k94>?Gj-9T*=M5NdRD|)e5($-s(4T}Li|F$pNJLLsfsP02R2NX zVfo|cr<`ynvGiIJt4{0pxy}coHz*}YT{2AM(qQk-5!z68go9;|ZZ=a|sl#fPo_Onz zTTGcg$%o9W^*mtll!Ijt>vORQvf@nlCSSh8`dU_%(ffEq!}TQ}bbh?tahFWn_#aL&%+1;gyT2NOeY@pN_1ymx2zU5^PH)85KzZfBl$XtVIc-seB>J)Ki6 zICG80rvo1)_StT}UaOveKAY*-$xE9JxVyS%Dzi*Eyj*Nvi$vyI;h#$yZW<=V_pIN! z(BA1l*9MlQp^v35hcSA#%=JFd)x0|S%ir&uHwsU$Q(wRn{o~r2EoqykF$KvN|CC~^ zua*4s>+wekapvffqT;w8Cm!GW@jpZ`WTNoPl?5_7k9sc3Xghm8G0uo{U9_X%VYGH% z7}8F1d62zCTy9Vy$JhU&p;PMuYqK(Fe{!5y#Dw zpJ?8DaO7U&^Z1M5$8ODjo3J-+Q=M65V(6M( z)+y}g>-Qc1KkuPe4QnK)#a}*up~~~Sc1)8gm1gbP>ygWt8G7L7_gPDOCs@tm{=3tQ zY1!tkWX>%OA2T$ywT-&P%hzyZ$SSKnPpk80Jn_20YmwjI55J=BDZ7EY^`T6gxMjt% z`X(-FNIJstUv9Hig3YXWov95U*K83oTKjAHq`up$Pi3!dxRfyUS|;Ps_F}yzL#6ctY1T8Z z8rZo#G=DwHU`+445aaN1C%Cr zXWmyHs*?3#?4Ii&_N(3};NJhm<%T}r{u{VEKC#rF!=b??KEI79A@CC(V1a z-5f*af`yNLlS=3EJ9L}1=c8F~(+%weD=C&eY+spPy>yuKaqFZXzv3lW154Sa%sn{A zr!T=v_Km^=&YM9@R@;3Yl(lseGDTdjN(Pj%X?cB`E#CXUXN%y3DKfPuCO_q2+~0Kg zUDI-dsP&C2D-Nx@{=O+W>JDps%hs|3Uyk0q+_ooM@M&(1(-PW_W5|{`qr*VS~!i*`H6hPcm42dFHRnFUox#*rfvMoLAko zUm73pee#k`OrpCLleT>IaekGq(-Y!*9-XOpa>R4plsykzazrPrx>j>s^wrN=<0a{sE${8S03Tml087eN&ln`c`)ui_m65Vin2{82TO0ylk{S{PpiHiyBUtj*< z5L1&i10%$-EG)i}ZzuT|^>avNd7dbIOgebjGb{ zN5$5vh)ssSyDD@}=`k9!gL9vR&VQeG`mJ$+n?v9}^A<)j_sT5iU^#Ptn1XY!<09lTQ5$9vj<^~|yC z6-8QdXFrOZS$E~_dWrsCd9n0ac0Rog{(_P+r}7wk<*)oGK5mqhX!~1r^S+azj6WNL z0&nXddn~V}UD^5kd4isI#QNxVrm6&>rXa zO@}WYVASEX2(7CVUn6V4(t7Ipj#;-({kg?qd2yBa;h2^Ss~Hmv8K!Kb@k!TkJYJQ=WI_ zrtHF6g$L87NVFAMpp8Xoa#%=YpB8x>7<1;!vwLe^`5%i)$z>BNkkr#m_%?$%W?Cp? zcBG$U=2d|*~%*KD+wvF&y%D zWYoM)^|dY)>0Gm%>*im-HFD;QPJ6DDKfUGVVwOF2_MWo34`l2rK0Z4d@-CEX#x93^ z(F;498ag+IELl03Yww4bj>TeL2P{h-FrDiWyT&0=z2FwprBw|#Gx!P_Rjzg2S+=}t_@fsQ`oHWx>-5(R z+ZtBQ;$X~g6uS`qbyxfKwbI!wk=75Iw=wM*Iw z2Er$DD;ApjY?otg5LHmfIF-kEb+b7`StGn`kGc>p^GU?Pf%{jN^Sq`|#*-1NeK#hq z&kFo@{!>WG&dhUnHr9Nwi96e<^4Ckm=$h6!SC_cFcyVzpo8tJ{$5l!Tjrs0P2 zS1;2wne9{Vdu`gz%y|0bn?IlPCy8y_HplzXZC21w*5>>dl6G?*_m^L_{`n(req+7K znj+4^=j&LjvL;DHK6^ef^5*8c7WYfnK1_PlnDkGxXvRO4B@*`T633m~V>gtTdVH%7 zS)UN{^Q+1_Ht&OSdwvTG8LvxuerG|x`$N{LD))q?s=_601E$4?@5-DfV0S_Hedp6t zvo&X4-FPf&Nz}aB1yiDGT11Woe{qjBC^M~EUiDw%WKh?J3z|tfrCF~Y+${g%xr$|p z=9C-XW#WtCzwe7F@~eCBX7fIc1e*mNM(YDLPW>o6sVm#Dl<%ZR%UX^d@0)JMH|&}8 z>ylu62BYRvb-&(t(?=7}Ydw(h`crjoMW=8V)2}a2vRFcjwXbk#uuoXxv%LQHvWBP3 zPj9YXuN`gusOjVCi}H#K_Uw9fE1R+OoP3g_!@pjueaDuibx#kfo2hqf^;~tATJ1Mo z8!NUbESTC<+fpHQO8&RQr;Jag*Hane_a*$vN{Lm;W|^{h-D##%I~&evJ=o*-^Xr*^ zb0hcX*)eG|i9P4kvy$An%+3F>*L2K&nlgs z@hs!=k}^E>=;%b%Yn&@zI&AV;Td|b6b+hPv-MXd;PMkk(u?ELG*gXGXxhdt-tX2I7 z^r~&v*{4^A?fubklcg^I`Gk80^Z$hxSNhc)^b2IldwS&h=Q(qJG{puu^z__0(W!sq zxj(Ng>Y%~$my`76e|9PIeLCioI^p?rxy_r(PpsJ>=RDunkX15Uns477u`}hdbrvU% zWU^cdJaI2vc|m+m=3Vpir@XF7Gfg{p{!Gc+2Va>c#ptMSTC_&xH;?+ON%3b=`x_o{ zemJ*ntt(^q$4|mh&P#4+B%D?B{<&7=H&49c1HVseVo#JP&YPNbxKLpMcfYm0t(wZ4 z0B4yi(yVJ0m5LU$)C)=P^Ze*CWq!&EwO`KPUS-5>`{$hO^r&^gA43D{UTa?VnHCpk zO!zTftZ&&S{SwQw%@!e=vLcR5%#P-E&lv* z%d;~C{_B@bZYbMhHzoA*ET6sp4&QegD#n}0q&#v-c9N04o~U*3o7MDK*PnAmy$%%I z`FZ5)@-to%AJ2Z?kn^^K`?-n&I z3I1T9rXX8(Fl0)r$fKfntC&~u*SaynS%y#OgRuNF8F8P>FV>(9)Er`c(^hCHQ^TSi+{N8iSiN9z!)-;c1B!ZlM@~(7 z)WpcRe`3sm?F)+3wE0;6xYb#ieg5)c+u;>+PIfUf?w>e6-n?yt;W{7DgPCTp3NIct zHFGY#^?P^8=7WXBO066J2;H>cx#VpBML4h^ye>wZ`#?nO>^~dDgmu=OHV>Zi^L>)6 z!B?e?zv>sT{9&rwA=4bsQaPzQ@w1_LWn%G!kIT-w>xe2UL@W(qy2WI*|JjYcm*#VF z&sDd-RR8f%K_NnAHDp@lJBR3=EhkxgW(VI2Jg{yXi%DCZ!-0Io{oHal&a3~{eOs}r z&vj;3z&6YExq=0E8XDreZVK=+9$c=$8Bs1SGdZ&K058)bz8_mV(hv4;NYJfa$d|F5 z*^kFBXOjm1{Q2>2zbt$d`<9-FV^ls?->|#so&luIiD-|4ItvO48D4k2PhECOG`X_$ zx|lpmr|H*gQ;wQ`%aeFqYVv?*k4A#wnOhqt&$#+$S?=tHT%0dsh9w^i;i@-S2Qy!<*AQ-D?y&6W3&%RE};H;C}ttw@hsp%bR4j zYqL}z*nD+c|CufQfd-@I(zJ zPkMNTKkLMlMPT}EXMU#I*$yS)e>;`57bux#Z3+) zzx?%wNp+`Z=H1#YQ~7hkGrq^q;#^*ywJcq`c@rO}4wtIJ(W8waDZ4V?7d?6EJj?nT z%bHh=aW*O&B7{EZ?wMq)o0n%bYiIKOhOO-^VgeHw8ST$EHMIATMH@C1q5B^LDGoN?m=>8uY*9c9RZ+!Bf#p!kyvDZv#w#%Pwy-`_tPen5S)|34G z+gZ+BTIH-3(LQs=A{~XLE^mruv>()*6Znw&jqAmqS;^LQ!UFf_d=Hw>udlget>A>O zx6&iO{!+QB$S-bCyG`Z6DaA(DsSWzQuN8i#PSRc;+MaSmOYysP%F#QMpB8OxxN1Cg zg79$xrggGH5h2qW{)D^5T>P5%Rh6-K%c}`YUqe5y`6N`ey5SFV*p!AsM#lJOcbK|P z2z=<1DHWHQ{LwdDDByX{`wj1!!da(SMb2Ec_~N1Cr~X~+FkT^Ukjfy^ntt&728)Ck z_SD1PNj7uCl>}0@@6iLdt5nt z-Dm8+dUDzci`VS)^zXD^XSrg$P+l$J-wWq2tj!JE89}RsOH{8$aa%Zjx@jNnaA5X_ z&#x~StUsq^CwKV${Nv8~=K_!UP8ah!aAWPd^KKJY9AA@cyUt`?a!RDb{Yrs54f5Ap z7v&k{#g%-1cEGH3%FYg%inu@5Cx!h!|K;3F>qE~t91raNJXs~Fj91T7%uIe^NKN1Y zf&M#94STpt_I#ha>x$28fe&UKLIoaWzO(yse`y@&l(-%eH7$A%XQ37=%O2iEd>X7Z zJ3`*=6iD6NaI=70F=35Np0Gf0-d(mb@fZ!pyQ@_fLm`gU34e5;7*9O%x%Y|A|*ds8FC-^J0<;&6WEYETX>i3hU$(pe-7VT z&b>G{bET|7-Z9f!@t7(x9>({Fx7{-c^>=6bF_Yyrj$sv5#*;Ue=uBJm?3(hP!y(Ug zE`7_%R}8-MEM~qM)3hS7%=tO`EPHgLSz}WB8x(t%zYS!nV_hyfLE%Ao0(ZomX@_Sp zB-b_Uwu$VRm)s)bpY8kPV)ybxdrkiz59ZTioPIzlap-3>81MM~AW3Oh`) znO(`VNu!+5DxTR9G&h5bSEdi5TWz-eQ*SpJp z_%!8R(_V3FmY(vp`kW`!RtYY@c>2$g!~?9A%?&#xZ?ir*$;otY#?;*pY?+SjPMCQ= zk?GcA-vg$RtC*!O7ro##NJ=rQbXzgYaQc^$UH*j&EE+p>6aJ?#b*UXtf7LMkdFwrc zude)DngL8qP0QYh6`anOXJJuAtB^N^9(b8+;4HUi$H|2|X1qMR_hv4)WVWCQTio=u zOnilk8CnW*?J+ZAnVwzYws6e68qlur;CsM4^&@GZj{E-JpVEIQS7{g1+f(8TDzh6k$iXS-oLvR&G=OgY<-Aqw|yJq+=`o4F|iMK+a4ku^TQ>liw z4?+QcuEra67Kk3r=li4Z?|W%b+n+AH zTo&$_vTNC?kD58XR~a93?T|{EHzAxgWj3?@i8n!EOuMugr^pI2(gR$7*65#SKZ zUg9SE!GJOIygK{xgxCZAcOgB?NK?TWK}-J=$qQu-I4!Pz2(eC>6Oyu4wjk^KZ1>p# zW}A*k$toRRzG17$P6O$~oO*3_BGc>K!dG^)9uN``FiG9s@G@Mg;V;^x`qH2SSCm&B z`E>78N)?k}L3Ny{fnBSD@|mvX3y%60RGV?^*wZ{s+1VlC)2fCkYj21}Cc0HO_BmeYgd_0I=<+Gjo{$`vsQe$c(a)W;h4>T}4OHlg2EoA++~KEETU z)pv!FBTBE5;#v?7%a^H+Z|v)9hy<2jkec$r1%f%lE;(#zue57nQTe)xbn zGoxehH;#y(ZZRAj*BqhiKa1iSUrx|?a28a{S}|7t-%{TAbnn!YXABj$?0D&AZYuIw zS#aA7(H|zIOhGXjRf4XqkssI0>7HF0c#7$j@OlfWA3D-3CO;b*R_mC&mYHZIEiQn(~#u4a(T;d^FA=_P4w}&ar5)zPnQFngAWO33H{AE^zv!& zm;8EhrCX!b&oa1twodxJy*Siz|IEKWNiUa0#iJEQ-}?DB(8DsJp?b7KpeaYm&4VEm~l>x81zRhcU# zEj8Tgj?WMfJ9dk4zN7kL*?{@;*UP4D-jC-fhyiv=k&07wFo@ z-fUD{d(YY756k@i`SHqZ`|Yls-<7HKYUMJsb*1}H7ruR%d}6P$VNTbbvnP-JuV7Q$ z)SdbF(ZTa-w#(156?Ht{SGeAEkBj}uwBE~wtaDoBuP-&qD2SUIv37RZGAE?t>jht)C z8pbOGK3o?n+Rak3P9dSiEKc@A*)^{NAM+QoJULnM#Ph(y+4b?EY;PodUNro#ojXNw z{|S~Qp^QHz7&$76Dr7$-Fj~hSJ31*yX7Oe(2eUU8I#(q67ByrY;fQvY{cxsZ_k)MA zOs7tyGCp6bhnW5UyhwVHzd}M7drO1)GgijRgFCen!cK{?#O&wRyUv))SSVhTwWHy& z@!iPImoMM#nDf=+ZA0eztXrQm_cpAYsGnl6A$fz^fgHIu?dYsGRg!MOej-sU+WowKUJF@N zw#EugSaU-4+GE#`YmbYs(buw6pEsl7^d`pxvnsmRxGxh}s`Egi)n5Ky3EP8`Io=Mc zvu(Ci9FsQxC|#2)#uB1z)Wd0Ykzdire%0rVX?W^Wy>T_?sXXDmm^7POqrQ^Qs4dTx=7*9TBI+xhi zAmYTubSy61uFpSlpQ=1l+-y_E`Mrq~n2#8Jik3!?yU$b$e0z^_3`t zZ=~Rbf0Z$!29?(~Z#N6pTVT78x7YRFss;_l_XlpXZZZCltMd|l&>EgMf*)iTODjH@-UJ_C1a}^$G&RgV zR;6-Gq{^mqs&`L1&*y-(Eq_|P+G9GJX3w6y`HJ!^v*&55$+NF9Pr0`;QSOzc=R9xC zYGW=I8_TJ&pR^Y~-NMTF)PSGkM`%rmgOK(Tx%W;7@{fpZs9MARlO-+UjB3cUO&mVOx?3QxNs@H#sv=LKXdANJ+8g<`O zk#ifbHoK>=`kkEB@c7iN2!8SWT1~wt^!d4P6<3Pe}B|`)*n%Bj1T2A+5|qV-|!=`_0j#+m4AL8 ziqmQh%v^u)v0=;AcY@cBEy(kjxNFy*1L+T${{5XE|L5%1XweVnxtU%?=Bjzj+`)Q|UHrrP{eO~XH~iP*-f_ah)g#frR&85Dl`zvhq0j0Svm3PSH=VxlOtx`} zEXy8)GFgMn@1H*#hjad)_o~(YnryMahyLI1_uoJIzP|q7@AvmB4gWfv&;I}S-C2eI z)AxSTKdWPOHY8}yhh&Z_L&oK|9&DMhy556Gug8UCr;$EZOW| zYPRw1h(FM{_6gG>zKXcIKf*gQZMS^cxGFA-ea@ZDw-}eZ?VD&{_t`%0&iwy-8q%y2 z%Y#vnR}j;{TFR`sK=;~H$x{@FyT1E!&3~u?bki3n(^`Wox6HBPVsF2lAQ8r zGEYawolWtvg|+FW;Xj&F8qDj1t?ZkE+-{~PWd2rZco(hxJ=f~P{eZ{U#9KB$Y`OjR z#q+GMPbX*GF`v`7GG$5jjd{Bv!He{8r^G>)O0VZQ=HiFts05R;@r zSZTi^b4M$GP(w0*-}|D?b9`7n7$(Geb}0D!Dfi|!u0FlTnmK%WWjWhAy~qE)d|SNu zZo>bbx2Mx5yq}wGwLtD;=3~2QYY$5=aq7Qnf7s=BaD(~o=lWIqmoY5gH*FdNcUa}* zchb}QQse(xU)Hk!dTEZoZ$0<=#$|UOd+X2rvO}`LzPC6;`~K?Z`_-8&_Io$b+l zW{=9gQs1(!yFMPctKX z*+#J$?WVd5SVNiJHB%VgKWFAR=gsKPP~dow*})Zg1{MM!)S*%q43#6fw%DHv9@%yMg(|D~B1ZD;{Vx_=z*E2xENH z$7+$=x?AGG%2y8ZH9>2bZMP*eCi!r*HBXp-WZv72f`#fee~xpSe0yJTiB)=56N8~b z>Y0Z>${D2RC(Tva#(euyq|x8db9PHkUf%vvgjt>Cfm>8!um-nx1)e9oGjF=Ho06++xKacurwlVk!K~c(-%F8{wDt z=WZ2IknXIQ@Kf`yqJ~t!>e-Qz*SZew*f4Q|ZuOZtS7e(G-RHXf_5O-IEK!e_%>92- z)_SEi%Tmqd%b7a1Z7q?SYrM-jU{-O}GAn~qVsoy|^5?R=`u@zZP{vioOg)=Qc?Js(m)@UiV7RkFl;Qk0vFRTBt(lVo?rJdZ z2n=p85B5@@by4KO&8iypaJCJ{XQmv=^sT=uZT*cyaj#Nmr-h%^y$c&1AM({HCGyts zPDztI&yaOMF)_~S-1`}8HCF$Awm9rFLvGs?$2_S&;yKeko@JQ4TV}aP!k-$J75}%+ zmJ(z5b>wykYrtPiO^3NppRzDat-roya@OC2u~$oNnniqN))v}MZ0|p7;nx+efA9m- z<7*BJC08Bayq&LjedarM8Jk;bk2f>y&dAfc=hRS_p>ZKaD*l0ctK9M5cb z@@c#M>*Ds`={8?Gd$wN^Zg|Fd;P-9DMUU0K-Al;*EAl{3a{iP9nhqTd_vAe7?gUA_ zXZE-!9sWa%(d_h(s)oPLrR58fCH8$PYsj&_E} z7YCmGCBv|_>+zaTLND|(rhMdd*vhaW(JpH$(;Gh)fq!$?Gw1$VqO~xgXFIdcbqCF> zjc1MQYQ^_G-n}8(jOBMD!xjFP(ltJOrDcpaU=Sw#$#!QRoQ0~uW39EVlwyg`6aE&VD43;B&}*U zvA61xaA#Yt@A|#dFUc-h;V$z@sQg8RA%E1w?K9^^ED>i^yJ>X!v+h*pHwyf<26l59 zbWR*z^qlK}*Hp%1MpOFEn_bdzs49N4bLAnY*9W&=aJ`)Ua1P7g`GVR zm!f^=qtJVE*EiRyzYxxx*dICH!%Ca&U;F2cleW$gFSGSI_CQKwkCh2ykM8#==@)#j zPn!F+bH^D6h8JQ{>0NfwDO`2@4#8|3j0=9N%9@mm=`bDf@n>L)G`i0`;Zu3_8D<^7 zhW(45F%-#q6bGj={QNI^mv`GeNj~-;tc(l%ycf(?J~rXv(Vw1u%r^?Pefrp%FM7Xw z8GnGGINNd#r$T|O$9-l#))nh`0{j_P9cJ9*zvKc}v7O&VuJX?em&1PCu)6TuCEO@-tTKvU#a%nYq?HVhQU5 zE?tH-^JjbIpYlrX^nWr(r~w@A40%^{4!qdVB|meR}{^Up*&PWb)VeZLqQ zf|I|nJSY@7{++pHOhZylNAs%X+;#?^-Ia^mhq7!z%fE8H2y^)M@3G z0teb}@3m*$u{}trZcUlwgU)4_k{VW8tDO7$oAKE`zQg}!8T%}K+PPA);e`v!mPOSK zJC0mBCmG2wWA-ashuXfd3dsd+nhv|ym!IokxU)iL!MW^-4~h*=-ZaFAp|6O#fDZIXMh z|7Na#<-6Nn40$1(3jc(XUVh10z30th&v&<9yQ&u$UGuyk1*MV0n`f%Ewtmz1 z_RfBy!}i|qyjftzX*K;|J^lJiGAal4<3d@N6`y}AKe>5wtoer!@k_c)S0p!;~vtw!@td}%pX6mZV_tN#tjgU{a^ zFiNo9*(tyyB~mNhG+FUTVn%Ys_k){5OebxXTk?;;CguNe-`)lWZ@IYw*NRquduwn& zZ(oksf};POA2UL>r0+BN`+8nsCZ1Ua4X)*l_Rm=*~Ef?11GdXxLy6x*c#$f!veeEOV2AdFjwhm^I7t#;I zD^`lOEfLbW@A#0Dm$jkf4W~mb!+|s3mN!llS4`UJA2@MpBjcgdsvEcxI$Spwr~5MQ zsuQelmrpLgJAHBFDyfw$0Y_B@LQXDb7P;0Gzwp}j!whGgWF^wi|GlvHbEvamf{fdO z_ig`+SITBBGi6A8HQ|Bql#}6%=dL#vev4z^D3CdDr)ECmmG^8JV$lzEtWti--n!nX z%`oltfefz=Sq*LZQt~>f=dZ>8&^z;bU9@`hSFHehfpn848j}Rt3#)(BDg20BC6)Q; z|AJW7IVVl3j&4|bmH%gB)O|j7#q}$ggI_+nc<1!~{g>-on|{W*&(91q*#fE`MHx=7 zxBj8r@s+RP;D5)NNAA864RZGGWG!8s#xTp-xG>e0RUi`#%1Y@)ac!XWmI~z5ylKGSed=7-q9N#ei$?210~PWr2o!)hjn4u*fbIRai9 z9Z2BvX56=Q`cAjij8gm;To`sU9q2pDui8*E-ETb;hdhhG@_-HTNAm8Td~~}~yhXRX zEVqx5@zIU^=(U+<8n=9ZfAisE?NzsNn75SeY{|(~sYfd17S0pA)@)^%y8ZNugeJGw zWqE%dB*%uR?ox@L=zMTN`GE)B`)p4yc7JJ~62&BZ+S^#>>nz3@{!2p{I6{_AoS(wB z;O}1R`Lbv0mu%Ebd#t!4d*_Lr!WllwyV)(hlp6{-8kfG?TE4w=}r+2Kq_m$cD!{gK(=Y;30rH)zKS!_I1 zxi(`858t|ve5`T>$^JZn0-L1o8?n4+W_rV(HD&!J{xz3&)&;YJnz?)o(S1qQ`_HAz z@0tN=<_a}Pi!xuiX=pt89nUA@MVIF1e7A@dQ+R%DN*R9$r^4woORSqWeYa$cSk37$ z$e1cjQci)$x?bnRn{7Z^_(c+@F27o;@vvN#Kge0=rc0H8<;fy2@A|aHTR_ ztz;(E)^(d(`S2;2KQ3+m~eyGZEZf)s)=I8p~Ix}1ywHTJ|j8@-v?8A;H z=X|}No^$)0vbEyhX)ZNoS?R|j&55C;IK`^oW@XRaGImy$iTt9~wYjEoC z^n0kg_O8|If6;pKCl<`U8_AG&MXN!*x_foF^Y%W85c5?ga#0}*H63Q^to90S__TOk z5SxL|G*iQw{q4<5ML)^!^(vozdjFisx}uxKdZv9pY&41SO5LIvnhu*;KD_X{vOn65 zp_b*(MEk{&4FdB!);=|^)e5$`<)d{^A%|&$Q!&HK7n}_Disv)4P56>`)a%RKf)#8( zB&QY1vEEG(yuVUpXF*<`+bqNE#3})qyC+u9d9X>5cWVH9nq7Ovxt@gYGu~~oU%FQ5 z^3=J)otxYEuO){6yt7@1d5+JtslQ}m7C(QkxQH_#Cdcjnhx^Ai?UrZ$Ajo*dKKa@% zzmFz`8x=3huw2+-ui9V`%vAU`S?bZ;8G8KsjmazJi`R;tJ+(O4`$HvT**S6PeZ8xh z`DfnIVy##_M~%UKA6ixWnsT zCq|`7E}#twIr{Ab}T%%w@I@>WzAM9F^GBDI(Ag1XHk?b_0R zitILP-TWE-Zavkr6LV+ZdSxl&r6~D5IlE$oTWXam%yd@W)0 z?VB%T*vGcr&w9go@o@kA>5K0M#|5%oSifR3P2sSL?SnzRO zE5n5iuD5)vlW#>cuIIUMy-~PfuJvh#=qjgm>;m=k9s26i<-Rja{oZN8Y&HEr+b#u% zzcLF9nF{0*(lxHAeJr*3-do<(qPvoL@?x`yBYU~a>!P%H_spw$UaK@Wo-KI$*Os0; zbK0Hf>%^okb^Q7%k?*YvgL#(dfz5mE=6XI7zPUoO;pV!GFn>m`8z(l+V-m5yyJ;Gu zTJ{=eLr=~R|1Ri17H%vFZ+HA{wf_>s-c>&5-|>GivJ>LrtPy5&_FS_z&F8$JWJ%e5hThXZTomqJm0RG&q(3Lr zV7gH0ZsV|B;p90+sq3r~*CNWjC5{S9X2vre-z=w~ z5zqAf^s`dNRnHlA`!VQkIrLKM^~sQ^W7`k0IoWiE1+gD{e<-oHbo;H2Mf2`e-1x!3 z`cDIa{;miChMh=GV_E)nS)Qy|wxUFVP z<8)aP%5c4v(m|eOgMe94gC2i@X3i6iSosBF4XJBccL$eT zCQ3wzpUaZHq#FJ*<8;IA=SOS;L^g4*n^uzAut8KIj+^O%y%Cc{qu1uen{|Jm-2YwV z!LB{~IVSWpy%1h#>b6Mq-0>-EZPmCAGi7L&n&cGJaQVKw|NkMYpFy^5*koDW!YezL zp4gH3I96mT;~qx7&v{${m6tp2{(89F+PHUbYH>g3^+pki1OC#*7cxIAWchYDpzI^} zkDl!L)#j~i^{fGjO{+sCBiRl3r4x78F?48q>{j5g-8SK@t=f)x9L!5XRk#=meZC7h zY&xy^kE@qqk{AP1MFSh(hZ!oW6T}=fm`binF`DF^mYB8oRO4Fz&ZE* zS=T3Tnr~moT2#VWQmrS%ThAQG*|LU3>(%`l#`AHe?{*dRZjqY*#q3!qXTVy96%R8S zs<;A-5A5L6VY<@Jv}D7?ZFZTJi0$!2Z%RUrU zeyiiZ&i`gM!D&A=Dt1Mv#r|?9oF3{d3HH#-j@Z3 zzTT{S>KnJR_P9a6UOaQ$>IZFsTdr+1s(I*Ji5W;hDhGY7@`TmvK}3vE+gEA`8lEWp*F4 zno@3TtJ|38X3qQO>33ndw>P=c8Mby5j*8KvsK`K&eIoG!F+qjED_RMFk@%U%CEyiQ+`d~BCi!P?L4>NlMyn?BenpK;gV z@os}%b`?**w^?0UP-Uckzz)=jS2!@aGO%pY4;QmnTf=rZx~*H)%b=Q*qU`W-QV!#c zmfjY%X}>pJbeyxp^SAB_wfp?~v1JY07_KCW%=osZge73*53aM(QmZ^)=$(C6d4*-c zRpExIS--8exmq)t$Svq(C`)ZPZLa_7Gvg+8mhJBx4oY;fJ!@Q8A=J!JDLLuu`;{{+ z3b)L0{ml?^PkG^i-Od-ToLjtG(Px*SYGkhFOslAKi>5P73zI&{aB0c!j|fE>$QKFxQWM8_rQ^hMPhF`CfF)YI~T#u$9Z6d zj@z<;z5Dpifb^I(ByJTndy>6qv52nfJcc9IuUD|FGcEM}%aVT4tRahWK^2q0>jU@s z45D*cda|olioRRMuvz7G^h3SUEpyzX8PrzyPEcprUUi*chB3*8v#rF2?G5{?gN$!p z|7!d@!7A$9;%{tMi~l|;JxAUw^A^rSev>1gV~htZfbR zZJD`Fuf8qxWB$j-`M-Wh|NHkx$YQ3>X$F7R7GbTWjf&@Xg-6BRoo0HQtN)FT8_SQk z4F4{>+-EQRJd4re(#MoY`v-TPMLlm`ba5X~NJ8nJEg$L`wfF4*SIF?c`{;&L&Odo& z&m}{d=ICwVd-Jg2(x+D}1uAW&5A<1XEM-XCe3uP;OU@~nykW01-~G8~ z8l%`!?Y#f@ySD6p@%sfk*MZ(u>=92xL>i>O-e$}*bLVzge|^tnrVFd*JebL%kabq- z^-^gGwhpz!KeK~h=Ggy8S;R5rkK>y6k94K8WE~0)Us_(Ttu{5((Y{4s``-A^0K=wKtV3Hstvf^IVwb_1gcS zk?%R1@SvArCf{wNQ;y#fu+`Rd5N~Xns-ftw_T<;Sz9oLzjCyM~p7ZvuVQjUzpz_a3 zzwzI6FIEi}U5&UnWupq_W7bnS0wz|OHE%M4icKe}wKbs=GUmw^a`poLX6NdJsrq}N7N^)j6 zVY~OEt{KCdzpLa+%8oeli7coPdc_{3{AjTU&-y8nN&yOAm=)(vcwNn|7Fu<3K~#nK zjEi2`XWpDn*t*X=E?yVh7FombR{)egcvsE-D$V##O5ziQe9fS@e$lzk!i%QN0bbKz z*)sT^h~Ff(dpcW`+%(td?uSyTsSICJBJAv4)H`=`Zp@kf#jat>!Y{703x&~jvbhFC(S=+}AE6!z~9zw%(qf$%H$-nuPR$!Cyw z&d>NM>di#`G>3-AziSKjEvQ!YIa~TY+r;*w%mIVbZC=;%|8gD+Hk+QJU4CStO1@@e z;6EqX7wW$|-Z*X%<9z;)dr`@rvTwnL2ez(WkEE<;NYXUX*?ssc_LC!^~Q9ggAL@@2F<%{4X0PHSgtxjYUhXg6T4^aos`eYAu!e9!8UXK z#tG9L9`GGs%`iPYP`qKM8pHfGi}vjkwPSR!&1_9>ICj0!YAy52)vH#i9`ZcBdWCkh z^d*_F#Fcz~5(jT?+!QeByNQzR^YdSs+cQq-esQ-xZ?djis3`xgwqopmr2{TUjrN3W z_?q(gdqlFAP=odsh7E6iMKBw*O!|}6Q2pXMd8Sa24boPraD3mvuz_Y7q` zOc@w>nLuj+$WITSgam`#txf}c{{wFvYUv0>!ES0TzFx%tRFN-~*YQABC<)sPiFMb_J-ZJZ_ z@ZN{9GujR;k(0grrg8nO_%FLs^0yq@l2fHD%P}E+YtE|t{iS&g+4aKjwHw{nCM(%Z z?46a}TE1&9!yO4TB?j|<$Lg&Q2$)^E$UaZQykKecT6T}m$|}b#t|`kDa*JG^^|r)@ zlj-NTmv>?>$82;xml}TA(W}S&>BNgy6ZvMoP`uS|na>w+{JigGLAmGm`d_dx`^@|O z_VPvEFovGD|Buer%hWyavL%u8&d~6OaGuQwq=!?0dd>b8cnBk2X0yZo7%I`!hBc|Hzmg9P|A0ii&S~ zEESKR-d7Gkb*g6ldghx_i(}7Sj5%^o(0;L7-F&Tv6x*MN?q=Lr{fYIYexuU^64AQ9EbY&*C7$osNZ@hAJQt!4UTzq?Lb zcy~HO=BknM`Jf@3EO}t4|bupCzF% z(Sw0+P6nI8xl>2^zt=sKQJ8v@MWJunwRdbkmTJdr^E=N_cAnu^!P3263YAyYR4-($ zV{u~H5X0Vf-z$pgt?$iudHc7RSu=;EGrYL1<|kAoQ^mGm9m9&GhN|?2Jzc+K4ydpF zP<8ftqo=kFqr|jqe$O<9bK4H7Mae9^EtJ4-y6{u#wg{rj-^)wSyK@4*s=Nj3Hb2WDp-4Q^U{ z=KJYy4u4IKW?s3nL!ZI?l{iDrYuU`4OaE(hnRDJ0{bGTP*=^-pP$-{xrgG9c{X3^} z&fnT}Be~dBAc3qBP^O29dw=y3sd3aqRf9pe@qv}#mleL0vs;wF} zFbJ?bSfs^}7QW@AyV~SAb6a|S#Q!jSIKA4w>)yn_#`7#QOP-t!c#wWZ$=EewUCQ?w zk7y>meSWzp81n+KoV?$%^wMEc;zj?YfNKosYTUvFVuA*$a;t zW<6$tENGY2{)K(p&8hEM_vv`qEZ#J&pg53Fm>{Dz1X-PhIT-{29BV zQzHG3A1f)2wYsciUEEdmtljSW&anPV*IQL4C!a0hf4zUk&ye>QTp8+0)vDgj+#$fZf&#r6b zSRmH$`mxIv2A%BOIUOI@7BFP6JosfHIC-6LQyf3HfCbaqwQhN4XAZYcn&Vaa&Ec+= z^!2F>cUEXM@WygapE~1w!}w)e)TSt|Ed z^?&UQ*!7IT{@{{$kEB0=$_&Ah-`(v$b|ySBYaFo2_53dbTFrPLRzbeX+&FxnDUMyzM43TxeyEEM7gEAuT;t{J|#w7RF1F z>Fe1Tcid7}Vle;ZBJtqMn{&Ig4y4-&zHo0yw!aX-`*XqO!}o6f$(Zl)@xl6u40l$@ z)+klYpA!?3bm=~*)_Np-Q{!W2&Xqbl!Hk=8{;i`Q;q(5$;(IMx0-O**2^ z#AGn3u>MQde2#Mw7q(aMmP!7UwVSBcCNX_t|M$+4SZ!~yS4GcsE>+F%xWu$!!B;!M zSq$zB%c9;3PZWx{QZ>KhN~7z`s`(vxlg_SYig_=5ux>%v*_*s*_+|~wcqSfk1xFcxt7!6sJGlQhSD#mgJZ!y>3>Hgd2Q}sQx z+omvlx}{hcrq6ysbcc)&#P=vT?B!Zu#q4nI0Vm_D zaK@&|ZL@W^H#jN!O!9a6o+6oZFI;z)NALZcW)~~J z_MY87?#EIZzuJXnFO#%SShi7o>7w(`U*1z=+!45()#LHKdC%VGJbvt=G|@vvsBOe8^wmbASBZJ`i*y59bIvZeT08YN({9yO z0g?}#x43;~a9!=n@SC0ajr{v_m$n{J6{z8?Deuz*c|3?c@5+te(ryjO@7HgSUa(YN z1M2mt)f<0Hyq)n}e2vGXRolvct*CSQBN~ zRK0M=uIt}}PT{HNT)^PWRei6grh}o)FuQzTD zzjCed>a+{DyefaDZF>;N8FQQK^)7XXr=m0F&tmWS|2Z^#g>Cesjq*1n7RZ(d6}GNo zi|CGd&l3=Sy|L8)dTH(B@EYsV@E5o84ldfj;+i#3lNpGB5mQc2J6e-tUWhGoY7^d;5%{haqGGpub}b>-yyLfHfNrks=0 zt(lnhDe(E)8lD-F>D-&kHxw%Im%lNKcpX=gedp@25APmN6+bukiok<_$^0@a`pZ~l z7Fy+%o>LR7wpg_J@wA_X)5QO>YU}FjUJ*zL6|9N4H|xwzxdl}`QcdlLBO{d>&b=wm z=M-R;7q^RIlF|C~(kp>)LZ2<;3VYE6U%Sax0)M4g8y?3Rb{%2-V_0^bG3NRIjD?r4 zH2(0E>pfRDjiK$6W!~N1yP-LZ=GK!VS8m>Nh`BC|k>y8DgL7?KT2!LO_Cs&}U+{f> z!24vA{4svhx*HvWuh?&Wzqj!*``+LCCC}S!n4YmizN6XsSNZdQDwmEf-uXE;X-3-8 zZgh z&&bVNqofO6C;A0h{=Oi( zWKms$uESmq1C^f{4IlJ<7}u>~j%WB$S*oya!Kd8@-Pt7zIf#Wnf@n N@O1TaS?83{1OUf=W6b~n diff --git a/vm-tests/default.nix b/vm-tests/default.nix index e516868..d546d1a 100644 --- a/vm-tests/default.nix +++ b/vm-tests/default.nix @@ -34,18 +34,10 @@ in ]; }; - checksFor = pkgs: { - vm-boot-and-shutdown = pkgs.testers.runNixOSTest ( - import ./slow-services/check.nix { machine = machines.slow-services; } - ); - vm-luks-passphrase = pkgs.testers.runNixOSTest ( - import ./encrypted-volume/check.nix { - machine = machines.encrypted-volume; - inherit pkgs; - } - ); - vm-failures = pkgs.testers.runNixOSTest ( - import ./failures/check.nix { machine = machines.failures; } - ); - }; + checksFor = + pkgs: + lib.mapAttrs ( + machineName: machine: + pkgs.testers.runNixOSTest (import ./${machineName}/check.nix { inherit machine pkgs; }) + ) machines; } diff --git a/vm-tests/encrypted-volume/check.nix b/vm-tests/encrypted-volume/check.nix index 086467b..b3174e5 100644 --- a/vm-tests/encrypted-volume/check.nix +++ b/vm-tests/encrypted-volume/check.nix @@ -3,16 +3,15 @@ let secretDisk = import ./secret-disk.nix { inherit pkgs; }; in { - name = "bootvine-luks-passphrase"; + name = "bootvine-encrypted-volume"; nodes.machine = { - imports = [ machine ]; + imports = [ + machine + ../shared/recorded-display.nix + ]; testing.initrdBackdoor = true; - virtualisation.qemu.options = [ - "-vga none" - "-device virtio-vga,xres=1280,yres=800" - ]; virtualisation.qemu.drives = [ secretDisk.qemuDrive ]; }; @@ -20,5 +19,6 @@ in PASSPHRASE = "${secretDisk.passphrase}" '' + builtins.readFile ../shared/console.py + + builtins.readFile ../shared/screen_recording.py + builtins.readFile ./check.py; } diff --git a/vm-tests/encrypted-volume/check.py b/vm-tests/encrypted-volume/check.py index 795a697..218cf59 100644 --- a/vm-tests/encrypted-volume/check.py +++ b/vm-tests/encrypted-volume/check.py @@ -7,7 +7,9 @@ def pending_prompt(): return next((name for name in listing if name.startswith("ask.")), "") +recording = ScreenRecording(machine) machine.start() +recording.start() retry(lambda _: pending_prompt() != "", timeout_seconds=60) wait_for_screen("passphrase") @@ -18,7 +20,6 @@ retry(lambda _: answered_prompts() == 1, timeout_seconds=30) retry(lambda _: pending_prompt() not in ("", first_prompt), timeout_seconds=30) wait_for_screen("passphrase") assert_text_visible(font_has_512_glyphs=True) -machine.screenshot("passphrase") machine.send_chars(f"{PASSPHRASE}\n") machine.wait_for_unit("systemd-cryptsetup@secret.service") machine.switch_root() @@ -29,3 +30,4 @@ journal = bootvine_journal() print(journal) assert " in Initrd " in journal, "never ran in the initrd" assert journal.count("answered a password prompt") == 2, "did not answer both attempts itself" +recording.stop() diff --git a/vm-tests/failures/check.nix b/vm-tests/failures/check.nix index ac18a18..41a0ae5 100644 --- a/vm-tests/failures/check.nix +++ b/vm-tests/failures/check.nix @@ -1,4 +1,4 @@ -{ machine }: +{ machine, ... }: { name = "bootvine-failures"; @@ -9,7 +9,10 @@ marker: "${pkgs.bash}/bin/bash -c 'until [ -e /run/${marker} ]; do sleep 0.1; done'"; in { - imports = [ machine ]; + imports = [ + machine + ../shared/recorded-display.nix + ]; bootvineSample.startFailuresByHand = true; testing.initrdBackdoor = true; @@ -36,14 +39,8 @@ }; }; - virtualisation = { - memorySize = 2048; - qemu.options = [ - "-vga none" - "-device virtio-vga,xres=1280,yres=800" - ]; - }; }; - testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py; + testScript = builtins.readFile ../shared/console.py + + builtins.readFile ../shared/screen_recording.py + builtins.readFile ./check.py; } diff --git a/vm-tests/failures/check.py b/vm-tests/failures/check.py index c61243c..2c6e4b2 100644 --- a/vm-tests/failures/check.py +++ b/vm-tests/failures/check.py @@ -12,7 +12,9 @@ def failure_panel(): return "\n".join(lines[start:]) +recording = ScreenRecording(machine) machine.start() +recording.start() wait_for_screen("Initrd", "Initrd held for the check") start_failing_unit("initrd-nic-firmware-load") @@ -52,8 +54,8 @@ wait_for_screen("upstream-uplink exited with status 1", "lost connection to ups wait_for_screen("× 9 failed", "connection refused") assert "Started Upstream uplink" not in failure_panel(), f"systemd's own start line reached the panel:\n{screen()}" print(screen()) -machine.screenshot("failures") machine.succeed("touch /run/release-boot-failures-hold") retry(lambda _: "handing the console over" in bootvine_journal(), timeout_seconds=60) print(bootvine_journal()) +recording.stop() diff --git a/vm-tests/scripted-initrd/check.nix b/vm-tests/scripted-initrd/check.nix new file mode 100644 index 0000000..f588fd4 --- /dev/null +++ b/vm-tests/scripted-initrd/check.nix @@ -0,0 +1,25 @@ +{ machine, ... }: +{ + name = "bootvine-scripted-initrd"; + + nodes.machine = + { lib, ... }: + let + waitFor = flag: "/bin/sh -c 'until [ -e /run/${flag} ]; do sleep 0.1; done'"; + in + { + imports = [ + machine + ../shared/recorded-display.nix + ]; + systemd.services = { + sample-database.serviceConfig.ExecStop = lib.mkForce (waitFor "release-sample-database"); + backdoor.before = [ "sample-database.service" ]; + }; + }; + + testScript = + builtins.readFile ../shared/console.py + + builtins.readFile ../shared/screen_recording.py + + builtins.readFile ./check.py; +} diff --git a/vm-tests/scripted-initrd/check.py b/vm-tests/scripted-initrd/check.py new file mode 100644 index 0000000..c4cb06c --- /dev/null +++ b/vm-tests/scripted-initrd/check.py @@ -0,0 +1,28 @@ +def unit_state(unit): + return machine.succeed(f"systemctl is-active {unit} || true").strip() + + +recording = ScreenRecording(machine) +machine.start() +recording.start() + +wait_for_screen("Stage 2", "sample-database") +assert_bootvine_in_front() +assert_text_visible(font_has_512_glyphs=True) + +machine.wait_for_unit("greetd.service") +retry(lambda _: unit_state("bootvine") == "inactive", timeout_seconds=60) +journal = bootvine_journal() +print(journal) +watched = [line for line in journal.splitlines() if line.startswith("bootvine: watching")] +assert watched, "bootvine never started" +assert not any(" in Initrd " in line for line in watched), "bootvine ran in a scripted initrd" +assert "handing the console over" in journal, "never handed the console over" + +machine.succeed("systemctl poweroff --no-block") +retry(lambda _: unit_state("sample-database") == "deactivating", timeout_seconds=60) +wait_for_screen("Shutdown", "sample-database") +assert_bootvine_in_front() +machine.succeed("touch /run/release-sample-database") +machine.wait_for_shutdown() +recording.stop() diff --git a/vm-tests/shared/recorded-display.nix b/vm-tests/shared/recorded-display.nix new file mode 100644 index 0000000..c5968a7 --- /dev/null +++ b/vm-tests/shared/recorded-display.nix @@ -0,0 +1,11 @@ +{ + virtualisation = { + memorySize = 2048; + qemu.options = [ + "-vga none" + "-device virtio-vga,xres=1280,yres=800" + "-chardev socket,id=recording-monitor,path=recording-monitor,server=on,wait=off" + "-mon chardev=recording-monitor,mode=readline" + ]; + }; +} diff --git a/vm-tests/shared/screen_recording.py b/vm-tests/shared/screen_recording.py new file mode 100644 index 0000000..9fc4a1f --- /dev/null +++ b/vm-tests/shared/screen_recording.py @@ -0,0 +1,74 @@ +import os +import socket +import subprocess +import threading +import time + +RECORDING_FRAME_SECONDS = 0.5 +RECORDING_CONNECT_SECONDS = 30 +QEMU_PROMPT = b"(qemu) " + + +class ScreenRecording: + def __init__(self, recorded_machine): + self.recorded_machine = recorded_machine + self.frames_directory = os.path.join(recorded_machine.out_dir, "recording") + self.raw_frame = os.path.join(recorded_machine.state_dir, "recording-frame.ppm") + self.stopping = threading.Event() + self.thread = threading.Thread(target=self.record_frames, daemon=True) + + def start(self): + os.makedirs(self.frames_directory, exist_ok=True) + self.thread.start() + + def stop(self): + self.stopping.set() + self.thread.join() + + def connect_monitor(self): + monitor_path = os.path.join(self.recorded_machine.state_dir, "recording-monitor") + deadline = time.monotonic() + RECORDING_CONNECT_SECONDS + while time.monotonic() < deadline and not self.stopping.is_set(): + monitor = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + monitor.connect(monitor_path) + return monitor + except OSError: + monitor.close() + time.sleep(0.1) + return None + + def read_until_prompt(self, monitor): + received = b"" + while not received.endswith(QEMU_PROMPT): + chunk = monitor.recv(4096) + if not chunk: + raise ConnectionError("the recording monitor closed") + received += chunk + + def save_frame(self, monitor, frame_number): + monitor.sendall(f"screendump {self.raw_frame}\n".encode()) + self.read_until_prompt(monitor) + frame_path = os.path.join(self.frames_directory, f"frame-{frame_number:05d}.png") + unfinished_path = f"{frame_path}.unfinished" + subprocess.run(f"pnmtopng '{self.raw_frame}' > '{unfinished_path}'", shell=True, check=True) + os.replace(unfinished_path, frame_path) + + def record_frames(self): + monitor = self.connect_monitor() + if monitor is None: + return + try: + self.read_until_prompt(monitor) + frame_number = 0 + while not self.stopping.is_set(): + self.save_frame(monitor, frame_number) + frame_number += 1 + self.stopping.wait(RECORDING_FRAME_SECONDS) + except (OSError, ConnectionError, subprocess.CalledProcessError): + pass + finally: + monitor.close() + for leftover in os.listdir(self.frames_directory): + if leftover.endswith(".unfinished"): + os.remove(os.path.join(self.frames_directory, leftover)) diff --git a/vm-tests/slow-services/check.nix b/vm-tests/slow-services/check.nix index b8bdf04..29c7e01 100644 --- a/vm-tests/slow-services/check.nix +++ b/vm-tests/slow-services/check.nix @@ -1,6 +1,6 @@ -{ machine }: +{ machine, ... }: { - name = "bootvine-boot-and-shutdown"; + name = "bootvine-slow-services"; nodes.machine = { lib, ... }: @@ -8,7 +8,10 @@ waitFor = flag: "/bin/sh -c 'until [ -e /run/${flag} ]; do sleep 0.1; done'"; in { - imports = [ machine ]; + imports = [ + machine + ../shared/recorded-display.nix + ]; testing.initrdBackdoor = true; boot.initrd.systemd.services.initrd-hardware-probe.serviceConfig.ExecStart = lib.mkForce ( waitFor "release-initrd-hardware-probe" @@ -26,14 +29,8 @@ }; }; }; - virtualisation = { - memorySize = 2048; - qemu.options = [ - "-vga none" - "-device virtio-vga,xres=1280,yres=800" - ]; - }; }; - testScript = builtins.readFile ../shared/console.py + builtins.readFile ./check.py; + testScript = builtins.readFile ../shared/console.py + + builtins.readFile ../shared/screen_recording.py + builtins.readFile ./check.py; } diff --git a/vm-tests/slow-services/check.py b/vm-tests/slow-services/check.py index f327ed6..24cecc4 100644 --- a/vm-tests/slow-services/check.py +++ b/vm-tests/slow-services/check.py @@ -6,7 +6,9 @@ def unit_state(unit): return machine.succeed(f"systemctl is-active {unit} || true").strip() +recording = ScreenRecording(machine) machine.start() +recording.start() wait_for_screen("Initrd", "Slow hardware probe") assert_bootvine_in_front() @@ -56,3 +58,4 @@ assert_bootvine_in_front() assert_text_visible(font_has_512_glyphs=True) machine.succeed("touch /run/release-sample-database") machine.wait_for_shutdown() +recording.stop() diff --git a/vm-tests/text-console/check.nix b/vm-tests/text-console/check.nix new file mode 100644 index 0000000..d8e89a9 --- /dev/null +++ b/vm-tests/text-console/check.nix @@ -0,0 +1,29 @@ +{ machine, ... }: +{ + name = "bootvine-text-console"; + + nodes.machine = + { lib, ... }: + let + waitFor = flag: "/bin/sh -c 'until [ -e /run/${flag} ]; do sleep 0.1; done'"; + in + { + imports = [ + machine + ../shared/recorded-display.nix + ]; + testing.initrdBackdoor = true; + boot.initrd.systemd.services.initrd-hardware-probe.serviceConfig.ExecStart = lib.mkForce ( + waitFor "release-initrd-hardware-probe" + ); + systemd.services = { + sample-database.serviceConfig.ExecStop = lib.mkForce (waitFor "release-sample-database"); + backdoor.before = [ "sample-database.service" ]; + }; + }; + + testScript = + builtins.readFile ../shared/console.py + + builtins.readFile ../shared/screen_recording.py + + builtins.readFile ./check.py; +} diff --git a/vm-tests/text-console/check.py b/vm-tests/text-console/check.py new file mode 100644 index 0000000..a754799 --- /dev/null +++ b/vm-tests/text-console/check.py @@ -0,0 +1,31 @@ +def unit_state(unit): + return machine.succeed(f"systemctl is-active {unit} || true").strip() + + +recording = ScreenRecording(machine) +machine.start() +recording.start() + +wait_for_screen("Initrd", "Slow hardware probe") +assert_bootvine_in_front() +machine.succeed("touch /run/release-initrd-hardware-probe") +machine.switch_root() + +wait_for_screen("Stage 2", "slow-warm-up") +assert_bootvine_in_front() +assert_text_visible(font_has_512_glyphs=True) + +retry(lambda _: unit_state("bootvine") == "inactive", timeout_seconds=60) +journal = bootvine_journal() +print(journal) +assert "handing the console over" in journal, "never handed the console over" +assert foreground_terminal() != f"tty{BOOTVINE_TERMINAL}", "bootvine kept the console without a display manager" +machine.wait_until_tty_matches("1", "login: ") + +machine.succeed("systemctl poweroff --no-block") +retry(lambda _: unit_state("sample-database") == "deactivating", timeout_seconds=60) +wait_for_screen("Shutdown", "sample-database") +assert_bootvine_in_front() +machine.succeed("touch /run/release-sample-database") +machine.wait_for_shutdown() +recording.stop() diff --git a/xtask/src/main.rs b/xtask/src/main.rs index f61c226..f627c3d 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -16,8 +16,8 @@ struct RepoTasks { #[derive(Subcommand)] enum Task { #[command( - about = "Record the screencasts with vhs and copy the passphrase and failures screenshots \ - into recordings/" + about = "Record the screencasts with vhs and convert the passphrase and failures \ + screenshots to WebP in recordings/" )] RecordScreencasts(RecordScreencastsOptions), #[command( @@ -28,19 +28,18 @@ enum Task { #[derive(Args)] struct RecordScreencastsOptions { - #[arg(long, help = "passphrase.png produced by the vm-luks-passphrase check")] - passphrase_png: PathBuf, - #[arg(long, help = "failures.png produced by the vm-failures check")] - failures_png: PathBuf, + #[arg( + long, + help = "Directory with one entry per sample machine, each the output of its vm- \ + check" + )] + vm_check_outputs: PathBuf, } fn main() -> ExitCode { let outcome = match RepoTasks::parse().task { Task::RecordScreencasts(options) => { - record_screencasts::record_screencasts(&record_screencasts::CheckScreenshots { - passphrase_png: &options.passphrase_png, - failures_png: &options.failures_png, - }) + record_screencasts::record_screencasts(&options.vm_check_outputs) } Task::SyncDefaultConfig => sync_default_config::sync_default_config(), }; diff --git a/xtask/src/record_screencasts.rs b/xtask/src/record_screencasts.rs index ec2ea50..d649cc7 100644 --- a/xtask/src/record_screencasts.rs +++ b/xtask/src/record_screencasts.rs @@ -1,7 +1,7 @@ use std::{ - fs::{self, Permissions}, - os::unix::fs::PermissionsExt, - path::Path, + fs::{self, File}, + io::Read, + path::{Path, PathBuf}, }; use xshell::{Shell, cmd}; @@ -12,27 +12,100 @@ const TAPES: [&str; 2] = [ "scripted-demos/tapes/scripted-boot.tape", "scripted-demos/tapes/scripted-shutdown.tape", ]; +const RECORDING_FRAME_MILLISECONDS: &str = "500"; -pub struct CheckScreenshots<'a> { - pub passphrase_png: &'a Path, - pub failures_png: &'a Path, +fn sorted_entries(directory: &Path) -> Result, TaskError> { + let read_failure = |source| TaskError::File { + path: directory.to_path_buf(), + source, + }; + let mut entries = fs::read_dir(directory) + .map_err(read_failure)? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>() + .map_err(read_failure)?; + entries.sort(); + Ok(entries) } -fn copy_screenshot( - screenshot: &Path, - recordings: &Path, - published_name: &str, -) -> Result<(), TaskError> { - let copied_png = recordings.join(published_name); - fs::copy(screenshot, &copied_png) - .and_then(|_| fs::set_permissions(&copied_png, Permissions::from_mode(0o644))) +#[derive(PartialEq, Eq)] +struct FrameSize { + width: u32, + height: u32, +} + +fn png_frame_size(frame: &Path) -> Result { + let mut header = [0_u8; 24]; + File::open(frame) + .and_then(|mut file| file.read_exact(&mut header)) .map_err(|source| TaskError::File { - path: copied_png, + path: frame.to_path_buf(), source, - }) + })?; + let big_endian = |offset: usize| { + u32::from_be_bytes([ + header[offset], + header[offset + 1], + header[offset + 2], + header[offset + 3], + ]) + }; + Ok(FrameSize { + width: big_endian(16), + height: big_endian(20), + }) } -pub fn record_screencasts(screenshots: &CheckScreenshots) -> Result<(), TaskError> { +fn frames_at_final_size(frames: Vec) -> Result, TaskError> { + let Some(final_frame) = frames.last() else { + return Ok(frames); + }; + let final_size = png_frame_size(final_frame)?; + let sizes = frames + .iter() + .map(|frame| png_frame_size(frame)) + .collect::, _>>()?; + Ok(frames + .into_iter() + .zip(sizes) + .filter(|(_, size)| *size == final_size) + .map(|(frame, _)| frame) + .collect()) +} + +fn encode_machine_recording( + shell: &Shell, + check_output: &Path, + recordings: &Path, +) -> Result<(), TaskError> { + let Some(machine_name) = check_output.file_name() else { + return Ok(()); + }; + let frames = frames_at_final_size( + sorted_entries(&check_output.join("recording"))? + .into_iter() + .filter(|frame| { + frame + .extension() + .is_some_and(|extension| extension == "png") + }) + .collect(), + )?; + if frames.is_empty() { + return Err(TaskError::NoRecordingFrames(check_output.to_path_buf())); + } + let animation = recordings.join(machine_name).with_extension("webp"); + let frame_duration = RECORDING_FRAME_MILLISECONDS; + cmd!( + shell, + "img2webp -loop 0 -min_size -mixed -d {frame_duration} {frames...} -o {animation}" + ) + .quiet() + .run()?; + Ok(()) +} + +pub fn record_screencasts(vm_check_outputs: &Path) -> Result<(), TaskError> { let recordings = Path::new("recordings"); if !recordings.is_dir() { return Err(TaskError::NotInRepositoryRoot); @@ -42,6 +115,7 @@ pub fn record_screencasts(screenshots: &CheckScreenshots) -> Result<(), TaskErro let tape_path = Path::new(tape); cmd!(shell, "vhs {tape_path}").run() })?; - copy_screenshot(screenshots.passphrase_png, recordings, "passphrase.png")?; - copy_screenshot(screenshots.failures_png, recordings, "failures.png") + sorted_entries(vm_check_outputs)? + .iter() + .try_for_each(|check_output| encode_machine_recording(&shell, check_output, recordings)) } diff --git a/xtask/src/task_error.rs b/xtask/src/task_error.rs index 88213d8..840fa83 100644 --- a/xtask/src/task_error.rs +++ b/xtask/src/task_error.rs @@ -8,6 +8,8 @@ pub enum TaskError { Shell(#[from] xshell::Error), #[error("{}: {source}", path.display())] File { path: PathBuf, source: io::Error }, + #[error("{} has no recording frames", .0.display())] + NoRecordingFrames(PathBuf), #[error("no recordings directory here; run from the repository root")] NotInRepositoryRoot, #[error("{0} has no ```toml block under its Defaults heading")]