Author SHA1 Message Date
Willem Vanhulle ce72695e29 ci: move CI scripting into Nushell scripts with separate dependency steps
Replace the shell wrappers build-and-push-to-cachix, check-conventional-commits
and the release-plz apps with Nushell scripts in ci/, and move the git commands
out of the workflow YAML, so that every workflow command is a plain nix call.

Build the workspace dependencies in their own step, skipped when already in
Cachix. Later steps import the exported paths and also run after an earlier
failure. A separate step alone holds the Cachix token and uploads on main.
Pull requests run the same steps without the upload.

Wrap the recording tasks with makeBinaryWrapper instead of shell, and run the
CI task tests in a step of their own.
2026-11-04 20:02:26 +01:00
7 changed files with 306 additions and 122 deletions

No files matched your search

-2
View File
@@ -34,8 +34,6 @@ steps:
image: nixos/nix:latest image: nixos/nix:latest
environment: *release_env environment: *release_env
commands: commands:
- git remote set-url origin "https://wvhulle:$${CODEBERG_TOKEN}@codeberg.org/$${CI_REPO}.git"
- git checkout -B main "$${CI_COMMIT_SHA}"
- nix run --quiet .#release-plz-release - nix run --quiet .#release-plz-release
- name: update-release-pr - name: update-release-pr
+30 -18
View File
@@ -8,6 +8,8 @@ variables:
experimental-features = nix-command flakes experimental-features = nix-command flakes
build-users-group = build-users-group =
accept-flake-config = true accept-flake-config = true
- &after_dependency_step
- status: [success, failure]
- &on_pull_requests - &on_pull_requests
- event: pull_request - event: pull_request
@@ -22,6 +24,7 @@ steps:
- name: build-and-test-bootvine - name: build-and-test-bootvine
image: nixos/nix:latest image: nixos/nix:latest
depends_on: [build-workspace-dependencies] depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config environment: *nix_config
commands: commands:
- nix run --quiet .#import-exported-paths - nix run --quiet .#import-exported-paths
@@ -30,11 +33,30 @@ steps:
- name: build-repo-tasks - name: build-repo-tasks
image: nixos/nix:latest image: nixos/nix:latest
depends_on: [build-workspace-dependencies] depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config environment: *nix_config
commands: commands:
- nix run --quiet .#import-exported-paths - nix run --quiet .#import-exported-paths
- nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.repo-tasks-build - nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.repo-tasks-build
- name: lint-rust-with-clippy
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints
- name: check-rust-formatting
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting
- name: upload-build-cache - name: upload-build-cache
image: nixos/nix:latest image: nixos/nix:latest
depends_on: [build-and-test-bootvine, build-repo-tasks] depends_on: [build-and-test-bootvine, build-repo-tasks]
@@ -49,22 +71,6 @@ steps:
commands: commands:
- nix run --quiet .#upload-exported-paths - nix run --quiet .#upload-exported-paths
- name: lint-rust-with-clippy
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints
- name: check-rust-formatting
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting
- name: evaluate-flake-outputs - name: evaluate-flake-outputs
image: nixos/nix:latest image: nixos/nix:latest
depends_on: [] depends_on: []
@@ -72,11 +78,17 @@ steps:
commands: commands:
- nix flake check --quiet --no-build - nix flake check --quiet --no-build
- name: test-ci-tasks
image: nixos/nix:latest
depends_on: []
environment: *nix_config
commands:
- nix build --quiet --print-build-logs .#checks.x86_64-linux.ci-task-tests
- name: check-conventional-commit-messages - name: check-conventional-commit-messages
image: nixos/nix:latest image: nixos/nix:latest
depends_on: [] depends_on: []
when: *on_pull_requests when: *on_pull_requests
environment: *nix_config environment: *nix_config
commands: commands:
- git fetch --quiet origin "$${CI_COMMIT_TARGET_BRANCH}" - nix run --quiet .#check-conventional-commits -- --fetch-target-branch FETCH_HEAD..HEAD
- nix run --quiet .#check-conventional-commits -- FETCH_HEAD..HEAD
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env nu
const EXPORT_DIRECTORY_NAME = ".nix-built-paths"
const PATH_LIST_FILE_NAME = "paths.txt"
const CACHIX_CACHE_NAME = "wvhulle"
const CACHIX_CACHE_URL = "https://wvhulle.cachix.org"
def export-directory []: nothing -> path {
$env.CI_WORKSPACE | path join $EXPORT_DIRECTORY_NAME
}
export def store-paths-from-lines []: string -> list<string> {
lines | compact --empty
}
def exported-store-paths []: nothing -> list<string> {
let path_list_file = export-directory | path join $PATH_LIST_FILE_NAME
if ($path_list_file | path exists) {
open --raw $path_list_file | store-paths-from-lines
} else {
[]
}
}
export def nix-config-with-hook [existing_config: string, hook_program: string]: nothing -> string {
$"($existing_config)\npost-build-hook = ($hook_program)"
}
def "main export-built-paths-hook" []: nothing -> nothing {
let built_paths = $env.OUT_PATHS | split row " " | compact --empty
let directory = export-directory
mkdir $directory
^nix copy --quiet --to $"file://($directory)?compression=zstd" ...$built_paths
$built_paths | each {|built_path| $"($built_path)\n" } | str join | save --append ($directory | path join $PATH_LIST_FILE_NAME)
}
def --wrapped "main build-and-export-paths" [
...build_arguments: string
]: nothing -> nothing {
let hook_config = nix-config-with-hook ($env.NIX_CONFIG? | default "") $env.EXPORT_HOOK_PROGRAM
with-env {NIX_CONFIG: $hook_config} {
^nix build --quiet --print-build-logs ...$build_arguments
}
}
def "main build-unless-cached" [
installable: string
]: nothing -> nothing {
let built_output = ^nix eval --quiet --raw $"($installable).outPath"
let cache_lookup = do { ^nix path-info --quiet --store $CACHIX_CACHE_URL $built_output } | complete
if $cache_lookup.exit_code == 0 {
print $"($built_output) is already in Cachix"
} else {
main build-and-export-paths $installable
}
}
def "main import-exported-paths" []: nothing -> nothing {
let exported_paths = exported-store-paths
if ($exported_paths | is-empty) {
print "No exported paths to import"
} else {
^nix copy --quiet --no-check-sigs --from $"file://(export-directory)" ...$exported_paths
}
}
def "main upload-exported-paths" []: nothing -> nothing {
let exported_paths = exported-store-paths
if ($exported_paths | is-empty) {
print "No newly built paths to upload"
} else {
main import-exported-paths
^cachix push $CACHIX_CACHE_NAME ...$exported_paths
}
}
def main []: nothing -> nothing {
print "Run a subcommand, for example: build-and-export-paths"
}
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env nu
const CODEBERG_USER = "wvhulle"
const RELEASE_BRANCH_NAME = "main"
export def authenticated-origin-url [repository_slug: string, access_token: string]: nothing -> string {
$"https://($CODEBERG_USER):($access_token)@codeberg.org/($repository_slug).git"
}
def run-release-plz [release_subcommand: string]: nothing -> nothing {
^release-plz $release_subcommand --forge gitea --git-token $env.CODEBERG_TOKEN
}
def "main release-plz-release" []: nothing -> nothing {
^git remote set-url origin (authenticated-origin-url $env.CI_REPO $env.CODEBERG_TOKEN)
^git checkout -B $RELEASE_BRANCH_NAME $env.CI_COMMIT_SHA
run-release-plz release
}
def "main release-plz-release-pr" []: nothing -> nothing {
run-release-plz release-pr
}
def "main check-conventional-commits" [
--fetch-target-branch
...revision_ranges: string
]: nothing -> nothing {
if $fetch_target_branch {
^git fetch --quiet origin $env.CI_COMMIT_TARGET_BRANCH
}
^convco check ...$revision_ranges
}
def main []: nothing -> nothing {
print "Run a subcommand, for example: check-conventional-commits"
}
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env nu
use std/assert
use ./nix-built-paths.nu [nix-config-with-hook store-paths-from-lines]
use ./release-tasks.nu [authenticated-origin-url]
def main []: nothing -> nothing {
assert equal (nix-config-with-hook "max-jobs = 1" "/nix/store/hook") "max-jobs = 1\npost-build-hook = /nix/store/hook"
assert equal (nix-config-with-hook "" "/nix/store/hook") "\npost-build-hook = /nix/store/hook"
assert equal ("/nix/store/first\n\n/nix/store/second\n" | store-paths-from-lines) ["/nix/store/first" "/nix/store/second"]
assert equal ("" | store-paths-from-lines) []
assert equal (authenticated-origin-url "wvhulle/bootvine" "sample-token") "https://wvhulle:sample-token@codeberg.org/wvhulle/bootvine.git"
print "CI task tests passed"
}
+12 -3
View File
@@ -9,6 +9,7 @@
| `xtask/` | repository tasks run with `cargo xtask` | | `xtask/` | repository tasks run with `cargo xtask` |
| `vm-tests/` | the sample NixOS machines, one folder each with its VM check | | `vm-tests/` | the sample NixOS machines, one folder each with its VM check |
| `nix/` | the crane builds and the NixOS module | | `nix/` | the crane builds and the NixOS module |
| `ci/` | the Nushell scripts behind the CI apps in `flake.nix` |
| `recordings/` | ignored output of `record-screencasts`: the scripted demo GIFs and the sample machine WebP recordings, published on the `recordings` branch for the README and the [guide](guide.md) | | `recordings/` | ignored output of `record-screencasts`: the scripted demo GIFs and the sample machine WebP recordings, published on the `recordings` branch for the README and the [guide](guide.md) |
`cargo build` and `cargo test` act on every crate. `cargo run` needs the binary: `cargo build` and `cargo test` act on every crate. `cargo run` needs the binary:
@@ -46,9 +47,17 @@ shell) or `pre-commit install` ([pre-commit](https://pre-commit.com)).
`.woodpecker/rust-and-flake-checks.yaml` runs on Codeberg's `.woodpecker/rust-and-flake-checks.yaml` runs on Codeberg's
[Woodpecker CI](https://woodpecker-ci.org) for every push and pull request: the bootvine build with its tests, the xtask build, clippy, [Woodpecker CI](https://woodpecker-ci.org) for every push and pull request: the bootvine build with its tests, the xtask build, clippy,
rustfmt, an evaluation of every flake output, and the commit-message check on rustfmt, an evaluation of every flake output, and the commit-message check on
pull requests. Builds on `main` are pushed to the `wvhulle` [Cachix](https://www.cachix.org) cache, which pull requests. Every step is a plain `nix` call; the scripting lives in `ci/`
`nixConfig` in `flake.nix` offers as a substituter. The two VM checks need KVM, and is tested by `nix build .#checks.x86_64-linux.ci-task-tests`.
which Codeberg's runners lack, so they only run locally.
Steps do not share `/nix/store`. The first step builds the workspace dependencies
unless they are already in the `wvhulle` [Cachix](https://www.cachix.org) cache,
which `nixConfig` in `flake.nix` offers as a substituter, and exports what it built
to the workspace. The build and lint steps import those paths with
`nix run .#import-exported-paths` and run even when an earlier step failed. On
`main` a final step, the only one holding the Cachix token, pushes the exported
paths with `nix run .#upload-exported-paths`. The two VM checks need KVM, which
Codeberg's runners lack, so they only run locally.
## Releases ## Releases
+135 -99
View File
@@ -91,12 +91,6 @@
... ...
}: }:
let let
shellScriptApp = name: description: scriptArgs: {
type = "app";
meta.description = description;
program = lib.getExe (pkgs.writeShellApplication ({ inherit name; } // scriptArgs));
};
sampleMachineApps = lib.mapAttrs' ( sampleMachineApps = lib.mapAttrs' (
machineName: machineDescription: machineName: machineDescription:
lib.nameValuePair "sample-vm-${machineName}" { lib.nameValuePair "sample-vm-${machineName}" {
@@ -106,42 +100,80 @@
} }
) vmTests.machineDescriptions; ) vmTests.machineDescriptions;
exportBuiltPathsHook = pkgs.writeShellApplication { wrappedProgramTool =
name = "export-built-paths-hook"; {
text = '' toolName,
exported_paths_dir="$CI_WORKSPACE/.nix-built-paths" wrappedProgram,
mkdir -p "$exported_paths_dir" addedFlags,
read -ra built_paths <<< "$OUT_PATHS" runtimeInputs ? [ ],
nix copy --quiet --to "file://$exported_paths_dir?compression=zstd" "''${built_paths[@]}" wrapperArguments ? [ ],
printf '%s\n' "''${built_paths[@]}" >> "$exported_paths_dir/paths.txt" }:
''; pkgs.runCommand toolName
{
nativeBuildInputs = [ pkgs.makeBinaryWrapper ];
meta.mainProgram = toolName;
}
''
makeWrapper ${wrappedProgram} $out/bin/${toolName} \
${
lib.escapeShellArgs (
lib.concatMap (addedFlag: [
"--add-flags"
addedFlag
]) addedFlags
)
} \
${lib.optionalString (runtimeInputs != [ ]) "--prefix PATH : ${lib.makeBinPath runtimeInputs}"} \
${lib.escapeShellArgs wrapperArguments}
'';
nuScriptTool =
{
scriptPath,
toolName,
runtimeInputs ? [ ],
wrapperArguments ? [ ],
}:
wrappedProgramTool {
inherit toolName runtimeInputs wrapperArguments;
wrappedProgram = lib.getExe pkgs.nushell;
addedFlags = [
"${scriptPath}"
toolName
];
};
describedApp = appDescription: appTool: {
type = "app";
meta.description = appDescription;
program = lib.getExe appTool;
}; };
buildAndExportPaths = pkgs.writeShellApplication { nixBuiltPathsTool =
name = "build-and-export-paths"; toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/nix-built-paths.nu; });
text = ''
export NIX_CONFIG="''${NIX_CONFIG:-}
post-build-hook = ${lib.getExe exportBuiltPathsHook}"
nix build --quiet --print-build-logs "$@"
'';
};
importExportedPaths = pkgs.writeShellApplication { releaseTaskTool =
name = "import-exported-paths"; toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/release-tasks.nu; });
text = ''
exported_paths_dir="$CI_WORKSPACE/.nix-built-paths"
if [[ ! -s "$exported_paths_dir/paths.txt" ]]; then
echo "No exported paths to import"
exit 0
fi
mapfile -t built_paths < "$exported_paths_dir/paths.txt"
nix copy --quiet --no-check-sigs --from "file://$exported_paths_dir" "''${built_paths[@]}"
'';
};
releasePlzOnCodeberg = exportBuiltPathsHook = nixBuiltPathsTool { toolName = "export-built-paths-hook"; };
subcommand: description:
shellScriptApp "release-plz-${subcommand}" description { exportingBuildTool =
toolArguments:
nixBuiltPathsTool (
toolArguments
// {
wrapperArguments = [
"--set"
"EXPORT_HOOK_PROGRAM"
(lib.getExe exportBuiltPathsHook)
];
}
);
releasePlzTool =
toolName:
releaseTaskTool {
inherit toolName;
runtimeInputs = [ runtimeInputs = [
pkgs.release-plz pkgs.release-plz
pkgs.gitMinimal pkgs.gitMinimal
@@ -149,98 +181,96 @@
pkgs.stdenv.cc pkgs.stdenv.cc
rustToolchain rustToolchain
]; ];
runtimeEnv.PKG_CONFIG_PATH = wrapperArguments = [
lib.makeSearchPathOutput "dev" "lib/pkgconfig" "--set"
builds.bootvine.buildInputs; "PKG_CONFIG_PATH"
text = ''exec release-plz ${subcommand} --forge gitea --git-token "$CODEBERG_TOKEN"''; (lib.makeSearchPathOutput "dev" "lib/pkgconfig" builds.bootvine.buildInputs)
];
}; };
repoTaskTool =
toolArguments:
wrappedProgramTool (
toolArguments
// {
wrappedProgram = lib.getExe builds.repo-tasks;
}
);
in in
sampleMachineApps sampleMachineApps
// { // {
release-plz-release = releasePlzOnCodeberg "release" "Publish unpublished versions to crates.io and tag them with a Codeberg release"; release-plz-release = describedApp "Publish unpublished versions to crates.io and tag them with a Codeberg release" (
release-plz-release-pr = releasePlzOnCodeberg "release-pr" "Open or update the Codeberg pull request that bumps the version and changelog"; releasePlzTool "release-plz-release"
);
release-plz-release-pr = describedApp "Open or update the Codeberg pull request that bumps the version and changelog" (
releasePlzTool "release-plz-release-pr"
);
record-screencasts = record-screencasts =
shellScriptApp "record-screencasts" describedApp
"Re-record the scripted demo animations and the sample machine recordings into recordings/; publish them with publish-recordings" "Re-record the scripted demo animations and the sample machine recordings into recordings/; publish them with publish-recordings"
{ (repoTaskTool {
toolName = "record-screencasts";
addedFlags = [
"record-screencasts"
"--vm-check-outputs"
"${pkgs.linkFarm "vm-check-outputs" (
lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines
)}"
];
runtimeInputs = [ runtimeInputs = [
pkgs.vhs pkgs.vhs
pkgs.libwebp pkgs.libwebp
builds.scripted-demos builds.scripted-demos
builds.repo-tasks builds.repo-tasks
]; ];
text = "exec xtask record-screencasts --vm-check-outputs ${ });
pkgs.linkFarm "vm-check-outputs" (
lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines
)
}";
};
publish-recordings = publish-recordings =
shellScriptApp "publish-recordings" describedApp "Replace the recordings branch on origin with a single commit holding recordings/"
"Replace the recordings branch on origin with a single commit holding recordings/" (repoTaskTool {
{ toolName = "publish-recordings";
addedFlags = [ "publish-recordings" ];
runtimeInputs = [ runtimeInputs = [
pkgs.gitMinimal pkgs.gitMinimal
builds.repo-tasks builds.repo-tasks
]; ];
text = "exec xtask publish-recordings"; });
};
build-and-export-paths = {
type = "app";
meta.description = "Build flake outputs and export newly built paths to the CI workspace";
program = lib.getExe buildAndExportPaths;
};
build-unless-cached = build-unless-cached =
shellScriptApp "build-unless-cached" describedApp "Build a flake output and export it to the CI workspace unless it is already in Cachix"
"Build a flake output and export it to the CI workspace unless it is already in Cachix" (exportingBuildTool {
{ toolName = "build-unless-cached";
text = '' });
built_output=$(nix eval --quiet --raw "$1.outPath")
if nix path-info --quiet --store https://wvhulle.cachix.org "$built_output" > /dev/null 2>&1; then
echo "$built_output is already in Cachix"
exit 0
fi
${lib.getExe buildAndExportPaths} "$1"
'';
};
import-exported-paths = { build-and-export-paths =
type = "app"; describedApp "Build flake outputs and export newly built paths to the CI workspace"
meta.description = "Import paths exported to the CI workspace by an earlier step"; (exportingBuildTool {
program = lib.getExe importExportedPaths; toolName = "build-and-export-paths";
}; });
import-exported-paths =
describedApp "Import paths exported to the CI workspace by an earlier step"
(nixBuiltPathsTool {
toolName = "import-exported-paths";
});
upload-exported-paths = upload-exported-paths =
shellScriptApp "upload-exported-paths" describedApp "Push paths exported to the CI workspace to the wvhulle Cachix cache"
"Push paths exported to the CI workspace to the wvhulle Cachix cache" (nixBuiltPathsTool {
{ toolName = "upload-exported-paths";
runtimeInputs = [ pkgs.cachix ]; runtimeInputs = [ pkgs.cachix ];
text = '' });
exported_paths_dir="$CI_WORKSPACE/.nix-built-paths"
if [[ ! -s "$exported_paths_dir/paths.txt" ]]; then
echo "No newly built paths to upload"
exit 0
fi
${lib.getExe importExportedPaths}
mapfile -t built_paths < "$exported_paths_dir/paths.txt"
cachix push wvhulle "''${built_paths[@]}"
'';
};
check-conventional-commits = check-conventional-commits =
shellScriptApp "check-conventional-commits" describedApp "Check that the commit messages in a git revision range are conventional commits"
"Check that the commit messages in a git revision range are conventional commits" (releaseTaskTool {
{ toolName = "check-conventional-commits";
runtimeInputs = [ runtimeInputs = [
pkgs.convco pkgs.convco
pkgs.gitMinimal pkgs.gitMinimal
]; ];
text = ''exec convco check "$@"''; });
};
} }
); );
@@ -250,6 +280,12 @@
inherit (builds) rustfmt-formatting clippy-lints; inherit (builds) rustfmt-formatting clippy-lints;
bootvine-build = builds.bootvine; bootvine-build = builds.bootvine;
repo-tasks-build = builds.repo-tasks; repo-tasks-build = builds.repo-tasks;
ci-task-tests =
pkgs.runCommand "bootvine-ci-task-tests" { nativeBuildInputs = [ pkgs.nushell ]; }
''
nu ${./ci}/test-ci-tasks.nu
touch $out
'';
} }
// lib.mapAttrs' (machineName: check: lib.nameValuePair "vm-${machineName}" check) ( // lib.mapAttrs' (machineName: check: lib.nameValuePair "vm-${machineName}" check) (
vmTests.checksFor pkgs vmTests.checksFor pkgs