Author SHA1 Message Date
Willem Vanhulle ce72695e29 ci: move CI scripting into Nushell scripts with separate dependency steps
Replace the shell wrappers build-and-push-to-cachix, check-conventional-commits
and the release-plz apps with Nushell scripts in ci/, and move the git commands
out of the workflow YAML, so that every workflow command is a plain nix call.

Build the workspace dependencies in their own step, skipped when already in
Cachix. Later steps import the exported paths and also run after an earlier
failure. A separate step alone holds the Cachix token and uploads on main.
Pull requests run the same steps without the upload.

Wrap the recording tasks with makeBinaryWrapper instead of shell, and run the
CI task tests in a step of their own.
2026-11-04 20:02:26 +01:00
7 changed files with 306 additions and 122 deletions

No files matched your search

-2
View File
@@ -34,8 +34,6 @@ steps:
image: nixos/nix:latest
environment: *release_env
commands:
- git remote set-url origin "https://wvhulle:$${CODEBERG_TOKEN}@codeberg.org/$${CI_REPO}.git"
- git checkout -B main "$${CI_COMMIT_SHA}"
- nix run --quiet .#release-plz-release
- name: update-release-pr
+30 -18
View File
@@ -8,6 +8,8 @@ variables:
experimental-features = nix-command flakes
build-users-group =
accept-flake-config = true
- &after_dependency_step
- status: [success, failure]
- &on_pull_requests
- event: pull_request
@@ -22,6 +24,7 @@ steps:
- name: build-and-test-bootvine
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
@@ -30,11 +33,30 @@ steps:
- name: build-repo-tasks
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.repo-tasks-build
- name: lint-rust-with-clippy
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints
- name: check-rust-formatting
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting
- name: upload-build-cache
image: nixos/nix:latest
depends_on: [build-and-test-bootvine, build-repo-tasks]
@@ -49,22 +71,6 @@ steps:
commands:
- nix run --quiet .#upload-exported-paths
- name: lint-rust-with-clippy
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints
- name: check-rust-formatting
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting
- name: evaluate-flake-outputs
image: nixos/nix:latest
depends_on: []
@@ -72,11 +78,17 @@ steps:
commands:
- nix flake check --quiet --no-build
- name: test-ci-tasks
image: nixos/nix:latest
depends_on: []
environment: *nix_config
commands:
- nix build --quiet --print-build-logs .#checks.x86_64-linux.ci-task-tests
- name: check-conventional-commit-messages
image: nixos/nix:latest
depends_on: []
when: *on_pull_requests
environment: *nix_config
commands:
- git fetch --quiet origin "$${CI_COMMIT_TARGET_BRANCH}"
- nix run --quiet .#check-conventional-commits -- FETCH_HEAD..HEAD
- nix run --quiet .#check-conventional-commits -- --fetch-target-branch FETCH_HEAD..HEAD
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env nu
const EXPORT_DIRECTORY_NAME = ".nix-built-paths"
const PATH_LIST_FILE_NAME = "paths.txt"
const CACHIX_CACHE_NAME = "wvhulle"
const CACHIX_CACHE_URL = "https://wvhulle.cachix.org"
def export-directory []: nothing -> path {
$env.CI_WORKSPACE | path join $EXPORT_DIRECTORY_NAME
}
export def store-paths-from-lines []: string -> list<string> {
lines | compact --empty
}
def exported-store-paths []: nothing -> list<string> {
let path_list_file = export-directory | path join $PATH_LIST_FILE_NAME
if ($path_list_file | path exists) {
open --raw $path_list_file | store-paths-from-lines
} else {
[]
}
}
export def nix-config-with-hook [existing_config: string, hook_program: string]: nothing -> string {
$"($existing_config)\npost-build-hook = ($hook_program)"
}
def "main export-built-paths-hook" []: nothing -> nothing {
let built_paths = $env.OUT_PATHS | split row " " | compact --empty
let directory = export-directory
mkdir $directory
^nix copy --quiet --to $"file://($directory)?compression=zstd" ...$built_paths
$built_paths | each {|built_path| $"($built_path)\n" } | str join | save --append ($directory | path join $PATH_LIST_FILE_NAME)
}
def --wrapped "main build-and-export-paths" [
...build_arguments: string
]: nothing -> nothing {
let hook_config = nix-config-with-hook ($env.NIX_CONFIG? | default "") $env.EXPORT_HOOK_PROGRAM
with-env {NIX_CONFIG: $hook_config} {
^nix build --quiet --print-build-logs ...$build_arguments
}
}
def "main build-unless-cached" [
installable: string
]: nothing -> nothing {
let built_output = ^nix eval --quiet --raw $"($installable).outPath"
let cache_lookup = do { ^nix path-info --quiet --store $CACHIX_CACHE_URL $built_output } | complete
if $cache_lookup.exit_code == 0 {
print $"($built_output) is already in Cachix"
} else {
main build-and-export-paths $installable
}
}
def "main import-exported-paths" []: nothing -> nothing {
let exported_paths = exported-store-paths
if ($exported_paths | is-empty) {
print "No exported paths to import"
} else {
^nix copy --quiet --no-check-sigs --from $"file://(export-directory)" ...$exported_paths
}
}
def "main upload-exported-paths" []: nothing -> nothing {
let exported_paths = exported-store-paths
if ($exported_paths | is-empty) {
print "No newly built paths to upload"
} else {
main import-exported-paths
^cachix push $CACHIX_CACHE_NAME ...$exported_paths
}
}
def main []: nothing -> nothing {
print "Run a subcommand, for example: build-and-export-paths"
}
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env nu
const CODEBERG_USER = "wvhulle"
const RELEASE_BRANCH_NAME = "main"
export def authenticated-origin-url [repository_slug: string, access_token: string]: nothing -> string {
$"https://($CODEBERG_USER):($access_token)@codeberg.org/($repository_slug).git"
}
def run-release-plz [release_subcommand: string]: nothing -> nothing {
^release-plz $release_subcommand --forge gitea --git-token $env.CODEBERG_TOKEN
}
def "main release-plz-release" []: nothing -> nothing {
^git remote set-url origin (authenticated-origin-url $env.CI_REPO $env.CODEBERG_TOKEN)
^git checkout -B $RELEASE_BRANCH_NAME $env.CI_COMMIT_SHA
run-release-plz release
}
def "main release-plz-release-pr" []: nothing -> nothing {
run-release-plz release-pr
}
def "main check-conventional-commits" [
--fetch-target-branch
...revision_ranges: string
]: nothing -> nothing {
if $fetch_target_branch {
^git fetch --quiet origin $env.CI_COMMIT_TARGET_BRANCH
}
^convco check ...$revision_ranges
}
def main []: nothing -> nothing {
print "Run a subcommand, for example: check-conventional-commits"
}
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env nu
use std/assert
use ./nix-built-paths.nu [nix-config-with-hook store-paths-from-lines]
use ./release-tasks.nu [authenticated-origin-url]
def main []: nothing -> nothing {
assert equal (nix-config-with-hook "max-jobs = 1" "/nix/store/hook") "max-jobs = 1\npost-build-hook = /nix/store/hook"
assert equal (nix-config-with-hook "" "/nix/store/hook") "\npost-build-hook = /nix/store/hook"
assert equal ("/nix/store/first\n\n/nix/store/second\n" | store-paths-from-lines) ["/nix/store/first" "/nix/store/second"]
assert equal ("" | store-paths-from-lines) []
assert equal (authenticated-origin-url "wvhulle/bootvine" "sample-token") "https://wvhulle:sample-token@codeberg.org/wvhulle/bootvine.git"
print "CI task tests passed"
}
+12 -3
View File
@@ -9,6 +9,7 @@
| `xtask/` | repository tasks run with `cargo xtask` |
| `vm-tests/` | the sample NixOS machines, one folder each with its VM check |
| `nix/` | the crane builds and the NixOS module |
| `ci/` | the Nushell scripts behind the CI apps in `flake.nix` |
| `recordings/` | ignored output of `record-screencasts`: the scripted demo GIFs and the sample machine WebP recordings, published on the `recordings` branch for the README and the [guide](guide.md) |
`cargo build` and `cargo test` act on every crate. `cargo run` needs the binary:
@@ -46,9 +47,17 @@ shell) or `pre-commit install` ([pre-commit](https://pre-commit.com)).
`.woodpecker/rust-and-flake-checks.yaml` runs on Codeberg's
[Woodpecker CI](https://woodpecker-ci.org) for every push and pull request: the bootvine build with its tests, the xtask build, clippy,
rustfmt, an evaluation of every flake output, and the commit-message check on
pull requests. Builds on `main` are pushed to the `wvhulle` [Cachix](https://www.cachix.org) cache, which
`nixConfig` in `flake.nix` offers as a substituter. The two VM checks need KVM,
which Codeberg's runners lack, so they only run locally.
pull requests. Every step is a plain `nix` call; the scripting lives in `ci/`
and is tested by `nix build .#checks.x86_64-linux.ci-task-tests`.
Steps do not share `/nix/store`. The first step builds the workspace dependencies
unless they are already in the `wvhulle` [Cachix](https://www.cachix.org) cache,
which `nixConfig` in `flake.nix` offers as a substituter, and exports what it built
to the workspace. The build and lint steps import those paths with
`nix run .#import-exported-paths` and run even when an earlier step failed. On
`main` a final step, the only one holding the Cachix token, pushes the exported
paths with `nix run .#upload-exported-paths`. The two VM checks need KVM, which
Codeberg's runners lack, so they only run locally.
## Releases
+135 -99
View File
@@ -91,12 +91,6 @@
...
}:
let
shellScriptApp = name: description: scriptArgs: {
type = "app";
meta.description = description;
program = lib.getExe (pkgs.writeShellApplication ({ inherit name; } // scriptArgs));
};
sampleMachineApps = lib.mapAttrs' (
machineName: machineDescription:
lib.nameValuePair "sample-vm-${machineName}" {
@@ -106,42 +100,80 @@
}
) vmTests.machineDescriptions;
exportBuiltPathsHook = pkgs.writeShellApplication {
name = "export-built-paths-hook";
text = ''
exported_paths_dir="$CI_WORKSPACE/.nix-built-paths"
mkdir -p "$exported_paths_dir"
read -ra built_paths <<< "$OUT_PATHS"
nix copy --quiet --to "file://$exported_paths_dir?compression=zstd" "''${built_paths[@]}"
printf '%s\n' "''${built_paths[@]}" >> "$exported_paths_dir/paths.txt"
'';
wrappedProgramTool =
{
toolName,
wrappedProgram,
addedFlags,
runtimeInputs ? [ ],
wrapperArguments ? [ ],
}:
pkgs.runCommand toolName
{
nativeBuildInputs = [ pkgs.makeBinaryWrapper ];
meta.mainProgram = toolName;
}
''
makeWrapper ${wrappedProgram} $out/bin/${toolName} \
${
lib.escapeShellArgs (
lib.concatMap (addedFlag: [
"--add-flags"
addedFlag
]) addedFlags
)
} \
${lib.optionalString (runtimeInputs != [ ]) "--prefix PATH : ${lib.makeBinPath runtimeInputs}"} \
${lib.escapeShellArgs wrapperArguments}
'';
nuScriptTool =
{
scriptPath,
toolName,
runtimeInputs ? [ ],
wrapperArguments ? [ ],
}:
wrappedProgramTool {
inherit toolName runtimeInputs wrapperArguments;
wrappedProgram = lib.getExe pkgs.nushell;
addedFlags = [
"${scriptPath}"
toolName
];
};
describedApp = appDescription: appTool: {
type = "app";
meta.description = appDescription;
program = lib.getExe appTool;
};
buildAndExportPaths = pkgs.writeShellApplication {
name = "build-and-export-paths";
text = ''
export NIX_CONFIG="''${NIX_CONFIG:-}
post-build-hook = ${lib.getExe exportBuiltPathsHook}"
nix build --quiet --print-build-logs "$@"
'';
};
nixBuiltPathsTool =
toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/nix-built-paths.nu; });
importExportedPaths = pkgs.writeShellApplication {
name = "import-exported-paths";
text = ''
exported_paths_dir="$CI_WORKSPACE/.nix-built-paths"
if [[ ! -s "$exported_paths_dir/paths.txt" ]]; then
echo "No exported paths to import"
exit 0
fi
mapfile -t built_paths < "$exported_paths_dir/paths.txt"
nix copy --quiet --no-check-sigs --from "file://$exported_paths_dir" "''${built_paths[@]}"
'';
};
releaseTaskTool =
toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/release-tasks.nu; });
releasePlzOnCodeberg =
subcommand: description:
shellScriptApp "release-plz-${subcommand}" description {
exportBuiltPathsHook = nixBuiltPathsTool { toolName = "export-built-paths-hook"; };
exportingBuildTool =
toolArguments:
nixBuiltPathsTool (
toolArguments
// {
wrapperArguments = [
"--set"
"EXPORT_HOOK_PROGRAM"
(lib.getExe exportBuiltPathsHook)
];
}
);
releasePlzTool =
toolName:
releaseTaskTool {
inherit toolName;
runtimeInputs = [
pkgs.release-plz
pkgs.gitMinimal
@@ -149,98 +181,96 @@
pkgs.stdenv.cc
rustToolchain
];
runtimeEnv.PKG_CONFIG_PATH =
lib.makeSearchPathOutput "dev" "lib/pkgconfig"
builds.bootvine.buildInputs;
text = ''exec release-plz ${subcommand} --forge gitea --git-token "$CODEBERG_TOKEN"'';
wrapperArguments = [
"--set"
"PKG_CONFIG_PATH"
(lib.makeSearchPathOutput "dev" "lib/pkgconfig" builds.bootvine.buildInputs)
];
};
repoTaskTool =
toolArguments:
wrappedProgramTool (
toolArguments
// {
wrappedProgram = lib.getExe builds.repo-tasks;
}
);
in
sampleMachineApps
// {
release-plz-release = releasePlzOnCodeberg "release" "Publish unpublished versions to crates.io and tag them with a Codeberg release";
release-plz-release-pr = releasePlzOnCodeberg "release-pr" "Open or update the Codeberg pull request that bumps the version and changelog";
release-plz-release = describedApp "Publish unpublished versions to crates.io and tag them with a Codeberg release" (
releasePlzTool "release-plz-release"
);
release-plz-release-pr = describedApp "Open or update the Codeberg pull request that bumps the version and changelog" (
releasePlzTool "release-plz-release-pr"
);
record-screencasts =
shellScriptApp "record-screencasts"
describedApp
"Re-record the scripted demo animations and the sample machine recordings into recordings/; publish them with publish-recordings"
{
(repoTaskTool {
toolName = "record-screencasts";
addedFlags = [
"record-screencasts"
"--vm-check-outputs"
"${pkgs.linkFarm "vm-check-outputs" (
lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines
)}"
];
runtimeInputs = [
pkgs.vhs
pkgs.libwebp
builds.scripted-demos
builds.repo-tasks
];
text = "exec xtask record-screencasts --vm-check-outputs ${
pkgs.linkFarm "vm-check-outputs" (
lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines
)
}";
};
});
publish-recordings =
shellScriptApp "publish-recordings"
"Replace the recordings branch on origin with a single commit holding recordings/"
{
describedApp "Replace the recordings branch on origin with a single commit holding recordings/"
(repoTaskTool {
toolName = "publish-recordings";
addedFlags = [ "publish-recordings" ];
runtimeInputs = [
pkgs.gitMinimal
builds.repo-tasks
];
text = "exec xtask publish-recordings";
};
build-and-export-paths = {
type = "app";
meta.description = "Build flake outputs and export newly built paths to the CI workspace";
program = lib.getExe buildAndExportPaths;
};
});
build-unless-cached =
shellScriptApp "build-unless-cached"
"Build a flake output and export it to the CI workspace unless it is already in Cachix"
{
text = ''
built_output=$(nix eval --quiet --raw "$1.outPath")
if nix path-info --quiet --store https://wvhulle.cachix.org "$built_output" > /dev/null 2>&1; then
echo "$built_output is already in Cachix"
exit 0
fi
${lib.getExe buildAndExportPaths} "$1"
'';
};
describedApp "Build a flake output and export it to the CI workspace unless it is already in Cachix"
(exportingBuildTool {
toolName = "build-unless-cached";
});
import-exported-paths = {
type = "app";
meta.description = "Import paths exported to the CI workspace by an earlier step";
program = lib.getExe importExportedPaths;
};
build-and-export-paths =
describedApp "Build flake outputs and export newly built paths to the CI workspace"
(exportingBuildTool {
toolName = "build-and-export-paths";
});
import-exported-paths =
describedApp "Import paths exported to the CI workspace by an earlier step"
(nixBuiltPathsTool {
toolName = "import-exported-paths";
});
upload-exported-paths =
shellScriptApp "upload-exported-paths"
"Push paths exported to the CI workspace to the wvhulle Cachix cache"
{
describedApp "Push paths exported to the CI workspace to the wvhulle Cachix cache"
(nixBuiltPathsTool {
toolName = "upload-exported-paths";
runtimeInputs = [ pkgs.cachix ];
text = ''
exported_paths_dir="$CI_WORKSPACE/.nix-built-paths"
if [[ ! -s "$exported_paths_dir/paths.txt" ]]; then
echo "No newly built paths to upload"
exit 0
fi
${lib.getExe importExportedPaths}
mapfile -t built_paths < "$exported_paths_dir/paths.txt"
cachix push wvhulle "''${built_paths[@]}"
'';
};
});
check-conventional-commits =
shellScriptApp "check-conventional-commits"
"Check that the commit messages in a git revision range are conventional commits"
{
describedApp "Check that the commit messages in a git revision range are conventional commits"
(releaseTaskTool {
toolName = "check-conventional-commits";
runtimeInputs = [
pkgs.convco
pkgs.gitMinimal
];
text = ''exec convco check "$@"'';
};
});
}
);
@@ -250,6 +280,12 @@
inherit (builds) rustfmt-formatting clippy-lints;
bootvine-build = builds.bootvine;
repo-tasks-build = builds.repo-tasks;
ci-task-tests =
pkgs.runCommand "bootvine-ci-task-tests" { nativeBuildInputs = [ pkgs.nushell ]; }
''
nu ${./ci}/test-ci-tasks.nu
touch $out
'';
}
// lib.mapAttrs' (machineName: check: lib.nameValuePair "vm-${machineName}" check) (
vmTests.checksFor pkgs