ci: move CI scripting into Nushell scripts with separate dependency steps

Replace the shell wrappers build-and-push-to-cachix, check-conventional-commits
and the release-plz apps with Nushell scripts in ci/, and move the git commands
out of the workflow YAML, so that every workflow command is a plain nix call.

Build the workspace dependencies in their own step, skipped when already in
Cachix. Later steps import the exported paths and also run after an earlier
failure. A separate step alone holds the Cachix token and uploads on main.
Pull requests run the same steps without the upload.

Wrap the recording tasks with makeBinaryWrapper instead of shell, and run the
CI task tests in a step of their own.
This commit is contained in:
Willem Vanhulle committed 2026-11-04 20:02:26 +01:00
1 parent d0833ad563
commit ce72695e29
8 files changed
+338 -84

No files matched your search

+2 -4
View File
@@ -34,12 +34,10 @@ steps:
image: nixos/nix:latest
environment: *release_env
commands:
- git remote set-url origin "https://wvhulle:$${CODEBERG_TOKEN}@codeberg.org/$${CI_REPO}.git"
- git checkout -B main "$${CI_COMMIT_SHA}"
- nix run .#release-plz-release
- nix run --quiet .#release-plz-release
- name: update-release-pr
image: nixos/nix:latest
environment: *release_env
commands:
- nix run .#release-plz-release-pr
- nix run --quiet .#release-plz-release-pr
+44 -37
View File
@@ -8,74 +8,82 @@ variables:
experimental-features = nix-command flakes
build-users-group =
accept-flake-config = true
- &cache_writing_env
<<: *nix_config
CACHIX_AUTH_TOKEN:
from_secret: cachix_auth_token
- &on_main_branch
- event: [push, manual]
branch: main
- &off_main_branch
- event: pull_request
- event: [push, manual]
branch:
exclude: main
- &after_dependency_step
- status: [success, failure]
- &on_pull_requests
- event: pull_request
steps:
- name: build-test-and-cache-bootvine
- name: build-workspace-dependencies
image: nixos/nix:latest
depends_on: []
when: *on_main_branch
environment: *cache_writing_env
environment: *nix_config
commands:
- nix run .#build-and-push-to-cachix -- .#checks.x86_64-linux.bootvine-build
- nix run --quiet .#build-unless-cached -- .#packages.x86_64-linux.workspace-dependencies
- name: build-and-test-bootvine
image: nixos/nix:latest
depends_on: []
when: *off_main_branch
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix build --print-build-logs .#checks.x86_64-linux.bootvine-build
- name: build-and-cache-repo-tasks
image: nixos/nix:latest
depends_on: []
when: *on_main_branch
environment: *cache_writing_env
commands:
- nix run .#build-and-push-to-cachix -- .#checks.x86_64-linux.repo-tasks-build
- nix run --quiet .#import-exported-paths
- nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.bootvine-build
- name: build-repo-tasks
image: nixos/nix:latest
depends_on: []
when: *off_main_branch
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix build --print-build-logs .#checks.x86_64-linux.repo-tasks-build
- nix run --quiet .#import-exported-paths
- nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.repo-tasks-build
- name: lint-rust-with-clippy
image: nixos/nix:latest
depends_on: []
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix build --print-build-logs .#checks.x86_64-linux.clippy-lints
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints
- name: check-rust-formatting
image: nixos/nix:latest
depends_on: []
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix build --print-build-logs .#checks.x86_64-linux.rustfmt-formatting
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting
- name: upload-build-cache
image: nixos/nix:latest
depends_on: [build-and-test-bootvine, build-repo-tasks]
when:
- event: [push, manual]
branch: main
status: [success, failure]
environment:
<<: *nix_config
CACHIX_AUTH_TOKEN:
from_secret: cachix_auth_token
commands:
- nix run --quiet .#upload-exported-paths
- name: evaluate-flake-outputs
image: nixos/nix:latest
depends_on: []
environment: *nix_config
commands:
- nix flake check --no-build
- nix flake check --quiet --no-build
- name: test-ci-tasks
image: nixos/nix:latest
depends_on: []
environment: *nix_config
commands:
- nix build --quiet --print-build-logs .#checks.x86_64-linux.ci-task-tests
- name: check-conventional-commit-messages
image: nixos/nix:latest
@@ -83,5 +91,4 @@ steps:
when: *on_pull_requests
environment: *nix_config
commands:
- git fetch --quiet origin "$${CI_COMMIT_TARGET_BRANCH}"
- nix run .#check-conventional-commits -- FETCH_HEAD..HEAD
- nix run --quiet .#check-conventional-commits -- --fetch-target-branch FETCH_HEAD..HEAD
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env nu
const EXPORT_DIRECTORY_NAME = ".nix-built-paths"
const PATH_LIST_FILE_NAME = "paths.txt"
const CACHIX_CACHE_NAME = "wvhulle"
const CACHIX_CACHE_URL = "https://wvhulle.cachix.org"
def export-directory []: nothing -> path {
$env.CI_WORKSPACE | path join $EXPORT_DIRECTORY_NAME
}
export def store-paths-from-lines []: string -> list<string> {
lines | compact --empty
}
def exported-store-paths []: nothing -> list<string> {
let path_list_file = export-directory | path join $PATH_LIST_FILE_NAME
if ($path_list_file | path exists) {
open --raw $path_list_file | store-paths-from-lines
} else {
[]
}
}
export def nix-config-with-hook [existing_config: string, hook_program: string]: nothing -> string {
$"($existing_config)\npost-build-hook = ($hook_program)"
}
def "main export-built-paths-hook" []: nothing -> nothing {
let built_paths = $env.OUT_PATHS | split row " " | compact --empty
let directory = export-directory
mkdir $directory
^nix copy --quiet --to $"file://($directory)?compression=zstd" ...$built_paths
$built_paths | each {|built_path| $"($built_path)\n" } | str join | save --append ($directory | path join $PATH_LIST_FILE_NAME)
}
def --wrapped "main build-and-export-paths" [
...build_arguments: string
]: nothing -> nothing {
let hook_config = nix-config-with-hook ($env.NIX_CONFIG? | default "") $env.EXPORT_HOOK_PROGRAM
with-env {NIX_CONFIG: $hook_config} {
^nix build --quiet --print-build-logs ...$build_arguments
}
}
def "main build-unless-cached" [
installable: string
]: nothing -> nothing {
let built_output = ^nix eval --quiet --raw $"($installable).outPath"
let cache_lookup = do { ^nix path-info --quiet --store $CACHIX_CACHE_URL $built_output } | complete
if $cache_lookup.exit_code == 0 {
print $"($built_output) is already in Cachix"
} else {
main build-and-export-paths $installable
}
}
def "main import-exported-paths" []: nothing -> nothing {
let exported_paths = exported-store-paths
if ($exported_paths | is-empty) {
print "No exported paths to import"
} else {
^nix copy --quiet --no-check-sigs --from $"file://(export-directory)" ...$exported_paths
}
}
def "main upload-exported-paths" []: nothing -> nothing {
let exported_paths = exported-store-paths
if ($exported_paths | is-empty) {
print "No newly built paths to upload"
} else {
main import-exported-paths
^cachix push $CACHIX_CACHE_NAME ...$exported_paths
}
}
def main []: nothing -> nothing {
print "Run a subcommand, for example: build-and-export-paths"
}
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env nu
const CODEBERG_USER = "wvhulle"
const RELEASE_BRANCH_NAME = "main"
export def authenticated-origin-url [repository_slug: string, access_token: string]: nothing -> string {
$"https://($CODEBERG_USER):($access_token)@codeberg.org/($repository_slug).git"
}
def run-release-plz [release_subcommand: string]: nothing -> nothing {
^release-plz $release_subcommand --forge gitea --git-token $env.CODEBERG_TOKEN
}
def "main release-plz-release" []: nothing -> nothing {
^git remote set-url origin (authenticated-origin-url $env.CI_REPO $env.CODEBERG_TOKEN)
^git checkout -B $RELEASE_BRANCH_NAME $env.CI_COMMIT_SHA
run-release-plz release
}
def "main release-plz-release-pr" []: nothing -> nothing {
run-release-plz release-pr
}
def "main check-conventional-commits" [
--fetch-target-branch
...revision_ranges: string
]: nothing -> nothing {
if $fetch_target_branch {
^git fetch --quiet origin $env.CI_COMMIT_TARGET_BRANCH
}
^convco check ...$revision_ranges
}
def main []: nothing -> nothing {
print "Run a subcommand, for example: check-conventional-commits"
}
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env nu
use std/assert
use ./nix-built-paths.nu [nix-config-with-hook store-paths-from-lines]
use ./release-tasks.nu [authenticated-origin-url]
def main []: nothing -> nothing {
assert equal (nix-config-with-hook "max-jobs = 1" "/nix/store/hook") "max-jobs = 1\npost-build-hook = /nix/store/hook"
assert equal (nix-config-with-hook "" "/nix/store/hook") "\npost-build-hook = /nix/store/hook"
assert equal ("/nix/store/first\n\n/nix/store/second\n" | store-paths-from-lines) ["/nix/store/first" "/nix/store/second"]
assert equal ("" | store-paths-from-lines) []
assert equal (authenticated-origin-url "wvhulle/bootvine" "sample-token") "https://wvhulle:sample-token@codeberg.org/wvhulle/bootvine.git"
print "CI task tests passed"
}
+12 -3
View File
@@ -9,6 +9,7 @@
| `xtask/` | repository tasks run with `cargo xtask` |
| `vm-tests/` | the sample NixOS machines, one folder each with its VM check |
| `nix/` | the crane builds and the NixOS module |
| `ci/` | the Nushell scripts behind the CI apps in `flake.nix` |
| `recordings/` | ignored output of `record-screencasts`: the scripted demo GIFs and the sample machine WebP recordings, published on the `recordings` branch for the README and the [guide](guide.md) |
`cargo build` and `cargo test` act on every crate. `cargo run` needs the binary:
@@ -46,9 +47,17 @@ shell) or `pre-commit install` ([pre-commit](https://pre-commit.com)).
`.woodpecker/rust-and-flake-checks.yaml` runs on Codeberg's
[Woodpecker CI](https://woodpecker-ci.org) for every push and pull request: the bootvine build with its tests, the xtask build, clippy,
rustfmt, an evaluation of every flake output, and the commit-message check on
pull requests. Builds on `main` are pushed to the `wvhulle` [Cachix](https://www.cachix.org) cache, which
`nixConfig` in `flake.nix` offers as a substituter. The two VM checks need KVM,
which Codeberg's runners lack, so they only run locally.
pull requests. Every step is a plain `nix` call; the scripting lives in `ci/`
and is tested by `nix build .#checks.x86_64-linux.ci-task-tests`.
Steps do not share `/nix/store`. The first step builds the workspace dependencies
unless they are already in the `wvhulle` [Cachix](https://www.cachix.org) cache,
which `nixConfig` in `flake.nix` offers as a substituter, and exports what it built
to the workspace. The build and lint steps import those paths with
`nix run .#import-exported-paths` and run even when an earlier step failed. On
`main` a final step, the only one holding the Cachix token, pushes the exported
paths with `nix run .#upload-exported-paths`. The two VM checks need KVM, which
Codeberg's runners lack, so they only run locally.
## Releases
+149 -40
View File
@@ -60,7 +60,12 @@
{ pkgs, builds, ... }:
{
default = builds.bootvine;
inherit (builds) bootvine scripted-demos repo-tasks;
inherit (builds)
bootvine
scripted-demos
repo-tasks
workspace-dependencies
;
sample-unit-files = pkgs.callPackage ./vm-tests/slow-services/sample-unit-files.nix {
inherit (vmTests) sampleUnits;
};
@@ -86,12 +91,6 @@
...
}:
let
shellScriptApp = name: description: scriptArgs: {
type = "app";
meta.description = description;
program = lib.getExe (pkgs.writeShellApplication ({ inherit name; } // scriptArgs));
};
sampleMachineApps = lib.mapAttrs' (
machineName: machineDescription:
lib.nameValuePair "sample-vm-${machineName}" {
@@ -101,9 +100,80 @@
}
) vmTests.machineDescriptions;
releasePlzOnCodeberg =
subcommand: description:
shellScriptApp "release-plz-${subcommand}" description {
wrappedProgramTool =
{
toolName,
wrappedProgram,
addedFlags,
runtimeInputs ? [ ],
wrapperArguments ? [ ],
}:
pkgs.runCommand toolName
{
nativeBuildInputs = [ pkgs.makeBinaryWrapper ];
meta.mainProgram = toolName;
}
''
makeWrapper ${wrappedProgram} $out/bin/${toolName} \
${
lib.escapeShellArgs (
lib.concatMap (addedFlag: [
"--add-flags"
addedFlag
]) addedFlags
)
} \
${lib.optionalString (runtimeInputs != [ ]) "--prefix PATH : ${lib.makeBinPath runtimeInputs}"} \
${lib.escapeShellArgs wrapperArguments}
'';
nuScriptTool =
{
scriptPath,
toolName,
runtimeInputs ? [ ],
wrapperArguments ? [ ],
}:
wrappedProgramTool {
inherit toolName runtimeInputs wrapperArguments;
wrappedProgram = lib.getExe pkgs.nushell;
addedFlags = [
"${scriptPath}"
toolName
];
};
describedApp = appDescription: appTool: {
type = "app";
meta.description = appDescription;
program = lib.getExe appTool;
};
nixBuiltPathsTool =
toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/nix-built-paths.nu; });
releaseTaskTool =
toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/release-tasks.nu; });
exportBuiltPathsHook = nixBuiltPathsTool { toolName = "export-built-paths-hook"; };
exportingBuildTool =
toolArguments:
nixBuiltPathsTool (
toolArguments
// {
wrapperArguments = [
"--set"
"EXPORT_HOOK_PROGRAM"
(lib.getExe exportBuiltPathsHook)
];
}
);
releasePlzTool =
toolName:
releaseTaskTool {
inherit toolName;
runtimeInputs = [
pkgs.release-plz
pkgs.gitMinimal
@@ -111,63 +181,96 @@
pkgs.stdenv.cc
rustToolchain
];
runtimeEnv.PKG_CONFIG_PATH =
lib.makeSearchPathOutput "dev" "lib/pkgconfig"
builds.bootvine.buildInputs;
text = ''exec release-plz ${subcommand} --forge gitea --git-token "$CODEBERG_TOKEN"'';
wrapperArguments = [
"--set"
"PKG_CONFIG_PATH"
(lib.makeSearchPathOutput "dev" "lib/pkgconfig" builds.bootvine.buildInputs)
];
};
repoTaskTool =
toolArguments:
wrappedProgramTool (
toolArguments
// {
wrappedProgram = lib.getExe builds.repo-tasks;
}
);
in
sampleMachineApps
// {
release-plz-release = releasePlzOnCodeberg "release" "Publish unpublished versions to crates.io and tag them with a Codeberg release";
release-plz-release-pr = releasePlzOnCodeberg "release-pr" "Open or update the Codeberg pull request that bumps the version and changelog";
release-plz-release = describedApp "Publish unpublished versions to crates.io and tag them with a Codeberg release" (
releasePlzTool "release-plz-release"
);
release-plz-release-pr = describedApp "Open or update the Codeberg pull request that bumps the version and changelog" (
releasePlzTool "release-plz-release-pr"
);
record-screencasts =
shellScriptApp "record-screencasts"
describedApp
"Re-record the scripted demo animations and the sample machine recordings into recordings/; publish them with publish-recordings"
{
(repoTaskTool {
toolName = "record-screencasts";
addedFlags = [
"record-screencasts"
"--vm-check-outputs"
"${pkgs.linkFarm "vm-check-outputs" (
lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines
)}"
];
runtimeInputs = [
pkgs.vhs
pkgs.libwebp
builds.scripted-demos
builds.repo-tasks
];
text = "exec xtask record-screencasts --vm-check-outputs ${
pkgs.linkFarm "vm-check-outputs" (
lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines
)
}";
};
});
publish-recordings =
shellScriptApp "publish-recordings"
"Replace the recordings branch on origin with a single commit holding recordings/"
{
describedApp "Replace the recordings branch on origin with a single commit holding recordings/"
(repoTaskTool {
toolName = "publish-recordings";
addedFlags = [ "publish-recordings" ];
runtimeInputs = [
pkgs.gitMinimal
builds.repo-tasks
];
text = "exec xtask publish-recordings";
};
});
build-and-push-to-cachix =
shellScriptApp "build-and-push-to-cachix"
"Build flake outputs and push everything built to the wvhulle Cachix cache"
{
build-unless-cached =
describedApp "Build a flake output and export it to the CI workspace unless it is already in Cachix"
(exportingBuildTool {
toolName = "build-unless-cached";
});
build-and-export-paths =
describedApp "Build flake outputs and export newly built paths to the CI workspace"
(exportingBuildTool {
toolName = "build-and-export-paths";
});
import-exported-paths =
describedApp "Import paths exported to the CI workspace by an earlier step"
(nixBuiltPathsTool {
toolName = "import-exported-paths";
});
upload-exported-paths =
describedApp "Push paths exported to the CI workspace to the wvhulle Cachix cache"
(nixBuiltPathsTool {
toolName = "upload-exported-paths";
runtimeInputs = [ pkgs.cachix ];
text = ''exec cachix watch-exec wvhulle -- nix build --print-build-logs "$@"'';
};
});
check-conventional-commits =
shellScriptApp "check-conventional-commits"
"Check that the commit messages in a git revision range are conventional commits"
{
describedApp "Check that the commit messages in a git revision range are conventional commits"
(releaseTaskTool {
toolName = "check-conventional-commits";
runtimeInputs = [
pkgs.convco
pkgs.gitMinimal
];
text = ''exec convco check "$@"'';
};
});
}
);
@@ -177,6 +280,12 @@
inherit (builds) rustfmt-formatting clippy-lints;
bootvine-build = builds.bootvine;
repo-tasks-build = builds.repo-tasks;
ci-task-tests =
pkgs.runCommand "bootvine-ci-task-tests" { nativeBuildInputs = [ pkgs.nushell ]; }
''
nu ${./ci}/test-ci-tasks.nu
touch $out
'';
}
// lib.mapAttrs' (machineName: check: lib.nameValuePair "vm-${machineName}" check) (
vmTests.checksFor pkgs
+2
View File
@@ -30,6 +30,8 @@ let
args: craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; } // args);
in
{
workspace-dependencies = cargoArtifacts;
rustfmt-formatting = craneLib.cargoFmt {
inherit (commonArgs) src pname version;
cargoExtraArgs = "--all";