Author SHA1 Message Date
Willem Vanhulle 8230de065f ci: separate dependency builds, checks and cache uploads
Build the workspace dependencies in their own step, skipped when they are
already in Cachix, so their compile output no longer shares a log with
the build, test and lint results. Steps do not share /nix/store, so later
steps import the paths exported by that step, on pull requests as well.

Push to Cachix in a separate step that alone holds the token and also
runs when a build fails. Pass --quiet to nix in all steps, which removes
the fetch lists and copying lines while keeping builder logs and errors.
2026-11-04 18:17:41 +01:00
7 changed files with 122 additions and 306 deletions

No files matched your search

+2
View File
@@ -34,6 +34,8 @@ steps:
image: nixos/nix:latest
environment: *release_env
commands:
- git remote set-url origin "https://wvhulle:$${CODEBERG_TOKEN}@codeberg.org/$${CI_REPO}.git"
- git checkout -B main "$${CI_COMMIT_SHA}"
- nix run --quiet .#release-plz-release
- name: update-release-pr
+18 -30
View File
@@ -8,8 +8,6 @@ variables:
experimental-features = nix-command flakes
build-users-group =
accept-flake-config = true
- &after_dependency_step
- status: [success, failure]
- &on_pull_requests
- event: pull_request
@@ -24,7 +22,6 @@ steps:
- name: build-and-test-bootvine
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
@@ -33,30 +30,11 @@ steps:
- name: build-repo-tasks
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.repo-tasks-build
- name: lint-rust-with-clippy
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints
- name: check-rust-formatting
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
when: *after_dependency_step
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting
- name: upload-build-cache
image: nixos/nix:latest
depends_on: [build-and-test-bootvine, build-repo-tasks]
@@ -71,6 +49,22 @@ steps:
commands:
- nix run --quiet .#upload-exported-paths
- name: lint-rust-with-clippy
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints
- name: check-rust-formatting
image: nixos/nix:latest
depends_on: [build-workspace-dependencies]
environment: *nix_config
commands:
- nix run --quiet .#import-exported-paths
- nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting
- name: evaluate-flake-outputs
image: nixos/nix:latest
depends_on: []
@@ -78,17 +72,11 @@ steps:
commands:
- nix flake check --quiet --no-build
- name: test-ci-tasks
image: nixos/nix:latest
depends_on: []
environment: *nix_config
commands:
- nix build --quiet --print-build-logs .#checks.x86_64-linux.ci-task-tests
- name: check-conventional-commit-messages
image: nixos/nix:latest
depends_on: []
when: *on_pull_requests
environment: *nix_config
commands:
- nix run --quiet .#check-conventional-commits -- --fetch-target-branch FETCH_HEAD..HEAD
- git fetch --quiet origin "$${CI_COMMIT_TARGET_BRANCH}"
- nix run --quiet .#check-conventional-commits -- FETCH_HEAD..HEAD
-79
View File
@@ -1,79 +0,0 @@
#!/usr/bin/env nu
const EXPORT_DIRECTORY_NAME = ".nix-built-paths"
const PATH_LIST_FILE_NAME = "paths.txt"
const CACHIX_CACHE_NAME = "wvhulle"
const CACHIX_CACHE_URL = "https://wvhulle.cachix.org"
def export-directory []: nothing -> path {
$env.CI_WORKSPACE | path join $EXPORT_DIRECTORY_NAME
}
export def store-paths-from-lines []: string -> list<string> {
lines | compact --empty
}
def exported-store-paths []: nothing -> list<string> {
let path_list_file = export-directory | path join $PATH_LIST_FILE_NAME
if ($path_list_file | path exists) {
open --raw $path_list_file | store-paths-from-lines
} else {
[]
}
}
export def nix-config-with-hook [existing_config: string, hook_program: string]: nothing -> string {
$"($existing_config)\npost-build-hook = ($hook_program)"
}
def "main export-built-paths-hook" []: nothing -> nothing {
let built_paths = $env.OUT_PATHS | split row " " | compact --empty
let directory = export-directory
mkdir $directory
^nix copy --quiet --to $"file://($directory)?compression=zstd" ...$built_paths
$built_paths | each {|built_path| $"($built_path)\n" } | str join | save --append ($directory | path join $PATH_LIST_FILE_NAME)
}
def --wrapped "main build-and-export-paths" [
...build_arguments: string
]: nothing -> nothing {
let hook_config = nix-config-with-hook ($env.NIX_CONFIG? | default "") $env.EXPORT_HOOK_PROGRAM
with-env {NIX_CONFIG: $hook_config} {
^nix build --quiet --print-build-logs ...$build_arguments
}
}
def "main build-unless-cached" [
installable: string
]: nothing -> nothing {
let built_output = ^nix eval --quiet --raw $"($installable).outPath"
let cache_lookup = do { ^nix path-info --quiet --store $CACHIX_CACHE_URL $built_output } | complete
if $cache_lookup.exit_code == 0 {
print $"($built_output) is already in Cachix"
} else {
main build-and-export-paths $installable
}
}
def "main import-exported-paths" []: nothing -> nothing {
let exported_paths = exported-store-paths
if ($exported_paths | is-empty) {
print "No exported paths to import"
} else {
^nix copy --quiet --no-check-sigs --from $"file://(export-directory)" ...$exported_paths
}
}
def "main upload-exported-paths" []: nothing -> nothing {
let exported_paths = exported-store-paths
if ($exported_paths | is-empty) {
print "No newly built paths to upload"
} else {
main import-exported-paths
^cachix push $CACHIX_CACHE_NAME ...$exported_paths
}
}
def main []: nothing -> nothing {
print "Run a subcommand, for example: build-and-export-paths"
}
-36
View File
@@ -1,36 +0,0 @@
#!/usr/bin/env nu
const CODEBERG_USER = "wvhulle"
const RELEASE_BRANCH_NAME = "main"
export def authenticated-origin-url [repository_slug: string, access_token: string]: nothing -> string {
$"https://($CODEBERG_USER):($access_token)@codeberg.org/($repository_slug).git"
}
def run-release-plz [release_subcommand: string]: nothing -> nothing {
^release-plz $release_subcommand --forge gitea --git-token $env.CODEBERG_TOKEN
}
def "main release-plz-release" []: nothing -> nothing {
^git remote set-url origin (authenticated-origin-url $env.CI_REPO $env.CODEBERG_TOKEN)
^git checkout -B $RELEASE_BRANCH_NAME $env.CI_COMMIT_SHA
run-release-plz release
}
def "main release-plz-release-pr" []: nothing -> nothing {
run-release-plz release-pr
}
def "main check-conventional-commits" [
--fetch-target-branch
...revision_ranges: string
]: nothing -> nothing {
if $fetch_target_branch {
^git fetch --quiet origin $env.CI_COMMIT_TARGET_BRANCH
}
^convco check ...$revision_ranges
}
def main []: nothing -> nothing {
print "Run a subcommand, for example: check-conventional-commits"
}
-14
View File
@@ -1,14 +0,0 @@
#!/usr/bin/env nu
use std/assert
use ./nix-built-paths.nu [nix-config-with-hook store-paths-from-lines]
use ./release-tasks.nu [authenticated-origin-url]
def main []: nothing -> nothing {
assert equal (nix-config-with-hook "max-jobs = 1" "/nix/store/hook") "max-jobs = 1\npost-build-hook = /nix/store/hook"
assert equal (nix-config-with-hook "" "/nix/store/hook") "\npost-build-hook = /nix/store/hook"
assert equal ("/nix/store/first\n\n/nix/store/second\n" | store-paths-from-lines) ["/nix/store/first" "/nix/store/second"]
assert equal ("" | store-paths-from-lines) []
assert equal (authenticated-origin-url "wvhulle/bootvine" "sample-token") "https://wvhulle:sample-token@codeberg.org/wvhulle/bootvine.git"
print "CI task tests passed"
}
+3 -12
View File
@@ -9,7 +9,6 @@
| `xtask/` | repository tasks run with `cargo xtask` |
| `vm-tests/` | the sample NixOS machines, one folder each with its VM check |
| `nix/` | the crane builds and the NixOS module |
| `ci/` | the Nushell scripts behind the CI apps in `flake.nix` |
| `recordings/` | ignored output of `record-screencasts`: the scripted demo GIFs and the sample machine WebP recordings, published on the `recordings` branch for the README and the [guide](guide.md) |
`cargo build` and `cargo test` act on every crate. `cargo run` needs the binary:
@@ -47,17 +46,9 @@ shell) or `pre-commit install` ([pre-commit](https://pre-commit.com)).
`.woodpecker/rust-and-flake-checks.yaml` runs on Codeberg's
[Woodpecker CI](https://woodpecker-ci.org) for every push and pull request: the bootvine build with its tests, the xtask build, clippy,
rustfmt, an evaluation of every flake output, and the commit-message check on
pull requests. Every step is a plain `nix` call; the scripting lives in `ci/`
and is tested by `nix build .#checks.x86_64-linux.ci-task-tests`.
Steps do not share `/nix/store`. The first step builds the workspace dependencies
unless they are already in the `wvhulle` [Cachix](https://www.cachix.org) cache,
which `nixConfig` in `flake.nix` offers as a substituter, and exports what it built
to the workspace. The build and lint steps import those paths with
`nix run .#import-exported-paths` and run even when an earlier step failed. On
`main` a final step, the only one holding the Cachix token, pushes the exported
paths with `nix run .#upload-exported-paths`. The two VM checks need KVM, which
Codeberg's runners lack, so they only run locally.
pull requests. Builds on `main` are pushed to the `wvhulle` [Cachix](https://www.cachix.org) cache, which
`nixConfig` in `flake.nix` offers as a substituter. The two VM checks need KVM,
which Codeberg's runners lack, so they only run locally.
## Releases
+99 -135
View File
@@ -91,6 +91,12 @@
...
}:
let
shellScriptApp = name: description: scriptArgs: {
type = "app";
meta.description = description;
program = lib.getExe (pkgs.writeShellApplication ({ inherit name; } // scriptArgs));
};
sampleMachineApps = lib.mapAttrs' (
machineName: machineDescription:
lib.nameValuePair "sample-vm-${machineName}" {
@@ -100,80 +106,42 @@
}
) vmTests.machineDescriptions;
wrappedProgramTool =
{
toolName,
wrappedProgram,
addedFlags,
runtimeInputs ? [ ],
wrapperArguments ? [ ],
}:
pkgs.runCommand toolName
{
nativeBuildInputs = [ pkgs.makeBinaryWrapper ];
meta.mainProgram = toolName;
}
''
makeWrapper ${wrappedProgram} $out/bin/${toolName} \
${
lib.escapeShellArgs (
lib.concatMap (addedFlag: [
"--add-flags"
addedFlag
]) addedFlags
)
} \
${lib.optionalString (runtimeInputs != [ ]) "--prefix PATH : ${lib.makeBinPath runtimeInputs}"} \
${lib.escapeShellArgs wrapperArguments}
'';
nuScriptTool =
{
scriptPath,
toolName,
runtimeInputs ? [ ],
wrapperArguments ? [ ],
}:
wrappedProgramTool {
inherit toolName runtimeInputs wrapperArguments;
wrappedProgram = lib.getExe pkgs.nushell;
addedFlags = [
"${scriptPath}"
toolName
];
};
describedApp = appDescription: appTool: {
type = "app";
meta.description = appDescription;
program = lib.getExe appTool;
exportBuiltPathsHook = pkgs.writeShellApplication {
name = "export-built-paths-hook";
text = ''
exported_paths_dir="$CI_WORKSPACE/.nix-built-paths"
mkdir -p "$exported_paths_dir"
read -ra built_paths <<< "$OUT_PATHS"
nix copy --quiet --to "file://$exported_paths_dir?compression=zstd" "''${built_paths[@]}"
printf '%s\n' "''${built_paths[@]}" >> "$exported_paths_dir/paths.txt"
'';
};
nixBuiltPathsTool =
toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/nix-built-paths.nu; });
buildAndExportPaths = pkgs.writeShellApplication {
name = "build-and-export-paths";
text = ''
export NIX_CONFIG="''${NIX_CONFIG:-}
post-build-hook = ${lib.getExe exportBuiltPathsHook}"
nix build --quiet --print-build-logs "$@"
'';
};
releaseTaskTool =
toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/release-tasks.nu; });
importExportedPaths = pkgs.writeShellApplication {
name = "import-exported-paths";
text = ''
exported_paths_dir="$CI_WORKSPACE/.nix-built-paths"
if [[ ! -s "$exported_paths_dir/paths.txt" ]]; then
echo "No exported paths to import"
exit 0
fi
mapfile -t built_paths < "$exported_paths_dir/paths.txt"
nix copy --quiet --no-check-sigs --from "file://$exported_paths_dir" "''${built_paths[@]}"
'';
};
exportBuiltPathsHook = nixBuiltPathsTool { toolName = "export-built-paths-hook"; };
exportingBuildTool =
toolArguments:
nixBuiltPathsTool (
toolArguments
// {
wrapperArguments = [
"--set"
"EXPORT_HOOK_PROGRAM"
(lib.getExe exportBuiltPathsHook)
];
}
);
releasePlzTool =
toolName:
releaseTaskTool {
inherit toolName;
releasePlzOnCodeberg =
subcommand: description:
shellScriptApp "release-plz-${subcommand}" description {
runtimeInputs = [
pkgs.release-plz
pkgs.gitMinimal
@@ -181,96 +149,98 @@
pkgs.stdenv.cc
rustToolchain
];
wrapperArguments = [
"--set"
"PKG_CONFIG_PATH"
(lib.makeSearchPathOutput "dev" "lib/pkgconfig" builds.bootvine.buildInputs)
];
runtimeEnv.PKG_CONFIG_PATH =
lib.makeSearchPathOutput "dev" "lib/pkgconfig"
builds.bootvine.buildInputs;
text = ''exec release-plz ${subcommand} --forge gitea --git-token "$CODEBERG_TOKEN"'';
};
repoTaskTool =
toolArguments:
wrappedProgramTool (
toolArguments
// {
wrappedProgram = lib.getExe builds.repo-tasks;
}
);
in
sampleMachineApps
// {
release-plz-release = describedApp "Publish unpublished versions to crates.io and tag them with a Codeberg release" (
releasePlzTool "release-plz-release"
);
release-plz-release-pr = describedApp "Open or update the Codeberg pull request that bumps the version and changelog" (
releasePlzTool "release-plz-release-pr"
);
release-plz-release = releasePlzOnCodeberg "release" "Publish unpublished versions to crates.io and tag them with a Codeberg release";
release-plz-release-pr = releasePlzOnCodeberg "release-pr" "Open or update the Codeberg pull request that bumps the version and changelog";
record-screencasts =
describedApp
shellScriptApp "record-screencasts"
"Re-record the scripted demo animations and the sample machine recordings into recordings/; publish them with publish-recordings"
(repoTaskTool {
toolName = "record-screencasts";
addedFlags = [
"record-screencasts"
"--vm-check-outputs"
"${pkgs.linkFarm "vm-check-outputs" (
lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines
)}"
];
{
runtimeInputs = [
pkgs.vhs
pkgs.libwebp
builds.scripted-demos
builds.repo-tasks
];
});
text = "exec xtask record-screencasts --vm-check-outputs ${
pkgs.linkFarm "vm-check-outputs" (
lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines
)
}";
};
publish-recordings =
describedApp "Replace the recordings branch on origin with a single commit holding recordings/"
(repoTaskTool {
toolName = "publish-recordings";
addedFlags = [ "publish-recordings" ];
shellScriptApp "publish-recordings"
"Replace the recordings branch on origin with a single commit holding recordings/"
{
runtimeInputs = [
pkgs.gitMinimal
builds.repo-tasks
];
});
text = "exec xtask publish-recordings";
};
build-and-export-paths = {
type = "app";
meta.description = "Build flake outputs and export newly built paths to the CI workspace";
program = lib.getExe buildAndExportPaths;
};
build-unless-cached =
describedApp "Build a flake output and export it to the CI workspace unless it is already in Cachix"
(exportingBuildTool {
toolName = "build-unless-cached";
});
shellScriptApp "build-unless-cached"
"Build a flake output and export it to the CI workspace unless it is already in Cachix"
{
text = ''
built_output=$(nix eval --quiet --raw "$1.outPath")
if nix path-info --quiet --store https://wvhulle.cachix.org "$built_output" > /dev/null 2>&1; then
echo "$built_output is already in Cachix"
exit 0
fi
${lib.getExe buildAndExportPaths} "$1"
'';
};
build-and-export-paths =
describedApp "Build flake outputs and export newly built paths to the CI workspace"
(exportingBuildTool {
toolName = "build-and-export-paths";
});
import-exported-paths =
describedApp "Import paths exported to the CI workspace by an earlier step"
(nixBuiltPathsTool {
toolName = "import-exported-paths";
});
import-exported-paths = {
type = "app";
meta.description = "Import paths exported to the CI workspace by an earlier step";
program = lib.getExe importExportedPaths;
};
upload-exported-paths =
describedApp "Push paths exported to the CI workspace to the wvhulle Cachix cache"
(nixBuiltPathsTool {
toolName = "upload-exported-paths";
shellScriptApp "upload-exported-paths"
"Push paths exported to the CI workspace to the wvhulle Cachix cache"
{
runtimeInputs = [ pkgs.cachix ];
});
text = ''
exported_paths_dir="$CI_WORKSPACE/.nix-built-paths"
if [[ ! -s "$exported_paths_dir/paths.txt" ]]; then
echo "No newly built paths to upload"
exit 0
fi
${lib.getExe importExportedPaths}
mapfile -t built_paths < "$exported_paths_dir/paths.txt"
cachix push wvhulle "''${built_paths[@]}"
'';
};
check-conventional-commits =
describedApp "Check that the commit messages in a git revision range are conventional commits"
(releaseTaskTool {
toolName = "check-conventional-commits";
shellScriptApp "check-conventional-commits"
"Check that the commit messages in a git revision range are conventional commits"
{
runtimeInputs = [
pkgs.convco
pkgs.gitMinimal
];
});
text = ''exec convco check "$@"'';
};
}
);
@@ -280,12 +250,6 @@
inherit (builds) rustfmt-formatting clippy-lints;
bootvine-build = builds.bootvine;
repo-tasks-build = builds.repo-tasks;
ci-task-tests =
pkgs.runCommand "bootvine-ci-task-tests" { nativeBuildInputs = [ pkgs.nushell ]; }
''
nu ${./ci}/test-ci-tasks.nu
touch $out
'';
}
// lib.mapAttrs' (machineName: check: lib.nameValuePair "vm-${machineName}" check) (
vmTests.checksFor pkgs