From 8230de065f230d486f00c2b74ed8bc3de22ab7d2 Mon Sep 17 00:00:00 2001 From: Willem Vanhulle Date: Wed, 4 Nov 2026 18:11:57 +0100 Subject: [PATCH] ci: separate dependency builds, checks and cache uploads Build the workspace dependencies in their own step, skipped when they are already in Cachix, so their compile output no longer shares a log with the build, test and lint results. Steps do not share /nix/store, so later steps import the paths exported by that step, on pull requests as well. Push to Cachix in a separate step that alone holds the token and also runs when a build fails. Pass --quiet to nix in all steps, which removes the fetch lists and copying lines while keeping builder logs and errors. --- .woodpecker/release-pr-and-publish.yaml | 4 +- .woodpecker/rust-and-flake-checks.yaml | 67 +++++++++----------- flake.nix | 83 +++++++++++++++++++++++-- nix/crane-builds.nix | 2 + 4 files changed, 113 insertions(+), 43 deletions(-) diff --git a/.woodpecker/release-pr-and-publish.yaml b/.woodpecker/release-pr-and-publish.yaml index 92dc7f8..74fa2de 100644 --- a/.woodpecker/release-pr-and-publish.yaml +++ b/.woodpecker/release-pr-and-publish.yaml @@ -36,10 +36,10 @@ steps: commands: - git remote set-url origin "https://wvhulle:$${CODEBERG_TOKEN}@codeberg.org/$${CI_REPO}.git" - git checkout -B main "$${CI_COMMIT_SHA}" - - nix run .#release-plz-release + - nix run --quiet .#release-plz-release - name: update-release-pr image: nixos/nix:latest environment: *release_env commands: - - nix run .#release-plz-release-pr + - nix run --quiet .#release-plz-release-pr diff --git a/.woodpecker/rust-and-flake-checks.yaml b/.woodpecker/rust-and-flake-checks.yaml index 2a0d04d..e93a0a3 100644 --- a/.woodpecker/rust-and-flake-checks.yaml +++ b/.woodpecker/rust-and-flake-checks.yaml @@ -8,74 +8,69 @@ variables: experimental-features = nix-command flakes build-users-group = accept-flake-config = true - - &cache_writing_env - <<: *nix_config - CACHIX_AUTH_TOKEN: - from_secret: cachix_auth_token - - &on_main_branch - - event: [push, manual] - branch: main - - &off_main_branch - - event: pull_request - - event: [push, manual] - branch: - exclude: main - &on_pull_requests - event: pull_request steps: - - name: build-test-and-cache-bootvine + - name: build-workspace-dependencies image: nixos/nix:latest depends_on: [] - when: *on_main_branch - environment: *cache_writing_env + environment: *nix_config commands: - - nix run .#build-and-push-to-cachix -- .#checks.x86_64-linux.bootvine-build + - nix run --quiet .#build-unless-cached -- .#packages.x86_64-linux.workspace-dependencies - name: build-and-test-bootvine image: nixos/nix:latest - depends_on: [] - when: *off_main_branch + depends_on: [build-workspace-dependencies] environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.bootvine-build - - - name: build-and-cache-repo-tasks - image: nixos/nix:latest - depends_on: [] - when: *on_main_branch - environment: *cache_writing_env - commands: - - nix run .#build-and-push-to-cachix -- .#checks.x86_64-linux.repo-tasks-build + - nix run --quiet .#import-exported-paths + - nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.bootvine-build - name: build-repo-tasks image: nixos/nix:latest - depends_on: [] - when: *off_main_branch + depends_on: [build-workspace-dependencies] environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.repo-tasks-build + - nix run --quiet .#import-exported-paths + - nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.repo-tasks-build + + - name: upload-build-cache + image: nixos/nix:latest + depends_on: [build-and-test-bootvine, build-repo-tasks] + when: + - event: [push, manual] + branch: main + status: [success, failure] + environment: + <<: *nix_config + CACHIX_AUTH_TOKEN: + from_secret: cachix_auth_token + commands: + - nix run --quiet .#upload-exported-paths - name: lint-rust-with-clippy image: nixos/nix:latest - depends_on: [] + depends_on: [build-workspace-dependencies] environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.clippy-lints + - nix run --quiet .#import-exported-paths + - nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints - name: check-rust-formatting image: nixos/nix:latest - depends_on: [] + depends_on: [build-workspace-dependencies] environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.rustfmt-formatting + - nix run --quiet .#import-exported-paths + - nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting - name: evaluate-flake-outputs image: nixos/nix:latest depends_on: [] environment: *nix_config commands: - - nix flake check --no-build + - nix flake check --quiet --no-build - name: check-conventional-commit-messages image: nixos/nix:latest @@ -84,4 +79,4 @@ steps: environment: *nix_config commands: - git fetch --quiet origin "$${CI_COMMIT_TARGET_BRANCH}" - - nix run .#check-conventional-commits -- FETCH_HEAD..HEAD + - nix run --quiet .#check-conventional-commits -- FETCH_HEAD..HEAD diff --git a/flake.nix b/flake.nix index 34029c9..5d5a7c1 100644 --- a/flake.nix +++ b/flake.nix @@ -60,7 +60,12 @@ { pkgs, builds, ... }: { default = builds.bootvine; - inherit (builds) bootvine scripted-demos repo-tasks; + inherit (builds) + bootvine + scripted-demos + repo-tasks + workspace-dependencies + ; sample-unit-files = pkgs.callPackage ./vm-tests/slow-services/sample-unit-files.nix { inherit (vmTests) sampleUnits; }; @@ -101,6 +106,39 @@ } ) vmTests.machineDescriptions; + exportBuiltPathsHook = pkgs.writeShellApplication { + name = "export-built-paths-hook"; + text = '' + exported_paths_dir="$CI_WORKSPACE/.nix-built-paths" + mkdir -p "$exported_paths_dir" + read -ra built_paths <<< "$OUT_PATHS" + nix copy --quiet --to "file://$exported_paths_dir?compression=zstd" "''${built_paths[@]}" + printf '%s\n' "''${built_paths[@]}" >> "$exported_paths_dir/paths.txt" + ''; + }; + + buildAndExportPaths = pkgs.writeShellApplication { + name = "build-and-export-paths"; + text = '' + export NIX_CONFIG="''${NIX_CONFIG:-} + post-build-hook = ${lib.getExe exportBuiltPathsHook}" + nix build --quiet --print-build-logs "$@" + ''; + }; + + importExportedPaths = pkgs.writeShellApplication { + name = "import-exported-paths"; + text = '' + exported_paths_dir="$CI_WORKSPACE/.nix-built-paths" + if [[ ! -s "$exported_paths_dir/paths.txt" ]]; then + echo "No exported paths to import" + exit 0 + fi + mapfile -t built_paths < "$exported_paths_dir/paths.txt" + nix copy --quiet --no-check-sigs --from "file://$exported_paths_dir" "''${built_paths[@]}" + ''; + }; + releasePlzOnCodeberg = subcommand: description: shellScriptApp "release-plz-${subcommand}" description { @@ -150,12 +188,47 @@ text = "exec xtask publish-recordings"; }; - build-and-push-to-cachix = - shellScriptApp "build-and-push-to-cachix" - "Build flake outputs and push everything built to the wvhulle Cachix cache" + build-and-export-paths = { + type = "app"; + meta.description = "Build flake outputs and export newly built paths to the CI workspace"; + program = lib.getExe buildAndExportPaths; + }; + + build-unless-cached = + shellScriptApp "build-unless-cached" + "Build a flake output and export it to the CI workspace unless it is already in Cachix" + { + text = '' + built_output=$(nix eval --quiet --raw "$1.outPath") + if nix path-info --quiet --store https://wvhulle.cachix.org "$built_output" > /dev/null 2>&1; then + echo "$built_output is already in Cachix" + exit 0 + fi + ${lib.getExe buildAndExportPaths} "$1" + ''; + }; + + import-exported-paths = { + type = "app"; + meta.description = "Import paths exported to the CI workspace by an earlier step"; + program = lib.getExe importExportedPaths; + }; + + upload-exported-paths = + shellScriptApp "upload-exported-paths" + "Push paths exported to the CI workspace to the wvhulle Cachix cache" { runtimeInputs = [ pkgs.cachix ]; - text = ''exec cachix watch-exec wvhulle -- nix build --print-build-logs "$@"''; + text = '' + exported_paths_dir="$CI_WORKSPACE/.nix-built-paths" + if [[ ! -s "$exported_paths_dir/paths.txt" ]]; then + echo "No newly built paths to upload" + exit 0 + fi + ${lib.getExe importExportedPaths} + mapfile -t built_paths < "$exported_paths_dir/paths.txt" + cachix push wvhulle "''${built_paths[@]}" + ''; }; check-conventional-commits = diff --git a/nix/crane-builds.nix b/nix/crane-builds.nix index 1ab5128..5abebfd 100644 --- a/nix/crane-builds.nix +++ b/nix/crane-builds.nix @@ -30,6 +30,8 @@ let args: craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; } // args); in { + workspace-dependencies = cargoArtifacts; + rustfmt-formatting = craneLib.cargoFmt { inherit (commonArgs) src pname version; cargoExtraArgs = "--all";