From ce72695e29055ed37ce2e6840f6ec867b08fed24 Mon Sep 17 00:00:00 2001 From: Willem Vanhulle Date: Wed, 4 Nov 2026 20:01:09 +0100 Subject: [PATCH] ci: move CI scripting into Nushell scripts with separate dependency steps Replace the shell wrappers build-and-push-to-cachix, check-conventional-commits and the release-plz apps with Nushell scripts in ci/, and move the git commands out of the workflow YAML, so that every workflow command is a plain nix call. Build the workspace dependencies in their own step, skipped when already in Cachix. Later steps import the exported paths and also run after an earlier failure. A separate step alone holds the Cachix token and uploads on main. Pull requests run the same steps without the upload. Wrap the recording tasks with makeBinaryWrapper instead of shell, and run the CI task tests in a step of their own. --- .woodpecker/release-pr-and-publish.yaml | 6 +- .woodpecker/rust-and-flake-checks.yaml | 81 +++++----- ci/nix-built-paths.nu | 79 ++++++++++ ci/release-tasks.nu | 36 +++++ ci/test-ci-tasks.nu | 14 ++ docs/development.md | 15 +- flake.nix | 189 +++++++++++++++++++----- nix/crane-builds.nix | 2 + 8 files changed, 338 insertions(+), 84 deletions(-) create mode 100755 ci/nix-built-paths.nu create mode 100755 ci/release-tasks.nu create mode 100755 ci/test-ci-tasks.nu diff --git a/.woodpecker/release-pr-and-publish.yaml b/.woodpecker/release-pr-and-publish.yaml index 92dc7f8..c22fbab 100644 --- a/.woodpecker/release-pr-and-publish.yaml +++ b/.woodpecker/release-pr-and-publish.yaml @@ -34,12 +34,10 @@ steps: image: nixos/nix:latest environment: *release_env commands: - - git remote set-url origin "https://wvhulle:$${CODEBERG_TOKEN}@codeberg.org/$${CI_REPO}.git" - - git checkout -B main "$${CI_COMMIT_SHA}" - - nix run .#release-plz-release + - nix run --quiet .#release-plz-release - name: update-release-pr image: nixos/nix:latest environment: *release_env commands: - - nix run .#release-plz-release-pr + - nix run --quiet .#release-plz-release-pr diff --git a/.woodpecker/rust-and-flake-checks.yaml b/.woodpecker/rust-and-flake-checks.yaml index 2a0d04d..97df0b9 100644 --- a/.woodpecker/rust-and-flake-checks.yaml +++ b/.woodpecker/rust-and-flake-checks.yaml @@ -8,74 +8,82 @@ variables: experimental-features = nix-command flakes build-users-group = accept-flake-config = true - - &cache_writing_env - <<: *nix_config - CACHIX_AUTH_TOKEN: - from_secret: cachix_auth_token - - &on_main_branch - - event: [push, manual] - branch: main - - &off_main_branch - - event: pull_request - - event: [push, manual] - branch: - exclude: main + - &after_dependency_step + - status: [success, failure] - &on_pull_requests - event: pull_request steps: - - name: build-test-and-cache-bootvine + - name: build-workspace-dependencies image: nixos/nix:latest depends_on: [] - when: *on_main_branch - environment: *cache_writing_env + environment: *nix_config commands: - - nix run .#build-and-push-to-cachix -- .#checks.x86_64-linux.bootvine-build + - nix run --quiet .#build-unless-cached -- .#packages.x86_64-linux.workspace-dependencies - name: build-and-test-bootvine image: nixos/nix:latest - depends_on: [] - when: *off_main_branch + depends_on: [build-workspace-dependencies] + when: *after_dependency_step environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.bootvine-build - - - name: build-and-cache-repo-tasks - image: nixos/nix:latest - depends_on: [] - when: *on_main_branch - environment: *cache_writing_env - commands: - - nix run .#build-and-push-to-cachix -- .#checks.x86_64-linux.repo-tasks-build + - nix run --quiet .#import-exported-paths + - nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.bootvine-build - name: build-repo-tasks image: nixos/nix:latest - depends_on: [] - when: *off_main_branch + depends_on: [build-workspace-dependencies] + when: *after_dependency_step environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.repo-tasks-build + - nix run --quiet .#import-exported-paths + - nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.repo-tasks-build - name: lint-rust-with-clippy image: nixos/nix:latest - depends_on: [] + depends_on: [build-workspace-dependencies] + when: *after_dependency_step environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.clippy-lints + - nix run --quiet .#import-exported-paths + - nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints - name: check-rust-formatting image: nixos/nix:latest - depends_on: [] + depends_on: [build-workspace-dependencies] + when: *after_dependency_step environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.rustfmt-formatting + - nix run --quiet .#import-exported-paths + - nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting + + - name: upload-build-cache + image: nixos/nix:latest + depends_on: [build-and-test-bootvine, build-repo-tasks] + when: + - event: [push, manual] + branch: main + status: [success, failure] + environment: + <<: *nix_config + CACHIX_AUTH_TOKEN: + from_secret: cachix_auth_token + commands: + - nix run --quiet .#upload-exported-paths - name: evaluate-flake-outputs image: nixos/nix:latest depends_on: [] environment: *nix_config commands: - - nix flake check --no-build + - nix flake check --quiet --no-build + + - name: test-ci-tasks + image: nixos/nix:latest + depends_on: [] + environment: *nix_config + commands: + - nix build --quiet --print-build-logs .#checks.x86_64-linux.ci-task-tests - name: check-conventional-commit-messages image: nixos/nix:latest @@ -83,5 +91,4 @@ steps: when: *on_pull_requests environment: *nix_config commands: - - git fetch --quiet origin "$${CI_COMMIT_TARGET_BRANCH}" - - nix run .#check-conventional-commits -- FETCH_HEAD..HEAD + - nix run --quiet .#check-conventional-commits -- --fetch-target-branch FETCH_HEAD..HEAD diff --git a/ci/nix-built-paths.nu b/ci/nix-built-paths.nu new file mode 100755 index 0000000..575156d --- /dev/null +++ b/ci/nix-built-paths.nu @@ -0,0 +1,79 @@ +#!/usr/bin/env nu + +const EXPORT_DIRECTORY_NAME = ".nix-built-paths" +const PATH_LIST_FILE_NAME = "paths.txt" +const CACHIX_CACHE_NAME = "wvhulle" +const CACHIX_CACHE_URL = "https://wvhulle.cachix.org" + +def export-directory []: nothing -> path { + $env.CI_WORKSPACE | path join $EXPORT_DIRECTORY_NAME +} + +export def store-paths-from-lines []: string -> list { + lines | compact --empty +} + +def exported-store-paths []: nothing -> list { + let path_list_file = export-directory | path join $PATH_LIST_FILE_NAME + if ($path_list_file | path exists) { + open --raw $path_list_file | store-paths-from-lines + } else { + [] + } +} + +export def nix-config-with-hook [existing_config: string, hook_program: string]: nothing -> string { + $"($existing_config)\npost-build-hook = ($hook_program)" +} + +def "main export-built-paths-hook" []: nothing -> nothing { + let built_paths = $env.OUT_PATHS | split row " " | compact --empty + let directory = export-directory + mkdir $directory + ^nix copy --quiet --to $"file://($directory)?compression=zstd" ...$built_paths + $built_paths | each {|built_path| $"($built_path)\n" } | str join | save --append ($directory | path join $PATH_LIST_FILE_NAME) +} + +def --wrapped "main build-and-export-paths" [ + ...build_arguments: string +]: nothing -> nothing { + let hook_config = nix-config-with-hook ($env.NIX_CONFIG? | default "") $env.EXPORT_HOOK_PROGRAM + with-env {NIX_CONFIG: $hook_config} { + ^nix build --quiet --print-build-logs ...$build_arguments + } +} + +def "main build-unless-cached" [ + installable: string +]: nothing -> nothing { + let built_output = ^nix eval --quiet --raw $"($installable).outPath" + let cache_lookup = do { ^nix path-info --quiet --store $CACHIX_CACHE_URL $built_output } | complete + if $cache_lookup.exit_code == 0 { + print $"($built_output) is already in Cachix" + } else { + main build-and-export-paths $installable + } +} + +def "main import-exported-paths" []: nothing -> nothing { + let exported_paths = exported-store-paths + if ($exported_paths | is-empty) { + print "No exported paths to import" + } else { + ^nix copy --quiet --no-check-sigs --from $"file://(export-directory)" ...$exported_paths + } +} + +def "main upload-exported-paths" []: nothing -> nothing { + let exported_paths = exported-store-paths + if ($exported_paths | is-empty) { + print "No newly built paths to upload" + } else { + main import-exported-paths + ^cachix push $CACHIX_CACHE_NAME ...$exported_paths + } +} + +def main []: nothing -> nothing { + print "Run a subcommand, for example: build-and-export-paths" +} diff --git a/ci/release-tasks.nu b/ci/release-tasks.nu new file mode 100755 index 0000000..c629c73 --- /dev/null +++ b/ci/release-tasks.nu @@ -0,0 +1,36 @@ +#!/usr/bin/env nu + +const CODEBERG_USER = "wvhulle" +const RELEASE_BRANCH_NAME = "main" + +export def authenticated-origin-url [repository_slug: string, access_token: string]: nothing -> string { + $"https://($CODEBERG_USER):($access_token)@codeberg.org/($repository_slug).git" +} + +def run-release-plz [release_subcommand: string]: nothing -> nothing { + ^release-plz $release_subcommand --forge gitea --git-token $env.CODEBERG_TOKEN +} + +def "main release-plz-release" []: nothing -> nothing { + ^git remote set-url origin (authenticated-origin-url $env.CI_REPO $env.CODEBERG_TOKEN) + ^git checkout -B $RELEASE_BRANCH_NAME $env.CI_COMMIT_SHA + run-release-plz release +} + +def "main release-plz-release-pr" []: nothing -> nothing { + run-release-plz release-pr +} + +def "main check-conventional-commits" [ + --fetch-target-branch + ...revision_ranges: string +]: nothing -> nothing { + if $fetch_target_branch { + ^git fetch --quiet origin $env.CI_COMMIT_TARGET_BRANCH + } + ^convco check ...$revision_ranges +} + +def main []: nothing -> nothing { + print "Run a subcommand, for example: check-conventional-commits" +} diff --git a/ci/test-ci-tasks.nu b/ci/test-ci-tasks.nu new file mode 100755 index 0000000..1062f4d --- /dev/null +++ b/ci/test-ci-tasks.nu @@ -0,0 +1,14 @@ +#!/usr/bin/env nu + +use std/assert +use ./nix-built-paths.nu [nix-config-with-hook store-paths-from-lines] +use ./release-tasks.nu [authenticated-origin-url] + +def main []: nothing -> nothing { + assert equal (nix-config-with-hook "max-jobs = 1" "/nix/store/hook") "max-jobs = 1\npost-build-hook = /nix/store/hook" + assert equal (nix-config-with-hook "" "/nix/store/hook") "\npost-build-hook = /nix/store/hook" + assert equal ("/nix/store/first\n\n/nix/store/second\n" | store-paths-from-lines) ["/nix/store/first" "/nix/store/second"] + assert equal ("" | store-paths-from-lines) [] + assert equal (authenticated-origin-url "wvhulle/bootvine" "sample-token") "https://wvhulle:sample-token@codeberg.org/wvhulle/bootvine.git" + print "CI task tests passed" +} diff --git a/docs/development.md b/docs/development.md index d089092..0ade169 100644 --- a/docs/development.md +++ b/docs/development.md @@ -9,6 +9,7 @@ | `xtask/` | repository tasks run with `cargo xtask` | | `vm-tests/` | the sample NixOS machines, one folder each with its VM check | | `nix/` | the crane builds and the NixOS module | +| `ci/` | the Nushell scripts behind the CI apps in `flake.nix` | | `recordings/` | ignored output of `record-screencasts`: the scripted demo GIFs and the sample machine WebP recordings, published on the `recordings` branch for the README and the [guide](guide.md) | `cargo build` and `cargo test` act on every crate. `cargo run` needs the binary: @@ -46,9 +47,17 @@ shell) or `pre-commit install` ([pre-commit](https://pre-commit.com)). `.woodpecker/rust-and-flake-checks.yaml` runs on Codeberg's [Woodpecker CI](https://woodpecker-ci.org) for every push and pull request: the bootvine build with its tests, the xtask build, clippy, rustfmt, an evaluation of every flake output, and the commit-message check on -pull requests. Builds on `main` are pushed to the `wvhulle` [Cachix](https://www.cachix.org) cache, which -`nixConfig` in `flake.nix` offers as a substituter. The two VM checks need KVM, -which Codeberg's runners lack, so they only run locally. +pull requests. Every step is a plain `nix` call; the scripting lives in `ci/` +and is tested by `nix build .#checks.x86_64-linux.ci-task-tests`. + +Steps do not share `/nix/store`. The first step builds the workspace dependencies +unless they are already in the `wvhulle` [Cachix](https://www.cachix.org) cache, +which `nixConfig` in `flake.nix` offers as a substituter, and exports what it built +to the workspace. The build and lint steps import those paths with +`nix run .#import-exported-paths` and run even when an earlier step failed. On +`main` a final step, the only one holding the Cachix token, pushes the exported +paths with `nix run .#upload-exported-paths`. The two VM checks need KVM, which +Codeberg's runners lack, so they only run locally. ## Releases diff --git a/flake.nix b/flake.nix index 34029c9..bee6f52 100644 --- a/flake.nix +++ b/flake.nix @@ -60,7 +60,12 @@ { pkgs, builds, ... }: { default = builds.bootvine; - inherit (builds) bootvine scripted-demos repo-tasks; + inherit (builds) + bootvine + scripted-demos + repo-tasks + workspace-dependencies + ; sample-unit-files = pkgs.callPackage ./vm-tests/slow-services/sample-unit-files.nix { inherit (vmTests) sampleUnits; }; @@ -86,12 +91,6 @@ ... }: let - shellScriptApp = name: description: scriptArgs: { - type = "app"; - meta.description = description; - program = lib.getExe (pkgs.writeShellApplication ({ inherit name; } // scriptArgs)); - }; - sampleMachineApps = lib.mapAttrs' ( machineName: machineDescription: lib.nameValuePair "sample-vm-${machineName}" { @@ -101,9 +100,80 @@ } ) vmTests.machineDescriptions; - releasePlzOnCodeberg = - subcommand: description: - shellScriptApp "release-plz-${subcommand}" description { + wrappedProgramTool = + { + toolName, + wrappedProgram, + addedFlags, + runtimeInputs ? [ ], + wrapperArguments ? [ ], + }: + pkgs.runCommand toolName + { + nativeBuildInputs = [ pkgs.makeBinaryWrapper ]; + meta.mainProgram = toolName; + } + '' + makeWrapper ${wrappedProgram} $out/bin/${toolName} \ + ${ + lib.escapeShellArgs ( + lib.concatMap (addedFlag: [ + "--add-flags" + addedFlag + ]) addedFlags + ) + } \ + ${lib.optionalString (runtimeInputs != [ ]) "--prefix PATH : ${lib.makeBinPath runtimeInputs}"} \ + ${lib.escapeShellArgs wrapperArguments} + ''; + + nuScriptTool = + { + scriptPath, + toolName, + runtimeInputs ? [ ], + wrapperArguments ? [ ], + }: + wrappedProgramTool { + inherit toolName runtimeInputs wrapperArguments; + wrappedProgram = lib.getExe pkgs.nushell; + addedFlags = [ + "${scriptPath}" + toolName + ]; + }; + + describedApp = appDescription: appTool: { + type = "app"; + meta.description = appDescription; + program = lib.getExe appTool; + }; + + nixBuiltPathsTool = + toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/nix-built-paths.nu; }); + + releaseTaskTool = + toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/release-tasks.nu; }); + + exportBuiltPathsHook = nixBuiltPathsTool { toolName = "export-built-paths-hook"; }; + + exportingBuildTool = + toolArguments: + nixBuiltPathsTool ( + toolArguments + // { + wrapperArguments = [ + "--set" + "EXPORT_HOOK_PROGRAM" + (lib.getExe exportBuiltPathsHook) + ]; + } + ); + + releasePlzTool = + toolName: + releaseTaskTool { + inherit toolName; runtimeInputs = [ pkgs.release-plz pkgs.gitMinimal @@ -111,63 +181,96 @@ pkgs.stdenv.cc rustToolchain ]; - runtimeEnv.PKG_CONFIG_PATH = - lib.makeSearchPathOutput "dev" "lib/pkgconfig" - builds.bootvine.buildInputs; - text = ''exec release-plz ${subcommand} --forge gitea --git-token "$CODEBERG_TOKEN"''; + wrapperArguments = [ + "--set" + "PKG_CONFIG_PATH" + (lib.makeSearchPathOutput "dev" "lib/pkgconfig" builds.bootvine.buildInputs) + ]; }; + + repoTaskTool = + toolArguments: + wrappedProgramTool ( + toolArguments + // { + wrappedProgram = lib.getExe builds.repo-tasks; + } + ); in sampleMachineApps // { - release-plz-release = releasePlzOnCodeberg "release" "Publish unpublished versions to crates.io and tag them with a Codeberg release"; - release-plz-release-pr = releasePlzOnCodeberg "release-pr" "Open or update the Codeberg pull request that bumps the version and changelog"; + release-plz-release = describedApp "Publish unpublished versions to crates.io and tag them with a Codeberg release" ( + releasePlzTool "release-plz-release" + ); + release-plz-release-pr = describedApp "Open or update the Codeberg pull request that bumps the version and changelog" ( + releasePlzTool "release-plz-release-pr" + ); record-screencasts = - shellScriptApp "record-screencasts" + describedApp "Re-record the scripted demo animations and the sample machine recordings into recordings/; publish them with publish-recordings" - { + (repoTaskTool { + toolName = "record-screencasts"; + addedFlags = [ + "record-screencasts" + "--vm-check-outputs" + "${pkgs.linkFarm "vm-check-outputs" ( + lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines + )}" + ]; runtimeInputs = [ pkgs.vhs pkgs.libwebp builds.scripted-demos builds.repo-tasks ]; - text = "exec xtask record-screencasts --vm-check-outputs ${ - pkgs.linkFarm "vm-check-outputs" ( - lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines - ) - }"; - }; + }); publish-recordings = - shellScriptApp "publish-recordings" - "Replace the recordings branch on origin with a single commit holding recordings/" - { + describedApp "Replace the recordings branch on origin with a single commit holding recordings/" + (repoTaskTool { + toolName = "publish-recordings"; + addedFlags = [ "publish-recordings" ]; runtimeInputs = [ pkgs.gitMinimal builds.repo-tasks ]; - text = "exec xtask publish-recordings"; - }; + }); - build-and-push-to-cachix = - shellScriptApp "build-and-push-to-cachix" - "Build flake outputs and push everything built to the wvhulle Cachix cache" - { + build-unless-cached = + describedApp "Build a flake output and export it to the CI workspace unless it is already in Cachix" + (exportingBuildTool { + toolName = "build-unless-cached"; + }); + + build-and-export-paths = + describedApp "Build flake outputs and export newly built paths to the CI workspace" + (exportingBuildTool { + toolName = "build-and-export-paths"; + }); + + import-exported-paths = + describedApp "Import paths exported to the CI workspace by an earlier step" + (nixBuiltPathsTool { + toolName = "import-exported-paths"; + }); + + upload-exported-paths = + describedApp "Push paths exported to the CI workspace to the wvhulle Cachix cache" + (nixBuiltPathsTool { + toolName = "upload-exported-paths"; runtimeInputs = [ pkgs.cachix ]; - text = ''exec cachix watch-exec wvhulle -- nix build --print-build-logs "$@"''; - }; + }); check-conventional-commits = - shellScriptApp "check-conventional-commits" - "Check that the commit messages in a git revision range are conventional commits" - { + describedApp "Check that the commit messages in a git revision range are conventional commits" + (releaseTaskTool { + toolName = "check-conventional-commits"; runtimeInputs = [ pkgs.convco pkgs.gitMinimal ]; - text = ''exec convco check "$@"''; - }; + }); } ); @@ -177,6 +280,12 @@ inherit (builds) rustfmt-formatting clippy-lints; bootvine-build = builds.bootvine; repo-tasks-build = builds.repo-tasks; + ci-task-tests = + pkgs.runCommand "bootvine-ci-task-tests" { nativeBuildInputs = [ pkgs.nushell ]; } + '' + nu ${./ci}/test-ci-tasks.nu + touch $out + ''; } // lib.mapAttrs' (machineName: check: lib.nameValuePair "vm-${machineName}" check) ( vmTests.checksFor pkgs diff --git a/nix/crane-builds.nix b/nix/crane-builds.nix index 1ab5128..5abebfd 100644 --- a/nix/crane-builds.nix +++ b/nix/crane-builds.nix @@ -30,6 +30,8 @@ let args: craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; } // args); in { + workspace-dependencies = cargoArtifacts; + rustfmt-formatting = craneLib.cargoFmt { inherit (commonArgs) src pname version; cargoExtraArgs = "--all";