diff --git a/.woodpecker/release-pr-and-publish.yaml b/.woodpecker/release-pr-and-publish.yaml index 92dc7f8..c22fbab 100644 --- a/.woodpecker/release-pr-and-publish.yaml +++ b/.woodpecker/release-pr-and-publish.yaml @@ -34,12 +34,10 @@ steps: image: nixos/nix:latest environment: *release_env commands: - - git remote set-url origin "https://wvhulle:$${CODEBERG_TOKEN}@codeberg.org/$${CI_REPO}.git" - - git checkout -B main "$${CI_COMMIT_SHA}" - - nix run .#release-plz-release + - nix run --quiet .#release-plz-release - name: update-release-pr image: nixos/nix:latest environment: *release_env commands: - - nix run .#release-plz-release-pr + - nix run --quiet .#release-plz-release-pr diff --git a/.woodpecker/rust-and-flake-checks.yaml b/.woodpecker/rust-and-flake-checks.yaml index 2a0d04d..97df0b9 100644 --- a/.woodpecker/rust-and-flake-checks.yaml +++ b/.woodpecker/rust-and-flake-checks.yaml @@ -8,74 +8,82 @@ variables: experimental-features = nix-command flakes build-users-group = accept-flake-config = true - - &cache_writing_env - <<: *nix_config - CACHIX_AUTH_TOKEN: - from_secret: cachix_auth_token - - &on_main_branch - - event: [push, manual] - branch: main - - &off_main_branch - - event: pull_request - - event: [push, manual] - branch: - exclude: main + - &after_dependency_step + - status: [success, failure] - &on_pull_requests - event: pull_request steps: - - name: build-test-and-cache-bootvine + - name: build-workspace-dependencies image: nixos/nix:latest depends_on: [] - when: *on_main_branch - environment: *cache_writing_env + environment: *nix_config commands: - - nix run .#build-and-push-to-cachix -- .#checks.x86_64-linux.bootvine-build + - nix run --quiet .#build-unless-cached -- .#packages.x86_64-linux.workspace-dependencies - name: build-and-test-bootvine image: nixos/nix:latest - depends_on: [] - when: *off_main_branch + depends_on: [build-workspace-dependencies] + when: *after_dependency_step environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.bootvine-build - - - name: build-and-cache-repo-tasks - image: nixos/nix:latest - depends_on: [] - when: *on_main_branch - environment: *cache_writing_env - commands: - - nix run .#build-and-push-to-cachix -- .#checks.x86_64-linux.repo-tasks-build + - nix run --quiet .#import-exported-paths + - nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.bootvine-build - name: build-repo-tasks image: nixos/nix:latest - depends_on: [] - when: *off_main_branch + depends_on: [build-workspace-dependencies] + when: *after_dependency_step environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.repo-tasks-build + - nix run --quiet .#import-exported-paths + - nix run --quiet .#build-and-export-paths -- .#checks.x86_64-linux.repo-tasks-build - name: lint-rust-with-clippy image: nixos/nix:latest - depends_on: [] + depends_on: [build-workspace-dependencies] + when: *after_dependency_step environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.clippy-lints + - nix run --quiet .#import-exported-paths + - nix build --quiet --print-build-logs .#checks.x86_64-linux.clippy-lints - name: check-rust-formatting image: nixos/nix:latest - depends_on: [] + depends_on: [build-workspace-dependencies] + when: *after_dependency_step environment: *nix_config commands: - - nix build --print-build-logs .#checks.x86_64-linux.rustfmt-formatting + - nix run --quiet .#import-exported-paths + - nix build --quiet --print-build-logs .#checks.x86_64-linux.rustfmt-formatting + + - name: upload-build-cache + image: nixos/nix:latest + depends_on: [build-and-test-bootvine, build-repo-tasks] + when: + - event: [push, manual] + branch: main + status: [success, failure] + environment: + <<: *nix_config + CACHIX_AUTH_TOKEN: + from_secret: cachix_auth_token + commands: + - nix run --quiet .#upload-exported-paths - name: evaluate-flake-outputs image: nixos/nix:latest depends_on: [] environment: *nix_config commands: - - nix flake check --no-build + - nix flake check --quiet --no-build + + - name: test-ci-tasks + image: nixos/nix:latest + depends_on: [] + environment: *nix_config + commands: + - nix build --quiet --print-build-logs .#checks.x86_64-linux.ci-task-tests - name: check-conventional-commit-messages image: nixos/nix:latest @@ -83,5 +91,4 @@ steps: when: *on_pull_requests environment: *nix_config commands: - - git fetch --quiet origin "$${CI_COMMIT_TARGET_BRANCH}" - - nix run .#check-conventional-commits -- FETCH_HEAD..HEAD + - nix run --quiet .#check-conventional-commits -- --fetch-target-branch FETCH_HEAD..HEAD diff --git a/ci/nix-built-paths.nu b/ci/nix-built-paths.nu new file mode 100755 index 0000000..575156d --- /dev/null +++ b/ci/nix-built-paths.nu @@ -0,0 +1,79 @@ +#!/usr/bin/env nu + +const EXPORT_DIRECTORY_NAME = ".nix-built-paths" +const PATH_LIST_FILE_NAME = "paths.txt" +const CACHIX_CACHE_NAME = "wvhulle" +const CACHIX_CACHE_URL = "https://wvhulle.cachix.org" + +def export-directory []: nothing -> path { + $env.CI_WORKSPACE | path join $EXPORT_DIRECTORY_NAME +} + +export def store-paths-from-lines []: string -> list { + lines | compact --empty +} + +def exported-store-paths []: nothing -> list { + let path_list_file = export-directory | path join $PATH_LIST_FILE_NAME + if ($path_list_file | path exists) { + open --raw $path_list_file | store-paths-from-lines + } else { + [] + } +} + +export def nix-config-with-hook [existing_config: string, hook_program: string]: nothing -> string { + $"($existing_config)\npost-build-hook = ($hook_program)" +} + +def "main export-built-paths-hook" []: nothing -> nothing { + let built_paths = $env.OUT_PATHS | split row " " | compact --empty + let directory = export-directory + mkdir $directory + ^nix copy --quiet --to $"file://($directory)?compression=zstd" ...$built_paths + $built_paths | each {|built_path| $"($built_path)\n" } | str join | save --append ($directory | path join $PATH_LIST_FILE_NAME) +} + +def --wrapped "main build-and-export-paths" [ + ...build_arguments: string +]: nothing -> nothing { + let hook_config = nix-config-with-hook ($env.NIX_CONFIG? | default "") $env.EXPORT_HOOK_PROGRAM + with-env {NIX_CONFIG: $hook_config} { + ^nix build --quiet --print-build-logs ...$build_arguments + } +} + +def "main build-unless-cached" [ + installable: string +]: nothing -> nothing { + let built_output = ^nix eval --quiet --raw $"($installable).outPath" + let cache_lookup = do { ^nix path-info --quiet --store $CACHIX_CACHE_URL $built_output } | complete + if $cache_lookup.exit_code == 0 { + print $"($built_output) is already in Cachix" + } else { + main build-and-export-paths $installable + } +} + +def "main import-exported-paths" []: nothing -> nothing { + let exported_paths = exported-store-paths + if ($exported_paths | is-empty) { + print "No exported paths to import" + } else { + ^nix copy --quiet --no-check-sigs --from $"file://(export-directory)" ...$exported_paths + } +} + +def "main upload-exported-paths" []: nothing -> nothing { + let exported_paths = exported-store-paths + if ($exported_paths | is-empty) { + print "No newly built paths to upload" + } else { + main import-exported-paths + ^cachix push $CACHIX_CACHE_NAME ...$exported_paths + } +} + +def main []: nothing -> nothing { + print "Run a subcommand, for example: build-and-export-paths" +} diff --git a/ci/release-tasks.nu b/ci/release-tasks.nu new file mode 100755 index 0000000..c629c73 --- /dev/null +++ b/ci/release-tasks.nu @@ -0,0 +1,36 @@ +#!/usr/bin/env nu + +const CODEBERG_USER = "wvhulle" +const RELEASE_BRANCH_NAME = "main" + +export def authenticated-origin-url [repository_slug: string, access_token: string]: nothing -> string { + $"https://($CODEBERG_USER):($access_token)@codeberg.org/($repository_slug).git" +} + +def run-release-plz [release_subcommand: string]: nothing -> nothing { + ^release-plz $release_subcommand --forge gitea --git-token $env.CODEBERG_TOKEN +} + +def "main release-plz-release" []: nothing -> nothing { + ^git remote set-url origin (authenticated-origin-url $env.CI_REPO $env.CODEBERG_TOKEN) + ^git checkout -B $RELEASE_BRANCH_NAME $env.CI_COMMIT_SHA + run-release-plz release +} + +def "main release-plz-release-pr" []: nothing -> nothing { + run-release-plz release-pr +} + +def "main check-conventional-commits" [ + --fetch-target-branch + ...revision_ranges: string +]: nothing -> nothing { + if $fetch_target_branch { + ^git fetch --quiet origin $env.CI_COMMIT_TARGET_BRANCH + } + ^convco check ...$revision_ranges +} + +def main []: nothing -> nothing { + print "Run a subcommand, for example: check-conventional-commits" +} diff --git a/ci/test-ci-tasks.nu b/ci/test-ci-tasks.nu new file mode 100755 index 0000000..1062f4d --- /dev/null +++ b/ci/test-ci-tasks.nu @@ -0,0 +1,14 @@ +#!/usr/bin/env nu + +use std/assert +use ./nix-built-paths.nu [nix-config-with-hook store-paths-from-lines] +use ./release-tasks.nu [authenticated-origin-url] + +def main []: nothing -> nothing { + assert equal (nix-config-with-hook "max-jobs = 1" "/nix/store/hook") "max-jobs = 1\npost-build-hook = /nix/store/hook" + assert equal (nix-config-with-hook "" "/nix/store/hook") "\npost-build-hook = /nix/store/hook" + assert equal ("/nix/store/first\n\n/nix/store/second\n" | store-paths-from-lines) ["/nix/store/first" "/nix/store/second"] + assert equal ("" | store-paths-from-lines) [] + assert equal (authenticated-origin-url "wvhulle/bootvine" "sample-token") "https://wvhulle:sample-token@codeberg.org/wvhulle/bootvine.git" + print "CI task tests passed" +} diff --git a/docs/development.md b/docs/development.md index d089092..0ade169 100644 --- a/docs/development.md +++ b/docs/development.md @@ -9,6 +9,7 @@ | `xtask/` | repository tasks run with `cargo xtask` | | `vm-tests/` | the sample NixOS machines, one folder each with its VM check | | `nix/` | the crane builds and the NixOS module | +| `ci/` | the Nushell scripts behind the CI apps in `flake.nix` | | `recordings/` | ignored output of `record-screencasts`: the scripted demo GIFs and the sample machine WebP recordings, published on the `recordings` branch for the README and the [guide](guide.md) | `cargo build` and `cargo test` act on every crate. `cargo run` needs the binary: @@ -46,9 +47,17 @@ shell) or `pre-commit install` ([pre-commit](https://pre-commit.com)). `.woodpecker/rust-and-flake-checks.yaml` runs on Codeberg's [Woodpecker CI](https://woodpecker-ci.org) for every push and pull request: the bootvine build with its tests, the xtask build, clippy, rustfmt, an evaluation of every flake output, and the commit-message check on -pull requests. Builds on `main` are pushed to the `wvhulle` [Cachix](https://www.cachix.org) cache, which -`nixConfig` in `flake.nix` offers as a substituter. The two VM checks need KVM, -which Codeberg's runners lack, so they only run locally. +pull requests. Every step is a plain `nix` call; the scripting lives in `ci/` +and is tested by `nix build .#checks.x86_64-linux.ci-task-tests`. + +Steps do not share `/nix/store`. The first step builds the workspace dependencies +unless they are already in the `wvhulle` [Cachix](https://www.cachix.org) cache, +which `nixConfig` in `flake.nix` offers as a substituter, and exports what it built +to the workspace. The build and lint steps import those paths with +`nix run .#import-exported-paths` and run even when an earlier step failed. On +`main` a final step, the only one holding the Cachix token, pushes the exported +paths with `nix run .#upload-exported-paths`. The two VM checks need KVM, which +Codeberg's runners lack, so they only run locally. ## Releases diff --git a/flake.nix b/flake.nix index 34029c9..bee6f52 100644 --- a/flake.nix +++ b/flake.nix @@ -60,7 +60,12 @@ { pkgs, builds, ... }: { default = builds.bootvine; - inherit (builds) bootvine scripted-demos repo-tasks; + inherit (builds) + bootvine + scripted-demos + repo-tasks + workspace-dependencies + ; sample-unit-files = pkgs.callPackage ./vm-tests/slow-services/sample-unit-files.nix { inherit (vmTests) sampleUnits; }; @@ -86,12 +91,6 @@ ... }: let - shellScriptApp = name: description: scriptArgs: { - type = "app"; - meta.description = description; - program = lib.getExe (pkgs.writeShellApplication ({ inherit name; } // scriptArgs)); - }; - sampleMachineApps = lib.mapAttrs' ( machineName: machineDescription: lib.nameValuePair "sample-vm-${machineName}" { @@ -101,9 +100,80 @@ } ) vmTests.machineDescriptions; - releasePlzOnCodeberg = - subcommand: description: - shellScriptApp "release-plz-${subcommand}" description { + wrappedProgramTool = + { + toolName, + wrappedProgram, + addedFlags, + runtimeInputs ? [ ], + wrapperArguments ? [ ], + }: + pkgs.runCommand toolName + { + nativeBuildInputs = [ pkgs.makeBinaryWrapper ]; + meta.mainProgram = toolName; + } + '' + makeWrapper ${wrappedProgram} $out/bin/${toolName} \ + ${ + lib.escapeShellArgs ( + lib.concatMap (addedFlag: [ + "--add-flags" + addedFlag + ]) addedFlags + ) + } \ + ${lib.optionalString (runtimeInputs != [ ]) "--prefix PATH : ${lib.makeBinPath runtimeInputs}"} \ + ${lib.escapeShellArgs wrapperArguments} + ''; + + nuScriptTool = + { + scriptPath, + toolName, + runtimeInputs ? [ ], + wrapperArguments ? [ ], + }: + wrappedProgramTool { + inherit toolName runtimeInputs wrapperArguments; + wrappedProgram = lib.getExe pkgs.nushell; + addedFlags = [ + "${scriptPath}" + toolName + ]; + }; + + describedApp = appDescription: appTool: { + type = "app"; + meta.description = appDescription; + program = lib.getExe appTool; + }; + + nixBuiltPathsTool = + toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/nix-built-paths.nu; }); + + releaseTaskTool = + toolArguments: nuScriptTool (toolArguments // { scriptPath = ./ci/release-tasks.nu; }); + + exportBuiltPathsHook = nixBuiltPathsTool { toolName = "export-built-paths-hook"; }; + + exportingBuildTool = + toolArguments: + nixBuiltPathsTool ( + toolArguments + // { + wrapperArguments = [ + "--set" + "EXPORT_HOOK_PROGRAM" + (lib.getExe exportBuiltPathsHook) + ]; + } + ); + + releasePlzTool = + toolName: + releaseTaskTool { + inherit toolName; runtimeInputs = [ pkgs.release-plz pkgs.gitMinimal @@ -111,63 +181,96 @@ pkgs.stdenv.cc rustToolchain ]; - runtimeEnv.PKG_CONFIG_PATH = - lib.makeSearchPathOutput "dev" "lib/pkgconfig" - builds.bootvine.buildInputs; - text = ''exec release-plz ${subcommand} --forge gitea --git-token "$CODEBERG_TOKEN"''; + wrapperArguments = [ + "--set" + "PKG_CONFIG_PATH" + (lib.makeSearchPathOutput "dev" "lib/pkgconfig" builds.bootvine.buildInputs) + ]; }; + + repoTaskTool = + toolArguments: + wrappedProgramTool ( + toolArguments + // { + wrappedProgram = lib.getExe builds.repo-tasks; + } + ); in sampleMachineApps // { - release-plz-release = releasePlzOnCodeberg "release" "Publish unpublished versions to crates.io and tag them with a Codeberg release"; - release-plz-release-pr = releasePlzOnCodeberg "release-pr" "Open or update the Codeberg pull request that bumps the version and changelog"; + release-plz-release = describedApp "Publish unpublished versions to crates.io and tag them with a Codeberg release" ( + releasePlzTool "release-plz-release" + ); + release-plz-release-pr = describedApp "Open or update the Codeberg pull request that bumps the version and changelog" ( + releasePlzTool "release-plz-release-pr" + ); record-screencasts = - shellScriptApp "record-screencasts" + describedApp "Re-record the scripted demo animations and the sample machine recordings into recordings/; publish them with publish-recordings" - { + (repoTaskTool { + toolName = "record-screencasts"; + addedFlags = [ + "record-screencasts" + "--vm-check-outputs" + "${pkgs.linkFarm "vm-check-outputs" ( + lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines + )}" + ]; runtimeInputs = [ pkgs.vhs pkgs.libwebp builds.scripted-demos builds.repo-tasks ]; - text = "exec xtask record-screencasts --vm-check-outputs ${ - pkgs.linkFarm "vm-check-outputs" ( - lib.mapAttrs (machineName: _: self.checks.${system}."vm-${machineName}") vmTests.machines - ) - }"; - }; + }); publish-recordings = - shellScriptApp "publish-recordings" - "Replace the recordings branch on origin with a single commit holding recordings/" - { + describedApp "Replace the recordings branch on origin with a single commit holding recordings/" + (repoTaskTool { + toolName = "publish-recordings"; + addedFlags = [ "publish-recordings" ]; runtimeInputs = [ pkgs.gitMinimal builds.repo-tasks ]; - text = "exec xtask publish-recordings"; - }; + }); - build-and-push-to-cachix = - shellScriptApp "build-and-push-to-cachix" - "Build flake outputs and push everything built to the wvhulle Cachix cache" - { + build-unless-cached = + describedApp "Build a flake output and export it to the CI workspace unless it is already in Cachix" + (exportingBuildTool { + toolName = "build-unless-cached"; + }); + + build-and-export-paths = + describedApp "Build flake outputs and export newly built paths to the CI workspace" + (exportingBuildTool { + toolName = "build-and-export-paths"; + }); + + import-exported-paths = + describedApp "Import paths exported to the CI workspace by an earlier step" + (nixBuiltPathsTool { + toolName = "import-exported-paths"; + }); + + upload-exported-paths = + describedApp "Push paths exported to the CI workspace to the wvhulle Cachix cache" + (nixBuiltPathsTool { + toolName = "upload-exported-paths"; runtimeInputs = [ pkgs.cachix ]; - text = ''exec cachix watch-exec wvhulle -- nix build --print-build-logs "$@"''; - }; + }); check-conventional-commits = - shellScriptApp "check-conventional-commits" - "Check that the commit messages in a git revision range are conventional commits" - { + describedApp "Check that the commit messages in a git revision range are conventional commits" + (releaseTaskTool { + toolName = "check-conventional-commits"; runtimeInputs = [ pkgs.convco pkgs.gitMinimal ]; - text = ''exec convco check "$@"''; - }; + }); } ); @@ -177,6 +280,12 @@ inherit (builds) rustfmt-formatting clippy-lints; bootvine-build = builds.bootvine; repo-tasks-build = builds.repo-tasks; + ci-task-tests = + pkgs.runCommand "bootvine-ci-task-tests" { nativeBuildInputs = [ pkgs.nushell ]; } + '' + nu ${./ci}/test-ci-tasks.nu + touch $out + ''; } // lib.mapAttrs' (machineName: check: lib.nameValuePair "vm-${machineName}" check) ( vmTests.checksFor pkgs diff --git a/nix/crane-builds.nix b/nix/crane-builds.nix index 1ab5128..5abebfd 100644 --- a/nix/crane-builds.nix +++ b/nix/crane-builds.nix @@ -30,6 +30,8 @@ let args: craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; } // args); in { + workspace-dependencies = cargoArtifacts; + rustfmt-formatting = craneLib.cargoFmt { inherit (commonArgs) src pname version; cargoExtraArgs = "--all";