diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4494a1656146337cf7c64c44eb1081ef172d39e1..14d8e4b46fc8339263d4f0111cdbf5df42749a02 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,23 @@ +# Define rules for a job that should run for events related to a merge request +# - merge request is opened, a new commit is pushed to its branch, etc. This +# definition does nothing by itself but can be referenced by jobs that want to +# run in this condition. +.merge_request_rules: &RUN_ON_MERGE_REQUEST + rules: + # If the pipeline is triggered by a merge request event then we should + # run. + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + + # If the pipeline is triggered by anything else then we should not run. + - when: "never" + +# As above, but rules for running when the scheduler triggers the pipeline. +.schedule_rules: &RUN_ON_SCHEDULE + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + + - when: "never" + default: # Guide the choice of an appropriate runner for all these jobs. # https://docs.gitlab.com/ee/ci/runners/#runner-runs-only-tagged-jobs @@ -10,7 +30,7 @@ variables: GET_SOURCES_ATTEMPTS: 10 docs: - stage: "build" + <<: *RUN_ON_MERGE_REQUEST script: - "nix-build --attr docs --out-link result-docs" # GitLab wants to lchown artifacts. It can't do that to store paths. Get @@ -22,12 +42,13 @@ docs: expose_as: "documentation" unit-tests: - stage: "test" + <<: *RUN_ON_MERGE_REQUEST script: - "nix-build --attr unit-tests && cat result" .morph-build: &MORPH_BUILD - stage: "test" + <<: *RUN_ON_MERGE_REQUEST + timeout: "3 hours" script: @@ -47,7 +68,7 @@ morph-build-localdev: # just needs this tweak. echo '{}' > morph/grid/${GRID}/public-keys/users.nix -morph-build-testing: +morph-build-staging: <<: *MORPH_BUILD variables: GRID: "testing" @@ -60,7 +81,7 @@ morph-build-production: vulnerability-scan: - stage: "test" + <<: *RUN_ON_MERGE_REQUEST script: - "ci-tools/vulnerability-scan security-report.json" - "ci-tools/count-vulnerabilities <security-report.json" @@ -71,14 +92,13 @@ vulnerability-scan: system-tests: - stage: "test" + <<: *RUN_ON_MERGE_REQUEST timeout: "3 hours" script: - "nix-build --attr system-tests" # A template for a job that can update one of the grids. .update-grid: &UPDATE_GRID - stage: "deploy" script: | env --ignore-environment - \ NIX_PATH="$NIX_PATH" \ @@ -91,6 +111,13 @@ system-tests: # Update the staging deployment - only on a commit to the develop branch. update-staging: <<: *UPDATE_GRID + + # https://docs.gitlab.com/ee/ci/yaml/index.html#needs + needs: + # Only deploy if the code looks good. + - "system-tests" + - "morph-build-staging" + # https://docs.gitlab.com/ee/ci/yaml/#rules rules: # https://docs.gitlab.com/ee/ci/yaml/index.html#rulesif @@ -113,6 +140,13 @@ update-staging: # Update the production deployment - only on a commit to the production branch. deploy-to-production: <<: *UPDATE_GRID + + # https://docs.gitlab.com/ee/ci/yaml/index.html#needs + needs: + # Only deploy if the code looks good. + - "system-tests" + - "morph-build-production" + # https://docs.gitlab.com/ee/ci/yaml/#rules rules: # https://docs.gitlab.com/ee/ci/yaml/index.html#rulesif @@ -124,3 +158,15 @@ deploy-to-production: # See notes in `update-staging`. name: "production" url: "https://monitoring.private.storage/" + +update-nixpkgs: + <<: *RUN_ON_SCHEDULE + script: + - | + ./ci-tools/with-ssh-agent \ + ./ci-tools/update-nixpkgs \ + "$CI_SERVER_URL" \ + "$CI_SERVER_HOST" \ + "$CI_PROJECT_PATH" \ + "$CI_PROJECT_ID" \ + "$CI_DEFAULT_BRANCH" diff --git a/ci-tools/update-nixpkgs b/ci-tools/update-nixpkgs new file mode 100755 index 0000000000000000000000000000000000000000..76f74a2e1848d1504af6925ae1991fdf335ad80f --- /dev/null +++ b/ci-tools/update-nixpkgs @@ -0,0 +1,248 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p nixUnstable git curl python3 + +# ^^ +# we get nixUnstable for the diff-closures command, mostly. +# we need git to commit and push our changes +# we need curl to create the gitlab MR +# we need python to format the data as json + +set -eux -o pipefail + +main() { + # This is a base64-encoded OpenSSH-format SSH private key that we can use + # to push and pull with git over ssh. + local SSHKEY=$1 + shift + + # This is a GitLab authentication token we can use to make API calls onto + # GitLab. + local TOKEN=$1 + shift + + # This is the URL of the root of the GitLab API. + local SERVER_URL=$1 + shift + + # This is the hostname of the GitLab server (suitable for use in a Git + # remote). + local SERVER_HOST=$1 + shift + + # This is the "group/project"-style identifier for the project we're working + # with. + local PROJECT_PATH=$1 + shift + + # The GitLab id of the project (eg, from CI_PROJECT_ID in the CI + # environment). + local PROJECT_ID=$1 + shift + + # The name of the branch on which to base changes and which to target with + # the resulting merge request. + local DEFAULT_BRANCH=$1 + shift + + # Only proceed if we have an ssh-agent. + check_agent + + # Pick a branch name into which to push our work. + local SOURCE_BRANCH="nixpkgs-upgrade-$(date +%Y-%m-%d)" + + setup_git + checkout_source_branch "$SSHKEY" "$SERVER_HOST" "$PROJECT_PATH" "$DEFAULT_BRANCH" "$SOURCE_BRANCH" + build "result-before" + + # If nothing changed, update_nixpkgs will just exit for us. + update_nixpkgs + + build "result-after" + local DIFF=$(compute_diff "./result-before" "./result-after") + commit_and_push "$SSHKEY" "$SOURCE_BRANCH" "$DIFF" + create_merge_request "$SERVER_URL" "$TOKEN" "$PROJECT_ID" "$DEFAULT_BRANCH" "$SOURCE_BRANCH" "$DIFF" +} + +# Add the ssh key required to push and (maybe) pull to the ssh-agent. This +# may have a limited lifetime in the agent so operations that are going to +# require the key should refresh it immediately before starting. +refresh_ssh_key() { + local KEY_BASE64=$1 + shift + + # A GitLab CI/CD variable set for us to use. + echo "${KEY_BASE64}" | base64 -d | ssh-add - +} + +# Make git usable by setting some global mandatory options. +setup_git() { + # We may not know the git/ssh server's host key yet. In that case, learn + # it and proceed. + export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new" + + git config --global user.email "update-bot@private.storage" + git config --global user.name "Update Bot" +} + +# Return with an error if no ssh-agent is detected. +check_agent() { + # We require an ssh-agent to be available so we can put the ssh private + # key in it. The key is given to us in memory and we don't really want to + # put it on disk anywhere so an agent is the easiest way to make it + # available for git/ssh operations. + if [ ! -v SSH_AUTH_SOCK ]; then + echo "ssh-agent is required but missing, aborting." + exit 1 + fi +} + +# Make a fresh clone of the repository, make it our working directory, and +# check out the branch we intend to commit to (the "source" of the MR). +checkout_source_branch() { + local SSHKEY=$1 + shift + local SERVER_HOST=$1 + shift + local PROJECT_PATH=$1 + shift + # The branch we'll start from. + local DEFAULT_BRANCH=$1 + shift + # The name of our branch. + local BRANCH=$1 + shift + + # To avoid messing with the checkout we're running from (which GitLab + # tends to like to share across builds) clone it to a new temporary path. + git clone . working-copy + cd working-copy + + # Make sure we know the name of a remote that points at the right place. + # Then use it to make sure the base branch is up-to-date. It usually + # should be already but in case it isn't we don't want to start from a + # stale revision. + git remote add upstream gitlab@"$SERVER_HOST":"$PROJECT_PATH".git + refresh_ssh_key "$SSHKEY" + git fetch upstream "$DEFAULT_BRANCH" + + # Typically this tool runs infrequently enough that the branch doesn't + # already exist. However, as a convenience for developing on this tool + # itself, if it does already exist, wipe it and start fresh for greater + # predictability. + git branch -D "${BRANCH}" || true + + # Then create a new branch starting from the mainline development branch. + git checkout -B "${BRANCH}" upstream/"$DEFAULT_BRANCH" +} + +# Build all of the grids (the `morph` attribute of `default.nix`) and link the +# result to the given parameter. This will give us some material to diff. +build() { + # The name of the nix result symlink. + local RESULT=$1 + shift + + # The local grid can only build if you populate its users. + echo '{}' > morph/grid/local/public-keys/users.nix + nix-build -A morph -o "$RESULT" +} + +# Perform the actual dependency update. If there are no changes, exit with an +# error code. +update_nixpkgs() { + # Spawn *another* nix-shell that has the *other* update-nixpkgs tool. + # Should sort out this mess sooner rather than later... Also, tell the + # tool (running from another checkout) to operate on this clone's package + # file instead of the one that's part of its own checkout. + nix-shell ../shell.nix --run 'update-nixpkgs ${PWD}/nixpkgs.json' + + # Show us what we did - and signal a kind of error if we did nothing + # (expected in the case where nixpkgs hasn't changed since we last ran). + if git diff --exit-code; then + echo "No changes." + exit 0 + fi +} + +# Return a description of the package changes resulting from the dependency +# update. +compute_diff() { + local LEFT=$1 + shift + local RIGHT=$1 + shift + nix --extra-experimental-features nix-command store diff-closures "$LEFT" "$RIGHT" +} + +# Commit and push all changes in the working tree along with a description of +# the package changes. +commit_and_push() { + local SSHKEY=$1 + shift + local BRANCH=$1 + shift + local DIFF=$1 + shift + + git commit -am "bump nixpkgs + +``` +$DIFF +``` +" + refresh_ssh_key "$SSHKEY" + git push --force upstream "${BRANCH}:${BRANCH}" +} + +# Create a GitLab MR for the branch we just pushed, including a description of +# the package changes it implies. +create_merge_request() { + local SERVER_URL=$1 + shift + local TOKEN=$1 + shift + local PROJECT_ID=$1 + shift + # The target branch of the MR. + local TARGET_BRANCH=$1 + shift + # The source branch of the MR. + local SOURCE_BRANCH=$1 + shift + local DIFF=$1 + shift + + local BODY=$(python3 -c ' +import sys, json, re +def rewrite_escapes(s): + # `nix store diff-closures` output is fancy and includes color codes and + # such. That looks a bit less than nice in a markdown-formatted comment so + # strip all of it. If we wanted to be fancy we could rewrite it in a + # markdown friendly way (eg using html). + return re.sub(r"\x1b\[[^m]*m", "", s) + +print(json.dumps({ + "id": sys.argv[1], + "target_branch": sys.argv[2], + "source_branch": sys.argv[3], + "remove_source_branch": True, + "title": "bump nixpkgs version", + "description": f"```\n{rewrite_escapes(sys.argv[4])}\n```", +})) +' "$PROJECT_ID" "$TARGET_BRANCH" "$SOURCE_BRANCH" "$DIFF") + + curl --verbose -X POST --data "${BODY}" --header "Content-Type: application/json" --header "PRIVATE-TOKEN: ${TOKEN}" "${SERVER_URL}/api/v4/projects/${PROJECT_ID}/merge_requests" +} + +# Pull the private ssh key and GitLab token from the environment here so we +# can work with them as arguments everywhere else. They're passed to us in +# the environment because *maybe* this is *slightly* safer than passing them +# in argv. +SSHKEY="$UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64" +TOKEN="$UPDATE_NIXPKGS_PRIVATE_TOKEN" + +# Before proceeding, remove the secrets from our environment so we don't pass +# them to child processes - none of which need them. +unset UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64 UPDATE_NIXPKGS_PRIVATE_TOKEN + +main "$SSHKEY" "$TOKEN" "$@" diff --git a/ci-tools/with-ssh-agent b/ci-tools/with-ssh-agent new file mode 100755 index 0000000000000000000000000000000000000000..7978f70b9840083ad855cb5d08e83dfab1f6a75a --- /dev/null +++ b/ci-tools/with-ssh-agent @@ -0,0 +1,14 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p openssh + +# This minimal helper just runs another process with an ssh-agent available to +# it. ssh-agent itself does most of that work for us so the main benefit of +# the script is that it guarantees ssh-agent is available for us to run. + +# Just give ssh-agent the commmand and it will run it and then exit when it +# does. This is a nice way to do process management so as to avoid leaking +# ssh-agents. Just in case cleanup fails for some reason, we'll also give +# keys a lifetime with `-t <seconds>` so secrets don't say in memory +# indefinitely. Note this means the process run by ssh-agent must finish its +# key-requiring operation within this number of seconds of adding the key. +ssh-agent -t 30 "$@" diff --git a/tools/update-nixpkgs b/tools/update-nixpkgs index 3c6832c95cee09632318f7a4ef4efc7099e317e1..12752892fab880582857d43c5cc4632a41c96d0f 100755 --- a/tools/update-nixpkgs +++ b/tools/update-nixpkgs @@ -49,7 +49,9 @@ def main(): args = parser.parse_args() repo_file = args.repo_file + print(f"reading {repo_file}") config = json.loads(repo_file.read_text()) + print(f"read {config!r}") config["url"] = get_nixos_channel_url(channel=args.channel) hash_data = get_url_hash(HASH_TYPE, name=config["name"], url=config["url"]) @@ -59,6 +61,7 @@ def main(): if args.dry_run: print(output) else: + print(f"writing to {repo_file}") repo_file.write_text(output)