From d845d30b30c1479d45ece42a39d0bebce0156c77 Mon Sep 17 00:00:00 2001 From: Update Bot <update-bot@private.storage> Date: Fri, 15 Jul 2022 11:26:57 -0400 Subject: [PATCH] Lots of refactoring, mostly same behavior Some terminal code stripping logic added though --- .gitlab-ci.yml | 3 +- ci-tools/update-nixpkgs | 235 +++++++++++++++++++++++++++++++--------- ci-tools/with-ssh-agent | 14 +++ 3 files changed, 199 insertions(+), 53 deletions(-) create mode 100755 ci-tools/with-ssh-agent diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 588f65e6..8492b8ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -163,4 +163,5 @@ update-nixpkgs: # <<: *RUN_ON_SCHEDULE <<: *RUN_ON_MERGE_REQUEST script: - - "./ci-tools/update-nixpkgs" + - | + ./ci-tools/with-ssh-agent ./ci-tools/update-nixpkgs whetstone.private.storage PrivateStorage/PrivateStorageio "$CI_PROJECT_ID" diff --git a/ci-tools/update-nixpkgs b/ci-tools/update-nixpkgs index 9267393f..c192cc8a 100755 --- a/ci-tools/update-nixpkgs +++ b/ci-tools/update-nixpkgs @@ -1,88 +1,219 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p nixUnstable git openssh curl python3 +#!nix-shell -i bash -p nixUnstable git curl python3 # ^^ # we get nixUnstable for the diff-closures command, mostly. # we need git to commit and push our changes -# we need openssh for ssh-agent to authenticate the push # we need curl to create the gitlab MR # we need python to format the data as json set -eux -o pipefail -HOST="whetstone.private.storage" - -__cleanup_ssh () { - ssh-agent -k +main() { + # This is a base64-encoded OpenSSH-format SSH private key that we can use + # to push and pull with git over ssh. + SSHKEY=$1 + shift + + # This is a GitLab authentication token we can use to make API calls onto + # GitLab. + TOKEN=$1 + shift + + # This is the hostname of the GitLab instance where the project lives. + HOST=$1 + shift + + # This is the "group/project"-style identifier for the project we're working + # with. + SLUG=$1 + shift + + # The GitLab id of the project (eg, from CI_PROJECT_ID in the CI + # environment). + PROJECT_ID=$1 + shift + + # Only proceed if we have an ssh-agent. + check_agent + + # Pick a branch name into which to push our work. + SOURCE_BRANCH="nixpkgs-upgrade-$(date +%Y-%m-%d)" + + setup_git + checkout_source_branch "$SSHKEY" "$HOST" "$SLUG" "$SOURCE_BRANCH" + build "result-before" + if ! update_nixpkgs; then + # If nothing changed, that's okay, just stop here. + echo "No changes." + exit 0 + fi + build "result-after" + DIFF=$(compute_diff "./result-before" "./result-after") + commit_and_push "$SSHKEY" "$SOURCE_BRANCH" "$DIFF" + create_merge_request "$HOST" "$TOKEN" "$PROJECT_ID" "$SOURCE_BRANCH" "$DIFF" } -setup_ssh() { - # -s makes the output sh compatible, in case it can't detect this for - # itself. - # - # -t sets a limit on how long the key will be kept in memory. we try to - # kill the agent when we're done but we can't be sure we'll always - # succeed. The value is a number of seconds. - eval $(ssh-agent -s -t 300) - - # On shell exit, run a function to kill the agent. - trap __cleanup_ssh EXIT +# Add the ssh key required to push and (maybe) pull to the ssh-agent. This +# may have a limited lifetime in the agent so operations that are going to +# require the key should refresh it immediately before starting. +refresh_ssh_key() { + KEY_BASE64=$1 + shift # A GitLab CI/CD variable set for us to use. - echo "${UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64}" | base64 -d | ssh-add - + echo "${KEY_BASE64}" | base64 -d | ssh-add - +} +# Make git usable by setting some global mandatory options. +setup_git() { # We may not know the git/ssh server's host key yet. In that case, learn # it and proceed. export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new" -} -setup_git() { git config --global user.email "update-bot@private.storage" git config --global user.name "Update Bot" } -setup_ssh -setup_git - -export SOURCE_BRANCH="nixpkgs-upgrade-$(date +%Y-%m-%d)" +# Return with an error if no ssh-agent is detected. +check_agent() { + # We require an ssh-agent to be available so we can put the ssh private + # key in it. The key is given to us in memory and we don't really want to + # put it on disk anywhere so an agent is the easiest way to make it + # available for git/ssh operations. + if [ ! -v SSH_AUTH_SOCK ]; then + echo "ssh-agent is required but missing, aborting." + exit 1 + fi +} -# Avoid messing with the checkout we're running from. -git clone . working-copy -cd working-copy -git remote add upstream gitlab@whetstone.private.storage:PrivateStorage/PrivateStorageio.git -git fetch upstream develop -git branch -D "${SOURCE_BRANCH}" || true -git checkout -B "${SOURCE_BRANCH}" upstream/develop +# Make a fresh clone of the repository, make it our working directory, and +# check out the branch we intend to commit to (the "source" of the MR). +checkout_source_branch() { + SSHKEY=$1 + shift + HOST=$1 + shift + SLUG=$1 + shift + BRANCH=$1 + shift + + # To avoid messing with the checkout we're running from (which GitLab + # tends to like to share across builds) clone it to a new temporary path. + git clone . working-copy + cd working-copy + git remote add upstream gitlab@"$HOST":"$SLUG".git + refresh_ssh_key "$SSHKEY" + git fetch upstream develop + # Typically this tool runs infrequently enough that the branch doesn't + # already exist. However, as a convenience for developing on this tool + # itself, if it does already exist, wipe it and start fresh for greater + # predictability. + git branch -D "${BRANCH}" || true + + # Then create a new branch starting from the mainline development branch. + git checkout -B "${BRANCH}" upstream/develop +} -echo '{}' > morph/grid/local/public-keys/users.nix -nix-build -A morph -o result-before +# Do the "before nixpkgs change" build to use as the base of the diff we +# compute later. +build() { + # The name of the nix result symlink. + RESULT=$1 + shift -# Spawn *another* nix-shell that has the *other* update-nixpkgs tool. Should -# sort out this mess sooner rather than later... -nix-shell ../shell.nix --run 'update-nixpkgs ${PWD}/nixpkgs.json' + # The local grid can only build if you populate its users. + echo '{}' > morph/grid/local/public-keys/users.nix + nix-build -A morph -o "$RESULT" +} -# Show us what we did -if git diff --exit-code; then - echo "No changes." - exit 0 -fi +# Perform the actual dependency update. If there are no changes, exit with an +# error code. +update_nixpkgs() { + # Spawn *another* nix-shell that has the *other* update-nixpkgs tool. + # Should sort out this mess sooner rather than later... Also, tell the + # tool (running from another checkout) to operate on this clone's package + # file instead of the one that's part of its own checkout. + nix-shell ../shell.nix --run 'update-nixpkgs ${PWD}/nixpkgs.json' + + # Show us what we did - and signal a kind of error if we did nothing + # (expected in the case where nixpkgs hasn't changed since we last ran). + if git diff --exit-code; then + exit 1 + fi +} -nix-build -A morph -o result-after -DIFF=$(nix --extra-experimental-features nix-command store diff-closures ./result-before/ ./result-after/) +# Return a description of the package changes resulting from the dependency +# update. +compute_diff() { + LEFT=$1 + shift + RIGHT=$1 + shift + nix --extra-experimental-features nix-command store diff-closures "$LEFT" "$RIGHT" +} -git commit -am "bump nixpkgs version" -git push --force upstream "${SOURCE_BRANCH}:${SOURCE_BRANCH}" +# Commit and push all changes in the working tree along with a description of +# the package changes. +commit_and_push() { + SSHKEY=$1 + shift + BRANCH=$1 + shift + DIFF=$1 + shift + + git commit -am "bump nixpkgs + +``` +$DIFF +``` +" + refresh_ssh_key "$SSHKEY" + git push --force upstream "${BRANCH}:${BRANCH}" +} -BODY=$(python3 -c ' -import os, sys, json +# Create a GitLab MR for the branch we just pushed, including a description of +# the package changes it implies. +create_merge_request() { + HOST=$1 + shift + TOKEN=$1 + shift + CI_PROJECT_ID=$1 + shift + BRANCH=$1 + shift + DIFF=$1 + shift + + BODY=$(python3 -c ' +import sys, json, re +def rewrite_escapes(s): + return re.sub(r"\x1b\[[^m]*m", "", s) print(json.dumps({ - "id": os.environ["CI_PROJECT_ID"], - "source_branch": os.environ["SOURCE_BRANCH"], + "id": sys.argv[1], + "source_branch": sys.argv[2], "target_branch": "develop", "remove_source_branch": True, "title": "bump nixpkgs version", - "description": f"```\n{sys.argv[1]}\n```", + "description": f"```\n{rewrite_escapes(sys.argv[3])}\n```", })) -' "${DIFF}") +' "$CI_PROJECT_ID" "$BRANCH" "$DIFF") + + curl --verbose -X POST --data "${BODY}" --header "Content-Type: application/json" --header "PRIVATE-TOKEN: ${TOKEN}" "https://${HOST}/api/v4/projects/${CI_PROJECT_ID}/merge_requests" +} + +# Pull the private ssh key and GitLab token from the environment here so we +# can work with them as arguments everywhere else. They're passed to us in +# the environment because *maybe* this is *slightly* safer than passing them +# in argv. +SSHKEY="$UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64" +TOKEN="$UPDATE_NIXPKGS_PRIVATE_TOKEN" + +# Before proceeding, remove the secrets from our environment so we don't pass +# them to child processes - none of which need them. +unset UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64 UPDATE_NIXPKGS_PRIVATE_TOKEN -curl --verbose -X POST --data "${BODY}" --header "Content-Type: application/json" --header "PRIVATE-TOKEN: ${UPDATE_NIXPKGS_PRIVATE_TOKEN}" "https://${HOST}/api/v4/projects/${CI_PROJECT_ID}/merge_requests" +main "$SSHKEY" "$TOKEN" "$@" diff --git a/ci-tools/with-ssh-agent b/ci-tools/with-ssh-agent new file mode 100755 index 00000000..025cfc75 --- /dev/null +++ b/ci-tools/with-ssh-agent @@ -0,0 +1,14 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p openssh + +# This minimal helper just runs another process with an ssh-agent available to +# it. ssh-agent itself does most of that work for us so the main benefit of +# the script is that it guarantees ssh-agent is available for us to run. + +# Just give ssh-agent the commmand and it will runn it and then exit when it +# does. This is a nice way to do process management so as to avoid leaking +# ssh-agents. Just in case cleanup fails for some reason, we'll also give +# keys a lifetime with `-t <seconds>` so secrets don't say in memory +# indefinitely. Note this means the process run by ssh-agent must finish its +# key-requiring operation within this number of seconds of adding the key. +ssh-agent -t 30 "$@" -- GitLab