From d845d30b30c1479d45ece42a39d0bebce0156c77 Mon Sep 17 00:00:00 2001
From: Update Bot <update-bot@private.storage>
Date: Fri, 15 Jul 2022 11:26:57 -0400
Subject: [PATCH] Lots of refactoring, mostly same behavior

Some terminal code stripping logic added though
---
 .gitlab-ci.yml          |   3 +-
 ci-tools/update-nixpkgs | 235 +++++++++++++++++++++++++++++++---------
 ci-tools/with-ssh-agent |  14 +++
 3 files changed, 199 insertions(+), 53 deletions(-)
 create mode 100755 ci-tools/with-ssh-agent

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 588f65e6..8492b8ce 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -163,4 +163,5 @@ update-nixpkgs:
   # <<: *RUN_ON_SCHEDULE
   <<: *RUN_ON_MERGE_REQUEST
   script:
-    - "./ci-tools/update-nixpkgs"
+    - |
+      ./ci-tools/with-ssh-agent ./ci-tools/update-nixpkgs whetstone.private.storage PrivateStorage/PrivateStorageio "$CI_PROJECT_ID"
diff --git a/ci-tools/update-nixpkgs b/ci-tools/update-nixpkgs
index 9267393f..c192cc8a 100755
--- a/ci-tools/update-nixpkgs
+++ b/ci-tools/update-nixpkgs
@@ -1,88 +1,219 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -i bash -p nixUnstable git openssh curl python3
+#!nix-shell -i bash -p nixUnstable git curl python3
 
 # ^^
 # we get nixUnstable for the diff-closures command, mostly.
 # we need git to commit and push our changes
-# we need openssh for ssh-agent to authenticate the push
 # we need curl to create the gitlab MR
 # we need python to format the data as json
 
 set -eux -o pipefail
 
-HOST="whetstone.private.storage"
-
-__cleanup_ssh () {
-    ssh-agent -k
+main() {
+    # This is a base64-encoded OpenSSH-format SSH private key that we can use
+    # to push and pull with git over ssh.
+    SSHKEY=$1
+    shift
+
+    # This is a GitLab authentication token we can use to make API calls onto
+    # GitLab.
+    TOKEN=$1
+    shift
+
+    # This is the hostname of the GitLab instance where the project lives.
+    HOST=$1
+    shift
+
+    # This is the "group/project"-style identifier for the project we're working
+    # with.
+    SLUG=$1
+    shift
+
+    # The GitLab id of the project (eg, from CI_PROJECT_ID in the CI
+    # environment).
+    PROJECT_ID=$1
+    shift
+
+    # Only proceed if we have an ssh-agent.
+    check_agent
+
+    # Pick a branch name into which to push our work.
+    SOURCE_BRANCH="nixpkgs-upgrade-$(date +%Y-%m-%d)"
+
+    setup_git
+    checkout_source_branch "$SSHKEY" "$HOST" "$SLUG" "$SOURCE_BRANCH"
+    build "result-before"
+    if ! update_nixpkgs; then
+	# If nothing changed, that's okay, just stop here.
+	echo "No changes."
+        exit 0
+    fi
+    build "result-after"
+    DIFF=$(compute_diff "./result-before" "./result-after")
+    commit_and_push "$SSHKEY" "$SOURCE_BRANCH" "$DIFF"
+    create_merge_request "$HOST" "$TOKEN" "$PROJECT_ID" "$SOURCE_BRANCH" "$DIFF"
 }
 
-setup_ssh() {
-    # -s makes the output sh compatible, in case it can't detect this for
-    # itself.
-    #
-    # -t sets a limit on how long the key will be kept in memory.  we try to
-    # kill the agent when we're done but we can't be sure we'll always
-    # succeed.  The value is a number of seconds.
-    eval $(ssh-agent -s -t 300)
-
-    # On shell exit, run a function to kill the agent.
-    trap __cleanup_ssh EXIT
+# Add the ssh key required to push and (maybe) pull to the ssh-agent.  This
+# may have a limited lifetime in the agent so operations that are going to
+# require the key should refresh it immediately before starting.
+refresh_ssh_key() {
+    KEY_BASE64=$1
+    shift
 
     # A GitLab CI/CD variable set for us to use.
-    echo "${UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64}" | base64 -d | ssh-add -
+    echo "${KEY_BASE64}" | base64 -d | ssh-add -
+}
 
+# Make git usable by setting some global mandatory options.
+setup_git() {
     # We may not know the git/ssh server's host key yet.  In that case, learn
     # it and proceed.
     export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new"
-}
 
-setup_git() {
     git config --global user.email "update-bot@private.storage"
     git config --global user.name "Update Bot"
 }
 
-setup_ssh
-setup_git
-
-export SOURCE_BRANCH="nixpkgs-upgrade-$(date +%Y-%m-%d)"
+# Return with an error if no ssh-agent is detected.
+check_agent() {
+    # We require an ssh-agent to be available so we can put the ssh private
+    # key in it.  The key is given to us in memory and we don't really want to
+    # put it on disk anywhere so an agent is the easiest way to make it
+    # available for git/ssh operations.
+    if [ ! -v SSH_AUTH_SOCK ]; then
+	echo "ssh-agent is required but missing, aborting."
+	exit 1
+    fi
+}
 
-# Avoid messing with the checkout we're running from.
-git clone . working-copy
-cd working-copy
-git remote add upstream gitlab@whetstone.private.storage:PrivateStorage/PrivateStorageio.git
-git fetch upstream develop
-git branch -D "${SOURCE_BRANCH}" || true
-git checkout -B "${SOURCE_BRANCH}" upstream/develop
+# Make a fresh clone of the repository, make it our working directory, and
+# check out the branch we intend to commit to (the "source" of the MR).
+checkout_source_branch() {
+    SSHKEY=$1
+    shift
+    HOST=$1
+    shift
+    SLUG=$1
+    shift
+    BRANCH=$1
+    shift
+
+    # To avoid messing with the checkout we're running from (which GitLab
+    # tends to like to share across builds) clone it to a new temporary path.
+    git clone . working-copy
+    cd working-copy
+    git remote add upstream gitlab@"$HOST":"$SLUG".git
+    refresh_ssh_key "$SSHKEY"
+    git fetch upstream develop
+    # Typically this tool runs infrequently enough that the branch doesn't
+    # already exist.  However, as a convenience for developing on this tool
+    # itself, if it does already exist, wipe it and start fresh for greater
+    # predictability.
+    git branch -D "${BRANCH}" || true
+
+    # Then create a new branch starting from the mainline development branch.
+    git checkout -B "${BRANCH}" upstream/develop
+}
 
-echo '{}' > morph/grid/local/public-keys/users.nix
-nix-build -A morph -o result-before
+# Do the "before nixpkgs change" build to use as the base of the diff we
+# compute later.
+build() {
+    # The name of the nix result symlink.
+    RESULT=$1
+    shift
 
-# Spawn *another* nix-shell that has the *other* update-nixpkgs tool.  Should
-# sort out this mess sooner rather than later...
-nix-shell ../shell.nix --run 'update-nixpkgs ${PWD}/nixpkgs.json'
+    # The local grid can only build if you populate its users.
+    echo '{}' > morph/grid/local/public-keys/users.nix
+    nix-build -A morph -o "$RESULT"
+}
 
-# Show us what we did
-if git diff --exit-code; then
-    echo "No changes."
-    exit 0
-fi
+# Perform the actual dependency update.  If there are no changes, exit with an
+# error code.
+update_nixpkgs() {
+    # Spawn *another* nix-shell that has the *other* update-nixpkgs tool.
+    # Should sort out this mess sooner rather than later...  Also, tell the
+    # tool (running from another checkout) to operate on this clone's package
+    # file instead of the one that's part of its own checkout.
+    nix-shell ../shell.nix --run 'update-nixpkgs ${PWD}/nixpkgs.json'
+
+    # Show us what we did - and signal a kind of error if we did nothing
+    # (expected in the case where nixpkgs hasn't changed since we last ran).
+    if git diff --exit-code; then
+	exit 1
+    fi
+}
 
-nix-build -A morph -o result-after
-DIFF=$(nix --extra-experimental-features nix-command store diff-closures ./result-before/ ./result-after/)
+# Return a description of the package changes resulting from the dependency
+# update.
+compute_diff() {
+    LEFT=$1
+    shift
+    RIGHT=$1
+    shift
+    nix --extra-experimental-features nix-command store diff-closures "$LEFT" "$RIGHT"
+}
 
-git commit -am "bump nixpkgs version"
-git push --force upstream "${SOURCE_BRANCH}:${SOURCE_BRANCH}"
+# Commit and push all changes in the working tree along with a description of
+# the package changes.
+commit_and_push() {
+    SSHKEY=$1
+    shift
+    BRANCH=$1
+    shift
+    DIFF=$1
+    shift
+
+    git commit -am "bump nixpkgs
+
+```
+$DIFF
+```
+"
+    refresh_ssh_key "$SSHKEY"
+    git push --force upstream "${BRANCH}:${BRANCH}"
+}
 
-BODY=$(python3 -c '
-import os, sys, json
+# Create a GitLab MR for the branch we just pushed, including a description of
+# the package changes it implies.
+create_merge_request() {
+    HOST=$1
+    shift
+    TOKEN=$1
+    shift
+    CI_PROJECT_ID=$1
+    shift
+    BRANCH=$1
+    shift
+    DIFF=$1
+    shift
+
+    BODY=$(python3 -c '
+import sys, json, re
+def rewrite_escapes(s):
+    return re.sub(r"\x1b\[[^m]*m", "", s)
 print(json.dumps({
-    "id": os.environ["CI_PROJECT_ID"],
-    "source_branch": os.environ["SOURCE_BRANCH"],
+    "id": sys.argv[1],
+    "source_branch": sys.argv[2],
     "target_branch": "develop",
     "remove_source_branch": True,
     "title": "bump nixpkgs version",
-    "description": f"```\n{sys.argv[1]}\n```",
+    "description": f"```\n{rewrite_escapes(sys.argv[3])}\n```",
 }))
-' "${DIFF}")
+' "$CI_PROJECT_ID" "$BRANCH" "$DIFF")
+
+    curl --verbose -X POST --data "${BODY}" --header "Content-Type: application/json" --header "PRIVATE-TOKEN: ${TOKEN}" "https://${HOST}/api/v4/projects/${CI_PROJECT_ID}/merge_requests"
+}
+
+# Pull the private ssh key and GitLab token from the environment here so we
+# can work with them as arguments everywhere else.  They're passed to us in
+# the environment because *maybe* this is *slightly* safer than passing them
+# in argv.
+SSHKEY="$UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64"
+TOKEN="$UPDATE_NIXPKGS_PRIVATE_TOKEN"
+
+# Before proceeding, remove the secrets from our environment so we don't pass
+# them to child processes - none of which need them.
+unset UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64 UPDATE_NIXPKGS_PRIVATE_TOKEN
 
-curl --verbose -X POST --data "${BODY}" --header "Content-Type: application/json" --header "PRIVATE-TOKEN: ${UPDATE_NIXPKGS_PRIVATE_TOKEN}" "https://${HOST}/api/v4/projects/${CI_PROJECT_ID}/merge_requests"
+main "$SSHKEY" "$TOKEN" "$@"
diff --git a/ci-tools/with-ssh-agent b/ci-tools/with-ssh-agent
new file mode 100755
index 00000000..025cfc75
--- /dev/null
+++ b/ci-tools/with-ssh-agent
@@ -0,0 +1,14 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p openssh
+
+# This minimal helper just runs another process with an ssh-agent available to
+# it.  ssh-agent itself does most of that work for us so the main benefit of
+# the script is that it guarantees ssh-agent is available for us to run.
+
+# Just give ssh-agent the commmand and it will runn it and then exit when it
+# does.  This is a nice way to do process management so as to avoid leaking
+# ssh-agents.  Just in case cleanup fails for some reason, we'll also give
+# keys a lifetime with `-t <seconds>` so secrets don't say in memory
+# indefinitely.  Note this means the process run by ssh-agent must finish its
+# key-requiring operation within this number of seconds of adding the key.
+ssh-agent -t 30 "$@"
-- 
GitLab