update-nixpkgs

#!/usr/bin/env nix-shell
#!nix-shell -i bash -p nixUnstable git curl python3

# ^^
# we get nixUnstable for the diff-closures command, mostly.
# we need git to commit and push our changes
# we need curl to create the gitlab MR
# we need python to format the data as json

set -eux -o pipefail

main() {
    # This is a base64-encoded OpenSSH-format SSH private key that we can use
    # to push and pull with git over ssh.
    local SSHKEY=$1
    shift

    # This is a GitLab authentication token we can use to make API calls onto
    # GitLab.
    local TOKEN=$1
    shift

    # This is the URL of the root of the GitLab API.
    local SERVER_URL=$1
    shift

    # This is the hostname of the GitLab server (suitable for use in a Git
    # remote).
    local SERVER_HOST=$1
    shift

    # This is the "group/project"-style identifier for the project we're working
    # with.
    local PROJECT_PATH=$1
    shift

    # The GitLab id of the project (eg, from CI_PROJECT_ID in the CI
    # environment).
    local PROJECT_ID=$1
    shift

    # The name of the branch on which to base changes and which to target with
    # the resulting merge request.
    local DEFAULT_BRANCH=$1
    shift

    # Only proceed if we have an ssh-agent.
    check_agent

    # Pick a branch name into which to push our work.
    local SOURCE_BRANCH="nixpkgs-upgrade-$(date +%Y-%m-%d)"

    setup_git
    checkout_source_branch "$SSHKEY" "$SERVER_HOST" "$PROJECT_PATH" "$DEFAULT_BRANCH" "$SOURCE_BRANCH"
    build "result-before"

    # If nothing changed, report this and exit without an error.
    if ! update_nixpkgs; then
	echo "No changes."
	exit 0
    fi

    build "result-after"
    local DIFF=$(compute_diff "./result-before" "./result-after")
    commit_and_push "$SSHKEY" "$SOURCE_BRANCH" "$DIFF"
    create_merge_request "$SERVER_URL" "$TOKEN" "$PROJECT_ID" "$DEFAULT_BRANCH" "$SOURCE_BRANCH" "$DIFF"
}

# Add the ssh key required to push and (maybe) pull to the ssh-agent.  This
# may have a limited lifetime in the agent so operations that are going to
# require the key should refresh it immediately before starting.
refresh_ssh_key() {
    local KEY_BASE64=$1
    shift

    # A GitLab CI/CD variable set for us to use.
    echo "${KEY_BASE64}" | base64 -d | ssh-add -
}

# Make git usable by setting some global mandatory options.
setup_git() {
    # We may not know the git/ssh server's host key yet.  In that case, learn
    # it and proceed.
    export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new"

    git config --global user.email "update-bot@private.storage"
    git config --global user.name "Update Bot"
}

# Return with an error if no ssh-agent is detected.
check_agent() {
    # We require an ssh-agent to be available so we can put the ssh private
    # key in it.  The key is given to us in memory and we don't really want to
    # put it on disk anywhere so an agent is the easiest way to make it
    # available for git/ssh operations.
    if [ ! -v SSH_AUTH_SOCK ]; then
	echo "ssh-agent is required but missing, aborting."
	exit 1
    fi
}

# Make a fresh clone of the repository, make it our working directory, and
# check out the branch we intend to commit to (the "source" of the MR).
checkout_source_branch() {
    local SSHKEY=$1
    shift
    local SERVER_HOST=$1
    shift
    local PROJECT_PATH=$1
    shift
    # The branch we'll start from.
    local DEFAULT_BRANCH=$1
    shift
    # The name of our branch.
    local BRANCH=$1
    shift

    # To avoid messing with the checkout we're running from (which GitLab
    # tends to like to share across builds) clone it to a new temporary path.
    git clone . working-copy
    cd working-copy

    # Make sure we know the name of a remote that points at the right place.
    # Then use it to make sure the base branch is up-to-date.  It usually
    # should be already but in case it isn't we don't want to start from a
    # stale revision.
    git remote add upstream gitlab@"$SERVER_HOST":"$PROJECT_PATH".git
    refresh_ssh_key "$SSHKEY"
    git fetch upstream "$DEFAULT_BRANCH"

    # Typically this tool runs infrequently enough that the branch doesn't
    # already exist.  However, as a convenience for developing on this tool
    # itself, if it does already exist, wipe it and start fresh for greater
    # predictability.
    git branch -D "${BRANCH}" || true

    # Then create a new branch starting from the mainline development branch.
    git checkout -B "${BRANCH}" upstream/"$DEFAULT_BRANCH"
}

# Build all of the grids (the `morph` attribute of `default.nix`) and link the
# result to the given parameter.  This will give us some material to diff.
build() {
    # The name of the nix result symlink.
    local RESULT=$1
    shift

    # The local grid can only build if you populate its users.
    echo '{}' > morph/grid/local/public-keys/users.nix
    nix-build -A morph -o "$RESULT"
}

# Perform the actual dependency update.  If there are no changes, exit with an
# error code.
update_nixpkgs() {
    # Spawn *another* nix-shell that has the *other* update-nixpkgs tool.
    # Should sort out this mess sooner rather than later...  Also, tell the
    # tool (running from another checkout) to operate on this clone's package
    # file instead of the one that's part of its own checkout.
    nix-shell ../shell.nix --run 'update-nixpkgs ${PWD}/nixpkgs.json'

    # Signal a kind of error if we did nothing (expected in the case where
    # nixpkgs hasn't changed since we last ran).
    if git diff --exit-code; then
	return 1
    fi
}

# Return a description of the package changes resulting from the dependency
# update.
compute_diff() {
    local LEFT=$1
    shift
    local RIGHT=$1
    shift
    nix --extra-experimental-features nix-command store diff-closures "$LEFT" "$RIGHT"
}

# Commit and push all changes in the working tree along with a description of
# the package changes.
commit_and_push() {
    local SSHKEY=$1
    shift
    local BRANCH=$1
    shift
    local DIFF=$1
    shift

    git commit -am "bump nixpkgs

```
$DIFF
```
"
    refresh_ssh_key "$SSHKEY"
    git push --force upstream "${BRANCH}:${BRANCH}"
}

# Create a GitLab MR for the branch we just pushed, including a description of
# the package changes it implies.
create_merge_request() {
    local SERVER_URL=$1
    shift
    local TOKEN=$1
    shift
    local PROJECT_ID=$1
    shift
    # The target branch of the MR.
    local TARGET_BRANCH=$1
    shift
    # The source branch of the MR.
    local SOURCE_BRANCH=$1
    shift
    local DIFF=$1
    shift

    local BODY=$(python3 -c '
import sys, json, re
def rewrite_escapes(s):
    # `nix store diff-closures` output is fancy and includes color codes and
    # such.  That looks a bit less than nice in a markdown-formatted comment so
    # strip all of it.  If we wanted to be fancy we could rewrite it in a
    # markdown friendly way (eg using html).
    return re.sub(r"\x1b\[[^m]*m", "", s)

print(json.dumps({
    "id": sys.argv[1],
    "target_branch": sys.argv[2],
    "source_branch": sys.argv[3],
    "remove_source_branch": True,
    "title": "bump nixpkgs version",
    "description": f"```\n{rewrite_escapes(sys.argv[4])}\n```",
}))
' "$PROJECT_ID" "$TARGET_BRANCH" "$SOURCE_BRANCH" "$DIFF")

    curl --verbose -X POST --data "${BODY}" --header "Content-Type: application/json" --header "PRIVATE-TOKEN: ${TOKEN}" "${SERVER_URL}/api/v4/projects/${PROJECT_ID}/merge_requests"
}

# Pull the private ssh key and GitLab token from the environment here so we
# can work with them as arguments everywhere else.  They're passed to us in
# the environment because *maybe* this is *slightly* safer than passing them
# in argv.
SSHKEY="$UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64"
TOKEN="$UPDATE_NIXPKGS_PRIVATE_TOKEN"

# Before proceeding, remove the secrets from our environment so we don't pass
# them to child processes - none of which need them.
unset UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64 UPDATE_NIXPKGS_PRIVATE_TOKEN

main "$SSHKEY" "$TOKEN" "$@"