update-nixpkgs

#!/usr/bin/env nix-shell
#!nix-shell -i bash -p nixUnstable git curl python3

# ^^
# we get nixUnstable for the diff-closures command, mostly.
# we need git to commit and push our changes
# we need curl to create the gitlab MR
# we need python to format the data as json

set -eux -o pipefail

main() {
    # This is a base64-encoded OpenSSH-format SSH private key that we can use
    # to push and pull with git over ssh.
    SSHKEY=$1
    shift

    # This is a GitLab authentication token we can use to make API calls onto
    # GitLab.
    TOKEN=$1
    shift

    # This is the hostname of the GitLab instance where the project lives.
    HOST=$1
    shift

    # This is the "group/project"-style identifier for the project we're working
    # with.
    SLUG=$1
    shift

    # The GitLab id of the project (eg, from CI_PROJECT_ID in the CI
    # environment).
    PROJECT_ID=$1
    shift

    # Only proceed if we have an ssh-agent.
    check_agent

    # Pick a branch name into which to push our work.
    SOURCE_BRANCH="nixpkgs-upgrade-$(date +%Y-%m-%d)"

    setup_git
    checkout_source_branch "$SSHKEY" "$HOST" "$SLUG" "$SOURCE_BRANCH"
    build "result-before"
    if ! update_nixpkgs; then
	# If nothing changed, that's okay, just stop here.
	echo "No changes."
        exit 0
    fi
    build "result-after"
    DIFF=$(compute_diff "./result-before" "./result-after")
    commit_and_push "$SSHKEY" "$SOURCE_BRANCH" "$DIFF"
    create_merge_request "$HOST" "$TOKEN" "$PROJECT_ID" "$SOURCE_BRANCH" "$DIFF"
}

# Add the ssh key required to push and (maybe) pull to the ssh-agent.  This
# may have a limited lifetime in the agent so operations that are going to
# require the key should refresh it immediately before starting.
refresh_ssh_key() {
    KEY_BASE64=$1
    shift

    # A GitLab CI/CD variable set for us to use.
    echo "${KEY_BASE64}" | base64 -d | ssh-add -
}

# Make git usable by setting some global mandatory options.
setup_git() {
    # We may not know the git/ssh server's host key yet.  In that case, learn
    # it and proceed.
    export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new"

    git config --global user.email "update-bot@private.storage"
    git config --global user.name "Update Bot"
}

# Return with an error if no ssh-agent is detected.
check_agent() {
    # We require an ssh-agent to be available so we can put the ssh private
    # key in it.  The key is given to us in memory and we don't really want to
    # put it on disk anywhere so an agent is the easiest way to make it
    # available for git/ssh operations.
    if [ ! -v SSH_AUTH_SOCK ]; then
	echo "ssh-agent is required but missing, aborting."
	exit 1
    fi
}

# Make a fresh clone of the repository, make it our working directory, and
# check out the branch we intend to commit to (the "source" of the MR).
checkout_source_branch() {
    SSHKEY=$1
    shift
    HOST=$1
    shift
    SLUG=$1
    shift
    BRANCH=$1
    shift

    # To avoid messing with the checkout we're running from (which GitLab
    # tends to like to share across builds) clone it to a new temporary path.
    git clone . working-copy
    cd working-copy
    git remote add upstream gitlab@"$HOST":"$SLUG".git
    refresh_ssh_key "$SSHKEY"
    git fetch upstream develop
    # Typically this tool runs infrequently enough that the branch doesn't
    # already exist.  However, as a convenience for developing on this tool
    # itself, if it does already exist, wipe it and start fresh for greater
    # predictability.
    git branch -D "${BRANCH}" || true

    # Then create a new branch starting from the mainline development branch.
    git checkout -B "${BRANCH}" upstream/develop
}

# Do the "before nixpkgs change" build to use as the base of the diff we
# compute later.
build() {
    # The name of the nix result symlink.
    RESULT=$1
    shift

    # The local grid can only build if you populate its users.
    echo '{}' > morph/grid/local/public-keys/users.nix
    nix-build -A morph -o "$RESULT"
}

# Perform the actual dependency update.  If there are no changes, exit with an
# error code.
update_nixpkgs() {
    # Spawn *another* nix-shell that has the *other* update-nixpkgs tool.
    # Should sort out this mess sooner rather than later...  Also, tell the
    # tool (running from another checkout) to operate on this clone's package
    # file instead of the one that's part of its own checkout.
    nix-shell ../shell.nix --run 'update-nixpkgs ${PWD}/nixpkgs.json'

    # Show us what we did - and signal a kind of error if we did nothing
    # (expected in the case where nixpkgs hasn't changed since we last ran).
    if git diff --exit-code; then
	exit 1
    fi
}

# Return a description of the package changes resulting from the dependency
# update.
compute_diff() {
    LEFT=$1
    shift
    RIGHT=$1
    shift
    nix --extra-experimental-features nix-command store diff-closures "$LEFT" "$RIGHT"
}

# Commit and push all changes in the working tree along with a description of
# the package changes.
commit_and_push() {
    SSHKEY=$1
    shift
    BRANCH=$1
    shift
    DIFF=$1
    shift

    git commit -am "bump nixpkgs

```
$DIFF
```
"
    refresh_ssh_key "$SSHKEY"
    git push --force upstream "${BRANCH}:${BRANCH}"
}

# Create a GitLab MR for the branch we just pushed, including a description of
# the package changes it implies.
create_merge_request() {
    HOST=$1
    shift
    TOKEN=$1
    shift
    CI_PROJECT_ID=$1
    shift
    BRANCH=$1
    shift
    DIFF=$1
    shift

    BODY=$(python3 -c '
import sys, json, re
def rewrite_escapes(s):
    return re.sub(r"\x1b\[[^m]*m", "", s)
print(json.dumps({
    "id": sys.argv[1],
    "source_branch": sys.argv[2],
    "target_branch": "develop",
    "remove_source_branch": True,
    "title": "bump nixpkgs version",
    "description": f"```\n{rewrite_escapes(sys.argv[3])}\n```",
}))
' "$CI_PROJECT_ID" "$BRANCH" "$DIFF")

    curl --verbose -X POST --data "${BODY}" --header "Content-Type: application/json" --header "PRIVATE-TOKEN: ${TOKEN}" "https://${HOST}/api/v4/projects/${CI_PROJECT_ID}/merge_requests"
}

# Pull the private ssh key and GitLab token from the environment here so we
# can work with them as arguments everywhere else.  They're passed to us in
# the environment because *maybe* this is *slightly* safer than passing them
# in argv.
SSHKEY="$UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64"
TOKEN="$UPDATE_NIXPKGS_PRIVATE_TOKEN"

# Before proceeding, remove the secrets from our environment so we don't pass
# them to child processes - none of which need them.
unset UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64 UPDATE_NIXPKGS_PRIVATE_TOKEN

main "$SSHKEY" "$TOKEN" "$@"