.gitlab-ci.yml

+# Define rules for a job that should run for events related to a merge request
+# - merge request is opened, a new commit is pushed to its branch, etc.  This
+# definition does nothing by itself but can be referenced by jobs that want to
+# run in this condition.
+.merge_request_rules: &RUN_ON_MERGE_REQUEST
+  rules:
+    # If the pipeline is triggered by a merge request event then we should
+    # run.
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+
+    # If the pipeline is triggered by anything else then we should not run.
+    - when: "never"
+
+# As above, but rules for running when the scheduler triggers the pipeline.
+.schedule_rules: &RUN_ON_SCHEDULE
+  rules:
+    # There are multiple schedules so make sure this one is for us.  The
+    # `SCHEDULE_TARGET` variable is explicitly, manually set by us in the
+    # schedule configuration.
+    - if: '$SCHEDULE_TARGET != $CI_JOB_NAME'
+      when: "never"
+
+    # Make sure this is actually a scheduled run
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
+      when: "never"
+
+    # Conditions look good: run.
+    - when: "always"
+
+stages:
+  - "build"
+  - "deploy"
+
 default:
   # Guide the choice of an appropriate runner for all these jobs.
   # https://docs.gitlab.com/ee/ci/runners/#runner-runs-only-tagged-jobs
@@ -5,23 +38,65 @@ default:
     - "nixos"
     - "shell"
 
+variables:
+  # https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts
+  GET_SOURCES_ATTEMPTS: 10
+
 docs:
+  <<: *RUN_ON_MERGE_REQUEST
   stage: "build"
   script:
-    - "nix-shell --run 'nix-build docs.nix'"
-    - "cp --recursive --no-preserve=mode result/docs/. docs/build/"
+    - "nix-build --attr docs --out-link result-docs"
+    # GitLab wants to lchown artifacts.  It can't do that to store paths.  Get
+    # a copy of the docs outside of the store.
+    - "cp --recursive --no-preserve=mode ./result-docs/docs ./docs-build/"
   artifacts:
     paths:
-      - "docs/build/"
+      - "./docs-build/"
     expose_as: "documentation"
 
 unit-tests:
-  stage: "test"
+  <<: *RUN_ON_MERGE_REQUEST
+  stage: "build"
   script:
-    - "nix-shell --run 'nix-build nixos/unit-tests.nix' && cat result"
+    - "nix-build --attr unit-tests && cat result"
+
+.morph-build: &MORPH_BUILD
+  <<: *RUN_ON_MERGE_REQUEST
+  timeout: "3 hours"
+  stage: "build"
+  script:
+    - |
+      set -x
+      # GRID is set in one of the "instantiations" of this job template.
+      nix-shell --pure --run "morph build --show-trace morph/grid/${GRID}/grid.nix"
+
+
+morph-build-localdev:
+  <<: *MORPH_BUILD
+  variables:
+    GRID: "local"
+
+  before_script:
+    - |
+      # The local grid configuration is *almost* complete enough to build.  It
+      # just needs this tweak.
+      echo '{}' > morph/grid/${GRID}/public-keys/users.nix
+
+morph-build-staging:
+  <<: *MORPH_BUILD
+  variables:
+    GRID: "testing"
+
+morph-build-production:
+  <<: *MORPH_BUILD
+  variables:
+    GRID: "production"
+
 
 vulnerability-scan:
-  stage: "test"
+  <<: *RUN_ON_MERGE_REQUEST
+  stage: "build"
   script:
     - "ci-tools/vulnerability-scan security-report.json"
     - "ci-tools/count-vulnerabilities <security-report.json"
@@ -32,29 +107,83 @@ vulnerability-scan:
 
 
 system-tests:
-  stage: "test"
+  <<: *RUN_ON_MERGE_REQUEST
   timeout: "3 hours"
+  stage: "build"
   script:
-    - "nix-shell --run 'nix-build nixos/system-tests.nix'"
+    - "nix-shell --pure --run 'nix-build --attr system-tests'"
 
-deploy-to-staging:
+# A template for a job that can update one of the grids.
+.update-grid: &UPDATE_GRID
   stage: "deploy"
-  only:
-    - "staging"
+  script: |
+    env --ignore-environment - \
+      NIX_PATH="$NIX_PATH" \
+      GITLAB_USER_LOGIN="$GITLAB_USER_LOGIN" \
+      CI_JOB_NAME="$CI_JOB_NAME" \
+      CI_PIPELINE_SOURCE="$CI_PIPELINE_SOURCE" \
+      CI_COMMIT_BRANCH="$CI_COMMIT_BRANCH" \
+      ./ci-tools/update-grid-servers "${PRIVATESTORAGEIO_SSH_DEPLOY_KEY_PATH}" "${CI_ENVIRONMENT_NAME}"
+
+# Update the staging deployment - only on a commit to the develop branch.
+update-staging:
+  <<: *UPDATE_GRID
+
+  # https://docs.gitlab.com/ee/ci/yaml/#rules
+  rules:
+    # https://docs.gitlab.com/ee/ci/yaml/index.html#rulesif
+    # https://docs.gitlab.com/ee/ci/jobs/job_control.html#cicd-variable-expressions
+    # https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+    - if: '$CI_COMMIT_BRANCH == "develop"'
   environment:
+    # You can find some status information about environments in GitLab at
+    # https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/environments.
     name: "staging"
-    url: "https://privatestorage-staging.com/"
-  script:
-    - echo "Hello $GITLAB_USER_LOGIN from $CI_JOB_NAME. I was triggered by $CI_PIPELINE_SOURCE "
-    - echo "and would like to deploy the $CI_COMMIT_BRANCH branch to the $CI_ENVIRONMENT_NAME environment."
+    # The URL controls where the "View Deployment" button for this environment
+    # will take you.  The main website isn't controlled by this codebase so we
+    # don't point there.  The monitoring system *is* controlled by this
+    # codebase and it also tells us lots of stuff about other things
+    # controlled by this codebase so that seems like a good place to land.
+    # Not that I make it a habit to visit the deployment using the GitLab
+    # button...  Still, discoverability or something.
+    url: "https://monitoring.privatestorage-staging.com/"
 
+# Update the production deployment - only on a commit to the production branch.
 deploy-to-production:
-  stage: "deploy"
-  only:
-    - "production"
+  <<: *UPDATE_GRID
+
+  # https://docs.gitlab.com/ee/ci/yaml/#rules
+  rules:
+    # https://docs.gitlab.com/ee/ci/yaml/index.html#rulesif
+    # https://docs.gitlab.com/ee/ci/jobs/job_control.html#cicd-variable-expressions
+    # https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+    - if: '$CI_COMMIT_BRANCH == "production"'
+
   environment:
+    # See notes in `update-staging`.
     name: "production"
-    url: "https://privatestorage.io/"
+    url: "https://monitoring.private.storage/"
+
+update-nixpkgs:
+  <<: *RUN_ON_SCHEDULE
+  stage: "build"
+  script:
+    - |
+      ./ci-tools/with-ssh-agent \
+          ./ci-tools/update-nixpkgs \
+              "$CI_SERVER_URL" \
+              "$CI_SERVER_HOST" \
+              "$CI_PROJECT_PATH" \
+              "$CI_PROJECT_ID" \
+              "$CI_DEFAULT_BRANCH"
+
+update-production:
+  <<: *RUN_ON_SCHEDULE
+  stage: "build"
   script:
-    - echo "Hello $GITLAB_USER_LOGIN from $CI_JOB_NAME. I was triggered by $CI_PIPELINE_SOURCE "
-    - echo "and would like to deploy the $CI_COMMIT_BRANCH branch to the $CI_ENVIRONMENT_NAME environment."
+    - |
+      ./ci-tools/update-production \
+              "$CI_SERVER_URL" \
+              "$CI_PROJECT_ID" \
+              "develop" \
+              "production"

DEPLOYMENT-NOTES.rst

+Deployment notes
+================
+
+- 2023-06-19
+
+  ZKAPAuthorizer's Tahoe-LAFS plugin name changed from "privatestorageio-zkapauthz-v1" to "privatestorageio-zkapauthz-v2".
+  This causes Tahoe-LAFS to use a different filename to persist the plugin's Foolscap fURL.
+  To preserve the original fURL value (required) each storage node needs this command run before the deployment::
+
+    cp /var/db/tahoe-lafs/storage/private/storage-plugin.privatestorageio-zkapauthz-v{1,2}.furl
+
+- 2023-04-19
+
+  The team switched from Slack to Zulip.
+  For the monitoring notifications to reach Zulip, a webhook bot has to be created in Zulip and a secret URL has to be constructed as described in `https://zulip.com/integrations/doc/grafana`_ and added to the ``private_keys`` directory (See ``grid/local/private-keys/grafana-zulip-url`` for an example).
+  Find the secret URL for production at `https://my.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/rb22ipb6gvokohzq2d2hhv6t6u`_.
+
+- 2021-12-20
+
+  `https://whetstone.private.storage/privatestorage/privatestorageops/-/issues/399`_ requires moving the PaymentServer database on the ``payments`` host onto a new dedicated filesystem.
+
+  Follow these steps *before* deploying this version of PrivateStorageio:
+
+  0. Deploy the `PrivateStorageOps change <https://whetstone.private.storage/privatestorage/privatestorageops/-/merge_requests/169>`_ that creates a new dedicated volume.
+
+  1. Put a disk label on the new dedicated volume ::
+
+     nix-shell -p parted --run 'parted /dev/nvme1n1 mklabel msdos'
+
+  2. Put a properly aligned partition in the new disk label ::
+
+     nix-shell -p parted --run 'parted /dev/nvme1n1 mkpart primary ext2 4096s 4G'
+
+  3. Create a labeled filesystem on the partition ::
+
+     mkfs.ext4 -L zkapissuer-data /dev/nvme1n1p1
+
+  4. Deploy the PrivateStorageio update.
+
+  5. Move the database file to the new location ::
+
+     mv -iv /var/lib/zkapissuer/vouchers.sqlite3 /var/lib/zkapissuer-v2
+
+  6. Clean up the old state directory ::
+
+     rm -ir /var/lib/zkapissuer
+
+  7. Start the PaymentServer service (not running because its path assertions were not met earlier) ::
+
+     systemctl start zkapissuer
+
+- 2021-10-12 The secret in ``private-keys/grafana-slack-url`` needs to be changed to remove the ``SLACKURL=`` prefix.
+
+- 2021-09-30 `Enable alerting <https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/185>`_ needs a secret in ``private-keys/grafana-slack-url`` looking like the template in ``morph/grid/local/private-keys/grafana-slack-url`` and pointing to the secret API endpoint URL saved in `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ (or create a new secret URL at https://www.slack.com/apps/A0F7XDUAZ).
+
+- 2021-09-07 `Manage access to payment metrics <https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/146>`_ requires moving and chown'ing the PaymentServer database on the ``payments`` host::
+
+   mkdir /var/lib/zkapissuer
+
+   mv /var/db/vouchers.sqlite3 /var/lib/zkapissuer/vouchers.sqlite3
+
+   chown -R zkapissuer:zkapissuer /var/lib/zkapissuer
+
+   chmod 750 /var/lib/zkapissuer
+   chmod 640 /var/lib/zkapissuer/vouchers.sqlite3

README.rst

-Project Hosting Moved
-=====================
-
-This project can now be found at https://whetstone.privatestorage.io/opensource/PrivateStorageio
-
 PrivateStorageio
 ================
 
@@ -13,12 +8,13 @@ Documentation
 
 There is documentation for:
 
-* Operators/Admins: ``docs/ops/README.rst``
-* Developers: ``docs/dev/README.rst``
+* Operators/Admins: `<docs/source/ops/README.rst>`_
+* Developers: `<docs/source/dev/README.rst>`_
 
 The documentation can be built using this command::
 
   $ nix-build docs.nix
 
-The documentation is also built on and published by CI.
+The documentation is also built on and published by CI:
+Navigate to the `list of finished jobs <https://whetstone.private.storage/privatestorage/PrivateStorageio/-/jobs>`_ and download the artefact of the latest ``docs`` build.
 

admin/create-payment-link.sh

+#!/usr/bin/env bash
+
+set -euo pipefail
+
+KEY=$1
+shift
+
+DOMAIN=$1
+shift
+
+PRODUCT_ID=$(
+    curl https://api.stripe.com/v1/products \
+	 -u "${KEY}:" \
+	 -d "name=30 GB-months" \
+	 -d "description=30 GB-months of Private.Storage storage × time" \
+	 -d "statement_descriptor=PRIVATE STORAGE" \
+	 -d "url=https://${DOMAIN}/" |
+	jp --unquoted id
+	  )
+
+echo "Product: $PRODUCT_ID"
+
+PRICE_ID=$(
+    curl https://api.stripe.com/v1/prices \
+	 -u "${KEY}:" \
+	 -d "currency=USD" \
+	 -d "unit_amount=650" \
+	 -d "tax_behavior=exclusive" \
+	 -d "product=${PRODUCT_ID}" |
+	jp --unquoted id
+	)
+
+echo "Price: $PRICE_ID"
+
+LINK_URL=$(
+    curl https://api.stripe.com/v1/payment_links \
+	 -u "${KEY}:" \
+	 -d "line_items[0][price]=${PRICE_ID}" \
+	 -d "line_items[0][quantity]=1" \
+	 -d "after_completion[type]"=redirect \
+	 -d "after_completion[redirect][url]"="https://${DOMAIN}/payment/success" |
+	jp --unquoted url
+	)
+
+echo "Payment link: $LINK_URL"

admin/create-webhook.sh

+#!/usr/bin/env bash
+
+set -euo pipefail
+
+KEY=$1
+shift
+
+DOMAIN=$1
+shift
+
+curl \
+    https://api.stripe.com/v1/webhook_endpoints \
+    -u "${KEY}:" \
+    -d url="https://payments.${DOMAIN}/v1/stripe/webhook" \
+    -d "enabled_events[]"="checkout.session.completed"

ci-tools/count-vulnerabilities

-#!/usr/bin/env python3
+#! /usr/bin/env nix-shell
+#! nix-shell -i python3 -p python3
 
 from sys import stdin
 from json import load

ci-tools/known_hosts.hro-cloud

+monitoring.deerfield.leastauthority.com,vps-50812a54.vps.ovh.net,51.38.134.175 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOIgegzAxXPhxFK8vglBlUAFTzUoCj5TxqcLS57NaL2l
+payments.deerfield.leastauthority.com,vps-3cbcf174.vps.ovh.net,217.182.78.151 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFI32csriKoUUD3e813gcEAD5CCuf8rUnary70HfJMSr
+storage001.deerfield.leastauthority.com,185.225.209.174 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKX9Ei+WdNVvIncHQZ9CdEXZeSj2zBM/NQEuqmMbep0A
+storage002.deerfield.leastauthority.com,38.170.241.34 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK3TAQj5agAv9AOZQhE95vATQKcNbNZj5Y3xMb5cjzGZ
+storage003.deerfield.leastauthority.com,ns3728736.ip-151-80-28.eu,151.80.28.108 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsh9No4PT3hHDsY/07kDSRCg1Jse38n7GY0Rk9DnyPe

ci-tools/known_hosts.production

+monitoring.private.storage ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGvKv2y+IAL4+oDnX7Cm5G9QuADBHUj9OxzLX0okf6hF
+payments.private.storage ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJBlyFTJyN+VDlzGWANKqBlXeexlX/xTpp6gb5sUlA9U
+storage001.private.storage ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILQojjGVvmjZfDcrlec8ZmpbzMEeHd4+t4DJq1R/NUXw
+storage002.private.storage ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKDWqK7FBzT4L1eoIU/iaEZNZxq3Jr613PmK2nbAXFs2
+storage003.private.storage ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHobJpQVv9GaTv8Xh9CGlL7BL5yKLxCiD3ZDdVTyt0Ep
+storage004.private.storage ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJy96VEPp617ewxdkt+8ZgWcYkLxlVG/C7bZAq0ULH+z
+storage005.private.storage ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKOgJBoER0lX2Rx8UIfv/3MVJXNFn9RldYmpU+EqAc9H

ci-tools/known_hosts.staging

+monitoring.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINI9kvEBaOMvpWqcFH+6nFvRriBECKB4RFShdPiIMkk9
+payments.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK0eO/01VFwdoZzpclrmu656eaMkE19BaxtDdkkFHMa8
+storage001.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA6iWHO9/4s3h9VIpaxgD+rgj/OQh8+jupxBoOmie3St

ci-tools/update-grid-servers

+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p jp nix openssh
+
+#
+# Tell all servers belonging to a certain grid that they should update
+# themselves to the latest configuration associated with that grid.
+#
+
+set -euxo pipefail
+
+# Find the location of this script so we can refer to data files with a known
+# relative location.
+HERE=$(dirname $0)
+
+# Get the path to the ssh key which authorizes us to deliver this
+# notification.  This path contains a base64-encoded key because of
+# limitations placed on the values of GitLab job environment variables.  We'll
+# decode it later.
+ENCODED_DEPLOY_KEY_PATH=$1
+shift
+
+# Get the name of the grid to which we're going to deliver notification.  This
+# corresponds to the name of one of the directories in the top-level `morph`
+# directory.
+GRIDNAME=$1
+shift
+
+# Tell one server to update itself.
+update_one_node() {
+    grid_name=$1
+    shift
+
+    deploy_key_path=$1
+    shift
+
+    node=$1
+    shift
+
+    # Avoid both the "host key unknown" prompt and the possibility for a
+    # man-in-the-middle attack (on every single deploy!) by referring to a
+    # pre-initialized known hosts file for this grid.
+    #
+    # Then use the specified deploy key to authenticate as the deployment user
+    # and trigger the update on the host.  There's no command here because the
+    # deployment key is restricted *only* the deloyment update command and the
+    # ssh server will supply that command itself.
+    ssh -o "UserKnownHostsFile=${HERE}/known_hosts.${grid_name}" -i "${deploy_key_path}" "deployment@${node}"
+}
+
+# Tell all servers belonging to one grid to update themselves.
+update_grid_nodes() {
+    deploy_key_path=$1
+    shift
+
+    gridname=$1
+    shift
+
+    case "$gridname" in
+	"production")
+	    grid_dir=./morph/grid/production
+	    domain=private.storage
+	    ;;
+
+	"staging")
+	    grid_dir=./morph/grid/testing
+	    domain=privatestorage-staging.com
+	    ;;
+
+	*)
+	    echo "Unknown grid: ${gridname}"
+	    exit 1
+    esac
+
+    # Find the names of all hosts that belong to this grid.  This list includes
+    # one extra string, "network", which is morph configuration stuff and we need
+    # to filter out later.
+    nodes=$(nix --extra-experimental-features nix-command eval --impure --json --expr "(builtins.concatStringsSep \" \" (builtins.attrNames (import $grid_dir/grid.nix)))" | jp --unquoted @)
+
+    # Tell every server in the network to update itself.
+    for node in ${nodes}; do
+	if [ "${node}" = "network" ]; then
+	    # This isn't a server, it's part of the morph configuration.
+	    continue
+	fi
+	update_one_node "${gridname}" "${deploy_key_path}" "${node}.${domain}"
+    done
+}
+
+decode_deploy_key() {
+    encoded_key_path=$1
+    shift
+
+    # Make sure the deploy key file is not readable by anyone else.  Not
+    # that there should be anyone else looking - but OpenSSH won't even read
+    # it if it looks like it is too open.
+    umask 077
+
+    # Make up a safe-ish place on the filesystem to write the key.
+    decoded_key_path="$(mktemp -d)/deploy_key"
+
+    # Decode the contents of the encoded key path into a decoded key path.
+    base64 --decode "${encoded_key_path}" > "${decoded_key_path}"
+
+    # If OpenSSH doesn't find a newline after the last line of the key
+    # material then it fails to parse it.  So, make sure there is one.  If
+    # there was already one, it's fine to have an extra.
+    echo >> "${decoded_key_path}"
+
+    # Remove the key from the filesystem to reduce the chance of unintentional
+    # disclosure.  Overall our handling of this key is still not *particulary*
+    # safe or secure but that's why the key is only authorized to perform a
+    # single very specific operation.
+    rm -v "${encoded_key_path}" >/dev/stderr
+
+    echo -n "${decoded_key_path}"
+}
+
+# Announce our intentions.
+show_banner() {
+    echo "Hello $GITLAB_USER_LOGIN from $CI_JOB_NAME. I was triggered by $CI_PIPELINE_SOURCE"
+    echo "and I am deploying the $CI_COMMIT_BRANCH branch to the $GRIDNAME environment."
+}
+
+show_banner
+
+DEPLOY_KEY_PATH="$(decode_deploy_key "${ENCODED_DEPLOY_KEY_PATH}")"
+
+# Update the deployment
+update_grid_nodes "${DEPLOY_KEY_PATH}" "${GRIDNAME}"
+
+# Remove the decoded key from the filesystem as well.
+rm -v "${DEPLOY_KEY_PATH}"

ci-tools/update-nixpkgs

+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p git curl python3
+
+# ^^
+# we need git to commit and push our changes
+# we need curl to create the gitlab MR
+# we need python to format the data as json
+
+set -eux -o pipefail
+
+main() {
+    # This is a base64-encoded OpenSSH-format SSH private key that we can use
+    # to push and pull with git over ssh.
+    local SSHKEY=$1
+    shift
+
+    # This is a GitLab authentication token we can use to make API calls onto
+    # GitLab.
+    local TOKEN=$1
+    shift
+
+    # This is the URL of the root of the GitLab API.
+    local SERVER_URL=$1
+    shift
+
+    # This is the hostname of the GitLab server (suitable for use in a Git
+    # remote).
+    local SERVER_HOST=$1
+    shift
+
+    # This is the "group/project"-style identifier for the project we're working
+    # with.
+    local PROJECT_PATH=$1
+    shift
+
+    # The GitLab id of the project (eg, from CI_PROJECT_ID in the CI
+    # environment).
+    local PROJECT_ID=$1
+    shift
+
+    # The name of the branch on which to base changes and which to target with
+    # the resulting merge request.
+    local DEFAULT_BRANCH=$1
+    shift
+
+    # Only proceed if we have an ssh-agent.
+    check_agent
+
+    # Pick a branch name into which to push our work.
+    local SOURCE_BRANCH="nixpkgs-upgrade-$(date +%Y-%m-%d)"
+
+    setup_git
+    checkout_source_branch "$SSHKEY" "$SERVER_HOST" "$PROJECT_PATH" "$DEFAULT_BRANCH" "$SOURCE_BRANCH"
+    build "result-before"
+
+    # If nothing changed, report this and exit without an error.
+    if ! update_nixpkgs; then
+	echo "No changes."
+	exit 0
+    fi
+
+    build "result-after"
+    local DIFF=$(compute_diff "./result-before" "./result-after")
+    commit_and_push "$SSHKEY" "$SOURCE_BRANCH" "$DIFF"
+    create_merge_request "$SERVER_URL" "$TOKEN" "$PROJECT_ID" "$DEFAULT_BRANCH" "$SOURCE_BRANCH" "$DIFF"
+}
+
+# Add the ssh key required to push and (maybe) pull to the ssh-agent.  This
+# may have a limited lifetime in the agent so operations that are going to
+# require the key should refresh it immediately before starting.
+refresh_ssh_key() {
+    local KEY_BASE64=$1
+    shift
+
+    # A GitLab CI/CD variable set for us to use.
+    echo "${KEY_BASE64}" | base64 -d | ssh-add -
+}
+
+# Make git usable by setting some global mandatory options.
+setup_git() {
+    # We may not know the git/ssh server's host key yet.  In that case, learn
+    # it and proceed.
+    export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new"
+
+    git config --global user.email "update-bot@private.storage"
+    git config --global user.name "Update Bot"
+}
+
+# Return with an error if no ssh-agent is detected.
+check_agent() {
+    # We require an ssh-agent to be available so we can put the ssh private
+    # key in it.  The key is given to us in memory and we don't really want to
+    # put it on disk anywhere so an agent is the easiest way to make it
+    # available for git/ssh operations.
+    if [ ! -v SSH_AUTH_SOCK ]; then
+	echo "ssh-agent is required but missing, aborting."
+	exit 1
+    fi
+}
+
+# Make a fresh clone of the repository, make it our working directory, and
+# check out the branch we intend to commit to (the "source" of the MR).
+checkout_source_branch() {
+    local SSHKEY=$1
+    shift
+    local SERVER_HOST=$1
+    shift
+    local PROJECT_PATH=$1
+    shift
+    # The branch we'll start from.
+    local DEFAULT_BRANCH=$1
+    shift
+    # The name of our branch.
+    local BRANCH=$1
+    shift
+
+    # To avoid messing with the checkout we're running from (which GitLab
+    # tends to like to share across builds) clone it to a new temporary path.
+    git clone . working-copy
+    cd working-copy
+
+    # Make sure we know the name of a remote that points at the right place.
+    # Then use it to make sure the base branch is up-to-date.  It usually
+    # should be already but in case it isn't we don't want to start from a
+    # stale revision.
+    git remote add upstream gitlab@"$SERVER_HOST":"$PROJECT_PATH".git
+    refresh_ssh_key "$SSHKEY"
+    git fetch upstream "$DEFAULT_BRANCH"
+
+    # Typically this tool runs infrequently enough that the branch doesn't
+    # already exist.  However, as a convenience for developing on this tool
+    # itself, if it does already exist, wipe it and start fresh for greater
+    # predictability.
+    git branch -D "${BRANCH}" || true
+
+    # Then create a new branch starting from the mainline development branch.
+    git checkout -B "${BRANCH}" upstream/"$DEFAULT_BRANCH"
+}
+
+# Build all of the grids (the `morph` attribute of `default.nix`) and link the
+# result to the given parameter.  This will give us some material to diff.
+build() {
+    # The name of the nix result symlink.
+    local RESULT=$1
+    shift
+
+    # The local grid can only build if you populate its users.
+    echo '{}' > morph/grid/local/public-keys/users.nix
+    nix-build -A morph -o "$RESULT"
+}
+
+# Perform the actual dependency update.  If there are no changes, exit with an
+# error code.
+update_nixpkgs() {
+    # Spawn *another* nix-shell that has the *other* update-nixpkgs tool.
+    # Should sort out this mess sooner rather than later...  Also, tell the
+    # tool (running from another checkout) to operate on this clone's package
+    # file instead of the one that's part of its own checkout.
+    nix-shell ../shell.nix --run 'update-nixpkgs ${PWD}/nixpkgs.json'
+
+    # Signal a kind of error if we did nothing (expected in the case where
+    # nixpkgs hasn't changed since we last ran).
+    if git diff --exit-code; then
+	return 1
+    fi
+}
+
+# Return a description of the package changes resulting from the dependency
+# update.
+compute_diff() {
+    local LEFT=$1
+    shift
+    local RIGHT=$1
+    shift
+    nix --extra-experimental-features nix-command store diff-closures "$LEFT" "$RIGHT"
+}
+
+# Commit and push all changes in the working tree along with a description of
+# the package changes.
+commit_and_push() {
+    local SSHKEY=$1
+    shift
+    local BRANCH=$1
+    shift
+    local DIFF=$1
+    shift
+
+    git commit -am "bump nixpkgs
+
+```
+$DIFF
+```
+"
+    refresh_ssh_key "$SSHKEY"
+    git push --force upstream "${BRANCH}:${BRANCH}"
+}
+
+# Create a GitLab MR for the branch we just pushed, including a description of
+# the package changes it implies.
+create_merge_request() {
+    local SERVER_URL=$1
+    shift
+    local TOKEN=$1
+    shift
+    local PROJECT_ID=$1
+    shift
+    # The target branch of the MR.
+    local TARGET_BRANCH=$1
+    shift
+    # The source branch of the MR.
+    local SOURCE_BRANCH=$1
+    shift
+    local DIFF=$1
+    shift
+
+    local BODY=$(python3 -c '
+import sys, json, re
+def rewrite_escapes(s):
+    # `nix store diff-closures` output is fancy and includes color codes and
+    # such.  That looks a bit less than nice in a markdown-formatted comment so
+    # strip all of it.  If we wanted to be fancy we could rewrite it in a
+    # markdown friendly way (eg using html).
+    return re.sub(r"\x1b\[[^m]*m", "", s)
+
+print(json.dumps({
+    "id": sys.argv[1],
+    "target_branch": sys.argv[2],
+    "source_branch": sys.argv[3],
+    "remove_source_branch": True,
+    "title": "bump nixpkgs version",
+    "description": f"```\n{rewrite_escapes(sys.argv[4])}\n```",
+}))
+' "$PROJECT_ID" "$TARGET_BRANCH" "$SOURCE_BRANCH" "$DIFF")
+
+    curl --verbose -X POST --data "${BODY}" --header "Content-Type: application/json" --header "PRIVATE-TOKEN: ${TOKEN}" "${SERVER_URL}/api/v4/projects/${PROJECT_ID}/merge_requests"
+}
+
+# Pull the private ssh key and GitLab token from the environment here so we
+# can work with them as arguments everywhere else.  They're passed to us in
+# the environment because *maybe* this is *slightly* safer than passing them
+# in argv.
+SSHKEY="$UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64"
+TOKEN="$UPDATE_NIXPKGS_PRIVATE_TOKEN"
+
+# Before proceeding, remove the secrets from our environment so we don't pass
+# them to child processes - none of which need them.
+unset UPDATE_NIXPKGS_PRIVATE_SSHKEY_BASE64 UPDATE_NIXPKGS_PRIVATE_TOKEN
+
+main "$SSHKEY" "$TOKEN" "$@"

ci-tools/update-production

+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p git curl python3
+
+set -eux -o pipefail
+
+main() {
+    local TOKEN=$1
+    shift
+    local SERVER_URL=$1
+    shift
+    local PROJECT_ID=$1
+    shift
+    local SOURCE_BRANCH=$1
+    shift
+    local TARGET_BRANCH=$1
+    shift
+
+    # Make sure the things we want to talk about are locally known.  GitLab
+    # seems to prefer to know about as few refs as possible.
+    checkout_git_ref "$SOURCE_BRANCH"
+    checkout_git_ref "$TARGET_BRANCH"
+
+    # If there have been no changes we'll just abandon this update.
+    if ! ensure_changes "$SOURCE_BRANCH" "$TARGET_BRANCH"; then
+	echo "No changes."
+	exit 0
+    fi
+
+    local NOTES=$(describe_update "$SOURCE_BRANCH" "$TARGET_BRANCH")
+
+    create_merge_request "$TOKEN" "$SERVER_URL" "$PROJECT_ID" "$SOURCE_BRANCH" "$TARGET_BRANCH" "$NOTES"
+}
+
+checkout_git_ref() {
+    local REF=$1
+    shift
+
+    git fetch origin "$REF"
+}
+
+ensure_changes() {
+    local SOURCE_BRANCH=$1
+    shift
+    local TARGET_BRANCH=$1
+    shift
+
+    if [ "$(git rev-parse origin/"$SOURCE_BRANCH")" = "$(git rev-parse origin/"$TARGET_BRANCH")" ]; then
+	return 1
+    fi
+}
+
+describe_merge_request() {
+    git show $rev | grep 'See merge request' | sed -e 's/See merge request //' | tr -d '[:space:]'
+}
+
+describe_merge_requests() {
+    local RANGE=$1
+    shift
+    local TARGET=$1
+    shift
+
+    # Find all of the relevant merge revisions
+    local onelines=$(git log --merges --first-parent -m --oneline "$RANGE" | grep "into '$TARGET'")
+
+    # Describe each merge revision
+    local IFS=$'\n'
+    for line in $onelines; do
+	local rev=$(echo "$line" | cut -d ' ' -f 1)
+	echo -n "* "
+	describe_merge_request $rev
+	echo
+    done
+}
+
+describe_update() {
+    local SOURCE_BRANCH=$1
+    shift
+    local TARGET_BRANCH=$1
+    shift
+
+    # Since the target (production) should not diverge from the source
+    # (develop) it is fine to use `..` instead of `...` in the git ranges here.
+    # `...` encounters problems related to discovering the merge base because
+    # of the way GitLab manages the git checkout on CI (I think).
+
+    local NOTES=$(git diff origin/"$TARGET_BRANCH"..origin/"$SOURCE_BRANCH" -- DEPLOYMENT-NOTES.rst)
+
+    # There often are no notes and that makes for boring reading so toss in a
+    # diffstat as well.
+    local DIFFSTAT=$(git diff --stat origin/"$TARGET_BRANCH"..origin/"$SOURCE_BRANCH")
+
+    local WHEN=$(git log --max-count=1 --format='%cI' origin/"$TARGET_BRANCH")
+
+    # Describe all of the MRs that were merged into the source branch that are
+    # about to be merged into the target branch.
+    local MR=$(describe_merge_requests origin/"$TARGET_BRANCH"..origin/"$SOURCE_BRANCH" "$SOURCE_BRANCH")
+
+    echo "\
+Changes from $SOURCE_BRANCH since $WHEN
+=======================================
+
+Deployment Notes
+----------------
+\`\`\`
+$NOTES
+\`\`\`
+
+Included Merge Requests
+-----------------------
+$MR
+
+Diff Stat
+---------
+\`\`\`
+$DIFFSTAT
+\`\`\`
+"
+}
+
+create_merge_request() {
+    local TOKEN=$1
+    shift
+    local SERVER_URL=$1
+    shift
+    local PROJECT_ID=$1
+    shift
+    # THe source branch of the MR.
+    local SOURCE_BRANCH=$1
+    shift
+    # The target branch of the MR.
+    local TARGET_BRANCH=$1
+    shift
+    local NOTES=$1
+    shift
+
+    local BODY=$(python3 -c '
+import sys, json
+print(json.dumps({
+    "id": sys.argv[1],
+    "source_branch": sys.argv[2],
+    "target_branch": sys.argv[3],
+    "remove_source_branch": True,
+    "title": f"update {sys.argv[3]}",
+    "description": sys.argv[4],
+}))
+' "$PROJECT_ID" "$SOURCE_BRANCH" "$TARGET_BRANCH" "$NOTES")
+
+    curl --verbose -X POST --data "${BODY}" --header "Content-Type: application/json" --header "PRIVATE-TOKEN: ${TOKEN}" "${SERVER_URL}/api/v4/projects/${PROJECT_ID}/merge_requests"
+}
+
+# Pull the GitLab token from the environment here so we can work with them as
+# arguments everywhere else.  They're passed to us in the environment because
+# *maybe* this is *slightly* safer than passing them in argv.
+#
+# The name is slightly weird because it is shared with the update-nixpkgs job.
+TOKEN="$UPDATE_NIXPKGS_PRIVATE_TOKEN"
+
+# Before proceeding, remove the secrets from our environment so we don't pass
+# them to child processes - none of which need them.
+unset UPDATE_NIXPKGS_PRIVATE_TOKEN
+
+main "$TOKEN" "$@"

ci-tools/vulnerability-scan

@@ -32,6 +32,12 @@ else
 fi
 '
 
+# The version (1.9.6) of vulnix in nixos-21.05 incorrectly collapses
+# derivations with the same name+version, but different sets of patches
+# applied. Therefore, we use a recent nixos-unstable version that has a newer
+# version of vulnix included.
+export NIX_PATH=nixpkgs=https://api.github.com/repos/NixOS/nixpkgs/tarball/ee084c02040e864eeeb4cf4f8538d92f7c675671
+
 # vulnix exits with an error status if there are vulnerabilities.  We told
 # GitLab to allow this by setting `allow_failure` to true in the GitLab CI
 # config.  vulnix exit status indicates what vulnix thinks happened.  If we

ci-tools/with-ssh-agent

+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p openssh
+
+# This minimal helper just runs another process with an ssh-agent available to
+# it.  ssh-agent itself does most of that work for us so the main benefit of
+# the script is that it guarantees ssh-agent is available for us to run.
+
+# Just give ssh-agent the commmand and it will run it and then exit when it
+# does.  This is a nice way to do process management so as to avoid leaking
+# ssh-agents.  Just in case cleanup fails for some reason, we'll also give
+# keys a lifetime with `-t <seconds>` so secrets don't say in memory
+# indefinitely.  Note this means the process run by ssh-agent must finish its
+# key-requiring operation within this number of seconds of adding the key.
+ssh-agent -t 30 "$@"

default.nix

+{ pkgs ? import ./nixpkgs.nix { } }:
+{
+  # Render the project documentation source to some presentation format (ie,
+  # html) with Sphinx.
+  docs = pkgs.callPackage ./docs.nix { };
+
+  # Run some system integration tests in VMs covering some of the software
+  # we're integrating (ie, application functionality).
+  system-tests = pkgs.callPackage ./nixos/system-tests.nix { };
+
+  # Run some unit tests of the Nix that ties all of these things together (ie,
+  # PrivateStorageio-internal library functionality).
+  unit-tests = pkgs.callPackage ./nixos/unit-tests.nix { };
+
+  # Build all grids into a single derivation. The derivation also has several
+  # attributes that are useful for exploring the configuration in a repl or
+  # with eval.
+  morph = pkgs.callPackage ./morph {};
+}

docs.nix

-{ pkgs ? import <nixpkgs> { } }:
+{ stdenv, lib, graphviz, plantuml, python3, sphinx }:
 let
-  # NixOS 19.03 packaged graphviz has trouble rendering our architecture
-  # overview.  Latest from upstream does alright, though.  Use that.
-  make-graphviz = (import (pkgs.path + /pkgs/tools/graphics/graphviz/base.nix) {
-    rev = "b29d8e369011b832f72e0d250a05a0a15dcb5daa";
-    sha256 = "1w61filywn9cif2nryf6vd34mxxbvv25q34fd34am1rx70bk08ps";
-    version = "b29d8e369011b832f72e0d250a05a0a15dcb5daa";
-  });
-  graphviz = (pkgs.callPackage make-graphviz { }).overrideAttrs (old: {
-    patches = [];
-  });
+  pyenv = python3.withPackages (ps: [ ps.sphinx ps.sphinxcontrib_plantuml ]);
 in
-  pkgs.callPackage ./privatestorageio.nix { inherit graphviz; }
+stdenv.mkDerivation rec {
+  version = "0.0";
+  name = "privatestorageio-${version}";
+  src = lib.cleanSource ./.;
+
+  phases = [ "unpackPhase" "buildPhase" ];
+
+  depsBuildBuild = [
+    graphviz
+    plantuml
+  ];
+
+  buildPhase = ''
+    ${pyenv}/bin/sphinx-build -W docs/ $out/docs
+  '';
+}

docs/source/conf.py → docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'PrivateStorageio'
-copyright = '2019, PrivateStorage.io, LLC'
+copyright = '2021, PrivateStorage.io, LLC'
 author = 'PrivateStorage.io, LLC'
 
 # The short X.Y version
@@ -38,8 +38,10 @@ release = '0.0'
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
+
 extensions = [
     "sphinx.ext.graphviz",
+    "sphinxcontrib.plantuml",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -59,7 +61,7 @@ master_doc = 'index'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -86,7 +88,6 @@ html_theme_options = {
     'logo': 'logo-ps.svg',
     'description': " ", # ugly hack to get some white space below the logo
     'fixed_sidebar': True,
-    'extra_nav_links': {"Fork me on GitHub": "https://github.com/PrivateStorageio/PrivateStorageio"},
 }
 
 # Add any paths that contain custom static files (such as style sheets) here,