diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e9dd394372d5cdd2d86d8c675387a3814303582c..cc536cdeb4fe839675773c9f33f1e762a3ea2c07 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -82,9 +82,17 @@ system-tests:
     # Copy the deploy key from the environment to a file so we can actually
     # tell ssh to use it.
     - |
-      # The environment variable is configured with GitLab using Terraform so
-      # we can retain some bare minimum level of confidentiality.
+      # Make sure the deploy key file is not readable by anyone else.  Not
+      # that there should be anyone else looking - but OpenSSH won't even read
+      # it if it looks like it is too open.
+      umask 077
+
+      # Make up a safe-ish place on the filesystem to write the key.
       KEY_PATH="$(mktemp -d)/deploy_key"
+
+      # The environment variable holding the key is configured with GitLab
+      # using Terraform so we can retain some bare minimum level of
+      # confidentiality.
       base64 --decode "${PRIVATESTORAGEIO_STAGING_SSH_DEPLOY_KEY}" > "${KEY_PATH}"
 
     # Update the deployment