.gitlab-ci.yml

@@ -39,7 +39,7 @@ morph-build-localdev:
     - |
       # The local grid configuration is *almost* complete enough to build.  It
       # just needs this tweak.
-      sed -i 's/undefined/\"unundefined\"/' morph/grid/${GRID}/public-keys/users.nix
+      echo '{}' > morph/grid/${GRID}/public-keys/users.nix
 
 morph-build-testing:
   <<: *MORPH_BUILD
@@ -74,22 +74,47 @@ system-tests:
 .update-grid: &UPDATE_GRID
   stage: "deploy"
   script: |
-    env --ignore-environment - NIX_PATH=$NIX_PATH GITLAB_USER_LOGIN=$GITLAB_USER_LOGIN CI_JOB_NAME=$CI_JOB_NAME CI_PIPELINE_SOURCE=$CI_PIPELINE_SOURCE CI_COMMIT_BRANCH=$CI_COMMIT_BRANCH ./ci-tools/update-grid-servers "${PRIVATESTORAGEIO_SSH_DEPLOY_KEY_PATH}" "${CI_ENVIRONMENT_NAME}"
-
-# Update the staging deployment - only on a merge to the staging branch.
+    env --ignore-environment - \
+      NIX_PATH="$NIX_PATH" \
+      GITLAB_USER_LOGIN="$GITLAB_USER_LOGIN" \
+      CI_JOB_NAME="$CI_JOB_NAME" \
+      CI_PIPELINE_SOURCE="$CI_PIPELINE_SOURCE" \
+      CI_COMMIT_BRANCH="$CI_COMMIT_BRANCH" \
+      ./ci-tools/update-grid-servers "${PRIVATESTORAGEIO_SSH_DEPLOY_KEY_PATH}" "${CI_ENVIRONMENT_NAME}"
+
+# Update the staging deployment - only on a commit to the develop branch.
 update-staging:
   <<: *UPDATE_GRID
-  only:
-    - "staging"
+  # https://docs.gitlab.com/ee/ci/yaml/#rules
+  rules:
+    # https://docs.gitlab.com/ee/ci/yaml/index.html#rulesif
+    # https://docs.gitlab.com/ee/ci/jobs/job_control.html#cicd-variable-expressions
+    # https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+    - if: '$CI_COMMIT_BRANCH == "develop"'
   environment:
+    # You can find some status information about environments in GitLab at
+    # https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/environments.
     name: "staging"
-    url: "https://privatestorage-staging.com/"
-
-# Update the production deployment - only on a merge to the production branch.
+    # The URL controls where the "View Deployment" button for this environment
+    # will take you.  The main website isn't controlled by this codebase so we
+    # don't point there.  The monitoring system *is* controlled by this
+    # codebase and it also tells us lots of stuff about other things
+    # controlled by this codebase so that seems like a good place to land.
+    # Not that I make it a habit to visit the deployment using the GitLab
+    # button...  Still, discoverability or something.
+    url: "https://monitoring.privatestorage-staging.com/"
+
+# Update the production deployment - only on a commit to the production branch.
 deploy-to-production:
   <<: *UPDATE_GRID
-  only:
-    - "production"
+  # https://docs.gitlab.com/ee/ci/yaml/#rules
+  rules:
+    # https://docs.gitlab.com/ee/ci/yaml/index.html#rulesif
+    # https://docs.gitlab.com/ee/ci/jobs/job_control.html#cicd-variable-expressions
+    # https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+    - if: '$CI_COMMIT_BRANCH == "production"'
+
   environment:
+    # See notes in `update-staging`.
     name: "production"
-    url: "https://privatestorage.io/"
+    url: "https://monitoring.private.storage/"

README.rst

 Project Hosting Moved
 =====================
 
-This project can now be found at https://whetstone.privatestorage.io/opensource/PrivateStorageio
+This project can now be found at https://whetstone.privatestorage.io/privatestorage/PrivateStorageio
 
 PrivateStorageio
 ================
@@ -13,8 +13,8 @@ Documentation
 
 There is documentation for:
 
-* Operators/Admins: ``docs/ops/README.rst``
-* Developers: ``docs/dev/README.rst``
+* Operators/Admins: `<docs/source/ops/README.rst>`_
+* Developers: `<docs/source/dev/README.rst>`_
 
 The documentation can be built using this command::
 

docs/source/conf.py

@@ -86,7 +86,6 @@ html_theme_options = {
     'logo': 'logo-ps.svg',
     'description': " ", # ugly hack to get some white space below the logo
     'fixed_sidebar': True,
-    'extra_nav_links': {"Fork me on GitHub": "https://github.com/PrivateStorageio/PrivateStorageio"},
 }
 
 # Add any paths that contain custom static files (such as style sheets) here,

docs/source/dev/README.rst

@@ -28,7 +28,7 @@ The system tests boot QEMU VMs which prevents them from running on CI at this ti
 The build requires > 10 GB of disk space,
 and the VMs might be timing out on slow or busy machines.
 If you run into timeouts,
-try `raising the number of retries <https://github.com/PrivateStorageio/PrivateStorageio/blob/e8233d2/nixos/modules/tests/run-introducer.py#L55-L62>`_.
+try `raising the number of retries <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/e8233d2/nixos/modules/tests/run-introducer.py#L55-62>`_.
 
 It is also possible go through the testing script interactively - useful for debugging::
 
@@ -36,9 +36,27 @@ It is also possible go through the testing script interactively - useful for deb
 
 This will give you a result symlink in the current directory.
 Inside that is bin/nixos-test-driver which gives you a kind of REPL for interacting with the VMs.
-The kind of `Perl in this testScript <https://github.com/PrivateStorageio/PrivateStorageio/blob/78881a3/nixos/modules/tests/private-storage.nix#L180>`_ is what you can enter into this REPL.
+The kind of `Python in this testScript <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/78881a3/nixos/modules/tests/private-storage.nix#L180>`_ is what you can enter into this REPL.
 Consult the `official documentation on NixOS Tests <https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests>`_ for more information.
 
+Updatings Pins
+--------------
+
+Nixpkgs
+```````
+
+To update the version of NixOS we deploy with, run:
+
+.. code: shell
+
+   nix-shell --run 'update-nixpkgs'
+
+That will update ``nixpkgs-2015.json`` to the latest release on the nixos-21.05 channel.
+
+To update the channel, the script will need to be updated,
+along with the filenames that have the channel in them.
+
+
 Architecture overview
 ---------------------
 
@@ -48,8 +66,5 @@ Architecture overview
 .. include::
       ../../../morph/grid/local/README.rst
 
-
-
-
 .. _Nix: https://nixos.org/nix
 

morph/grid/local/.gitignore

-.vagrant
+/.vagrant
+/public-keys/users.nix

morph/grid/local/README.rst

@@ -35,7 +35,7 @@ Use the local development environment
 
   Latest Morph honors the ``SSH_CONFIG_FILE`` environment variable (`since 3f90aa88 (March 2020, v 1.5.0) <https://github.com/DBCDK/morph/commit/3f90aa885fac1c29fce9242452fa7c0c505744ef#diff-d155ad793bd62e6ea4c44ba985049ecb13a4f4f32f799791b2bce695a16c0101>`_), so in the future this should get a bit more convenient.
 
-6. Add your SSH key to ``users.nix`` so you'll be able to log in after deploying the new configuration::
+6. Create a ``public-keys/users.nix`` file with your SSH key (see ``public-keys/users.nix.example`` for the format) so you'll be able to log in after deploying the new configuration::
 
     $EDITOR public-keys/users.nix
 

morph/grid/local/grid.nix

@@ -4,6 +4,21 @@ let
   gridlib = import ../../lib;
   grid-config = pkgs.lib.trivial.importJSON ./config.json;
 
+  ssh-users = let
+    ssh-users-file = ./public-keys/users.nix;
+  in
+    if builtins.pathExists ssh-users-file then
+      import ssh-users-file
+    else
+      # Use builtins.toString so that nix does not add the file
+      # to the nix store before including it in the string.
+      throw ''
+        ssh-keys for local grid are not configured.
+        Refusing to build a possibly inaccessible configuration.
+        Please create ${builtins.toString ssh-users-file} before building.
+        See ${builtins.toString ./README.rst} for more information.
+      '';
+
   # Module with per-grid configuration
   grid-module = {config, ...}: {
     imports = [
@@ -13,7 +28,29 @@ let
       # Give it a good SSH configuration.
       ../../../nixos/modules/ssh.nix
     ];
-    services.private-storage.sshUsers = import ./public-keys/users.nix;
+    services.private-storage.sshUsers = ssh-users;
+
+    # Include the ssh-users config in a form that can be read by nix,
+    # so the self-update deployment system can access it.
+    # nixos/modules/update-deployment imports the nix file into
+    # the checkout of this repository it creates.
+    environment.etc."nixos/ssh-users.json" = {
+      # Output the loaded value, rather than just copying the file, in case the
+      # file has external references.
+      mode = "0666";
+      text = builtins.toJSON ssh-users;
+    };
+    environment.etc."nixos/ssh-users.nix" = {
+      # This is the file that is imported by update-deployment.
+      # We don't directly read the JSON so that the script doesn't
+      # depend on the format we use.
+      mode = "0666";
+      text = ''
+        # Include the ssh-users config 
+        builtins.fromJSON (builtins.readFile ./ssh-users.json)
+      '';
+    };
+
     networking.domain = grid-config.domain;
     # Convert relative paths to absolute so library code can resolve names
     # correctly.

morph/grid/local/public-keys/users.nix → morph/grid/local/public-keys/users.nix.example

 # Add your public key. Example:
 # let key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la";
+# You can use the following to get key from the local machine.
+# let key = builtins.readFile ~/.ssh/id_ed25519.pub;
 let key = undefined;
 in { "root" = key; "vagrant" = key; }

nixos/modules/issuer.nix

@@ -122,26 +122,6 @@ in {
       description = "ZKAP Issuer";
       wantedBy = [ "multi-user.target" ];
 
-      # Make sure we have a certificate the first time, if we are running over
-      # TLS and require a certificate.
-      # ACME will issue an interim self-signed certificate, which we want to
-      # use at least in the local dev network.  But if ACME cannot get the
-      # created key signed by LE (probably because the host is not reachable
-      # from outside, or the domain is not a legit TLD) the ACME cert service
-      # will "fail". We still want to start our PaymentServer. Hence a weaker
-      # "wants" instead of a "requires" dependency.
-      # When ACME receives a fully signed cert from LE, it will reload NGINX.
-      wants = lib.optional cfg.tls "${certServiceName}.service";
-
-      after = [
-        # Make sure there is a network so we can bind to all of the
-        # interfaces.
-        "network.target"
-      ] ++
-        # Make sure we run after the certificate is issued, if we are running
-        # over TLS and require a certificate.
-        lib.optional cfg.tls "${certServiceName}.service";
-
       # It really shouldn't ever exit on its own!  If it does, it's a bug
       # we'll have to fix.  Restart it and hope it doesn't happen too much
       # before we can fix whatever the issue is.

nixos/modules/update-deployment

@@ -13,14 +13,10 @@ shift
 # configuration that controls what value is actually passed when an update is
 # triggered.
 case "${GRIDNAME}" in
-    "local")
+    "local"|"testing")
 	BRANCH="develop"
 	;;
 
-    "testing")
-	BRANCH="staging"
-	;;
-
     "production")
 	BRANCH="production"
 	;;
@@ -50,10 +46,10 @@ fi
 # Get us to a pristine checkout of the right branch.
 git -C "${CHECKOUT}" reset --hard "origin/${BRANCH}"
 
-# If we happen to be on the local grid then fix the undefined key.
+# If we happen to be on the local grid then add the required user.nix file
+# containing ssh-keys.
 if [ "${GRIDNAME}" = "local" ]; then
-    KEY="$(cat /etc/ssh/authorized_keys.d/vagrant)"
-    sed -i "s_undefined_\"${KEY}\"_" "${CHECKOUT}"/morph/grid/${GRIDNAME}/public-keys/users.nix
+    echo "import /etc/nixos/ssh-users.nix" > "${CHECKOUT}"/morph/grid/"${GRIDNAME}"/public-keys/users.nix
 fi
 
 # Compute a log message explaining what we're doing.

nixos/pkgs/zkapissuer/repo.nix

@@ -2,6 +2,6 @@
 fetchFromGitHub {
   owner = "PrivateStorageio";
   repo = "PaymentServer";
-  rev = "2ca189c51a5314eb476d1866d28ee483701af6f5";
-  sha256 = "1sb53lvi6alwdjhx4cv1gg7ywkqf18n1zjgdj3nigq3vjwxhdcpb";
+  rev = "ff30e85c231a3b5ad76426bbf8801f8f76884367";
+  sha256 = "1spz19f5z96shmfpazj0rv6877xvchf3gl49a4xahjbbsz39x34x";
 }

nixos/system-tests.nix

@@ -2,6 +2,6 @@
 let
   pkgs = import ../nixpkgs-2105.nix { };
 in {
-  private-storage = pkgs.nixosTest ./modules/tests/private-storage.nix;
-  tahoe = pkgs.nixosTest ./modules/tests/tahoe.nix;
+  private-storage = pkgs.nixosTest ./tests/private-storage.nix;
+  tahoe = pkgs.nixosTest ./tests/tahoe.nix;
 }

nixos/modules/tests/private-storage.nix → nixos/tests/private-storage.nix

@@ -14,7 +14,7 @@ let
     "ssh" "-oStrictHostKeyChecking=no" "-i" "/tmp/ssh_key" "${username}@${hostname}" ":"
   ];
 
-  # Separate helper programs so we can write as little perl inside a string
+  # Separate helper programs so we can write as little python inside a string
   # inside a nix expression as possible.
   run-introducer = ./run-introducer.py;
   run-client = ./run-client.py;
@@ -72,11 +72,11 @@ let
     networking.dhcpcd.enable = false;
   };
 
-  # Return a Perl program fragment to run a shell command on one of the nodes.
+  # Return a python program fragment to run a shell command on one of the nodes.
   # The first argument is the name of the node.  The second is a list of the
   # argv to run.
   #
-  # The program's output is piped to systemd-cat and the Perl fragment
+  # The program's output is piped to systemd-cat and the python fragment
   # evaluates to success if the command exits with a success status.
   runOnNode = node: argv:
     let
@@ -90,7 +90,7 @@ in {
     # Get a machine where we can run a Tahoe-LAFS client node.
     client =
       { config, pkgs, ourpkgs, ... }:
-      { imports = [ ../packages.nix ];
+      { imports = [ ../modules/packages.nix ];
         environment.systemPackages = [
           pkgs.daemonize
           # A Tahoe-LAFS configuration capable of using the right storage
@@ -108,9 +108,9 @@ in {
     storage =
       { config, pkgs, ... }:
       { imports =
-        [ ../packages.nix
-          ../private-storage.nix
-          ../ssh.nix
+        [ ../modules/packages.nix
+          ../modules/private-storage.nix
+          ../modules/ssh.nix
         ];
         services.private-storage = {
           enable = true;
@@ -127,9 +127,9 @@ in {
     issuer =
     { config, pkgs, ... }:
     { imports =
-      [ ../packages.nix
-        ../issuer.nix
-        ../ssh.nix
+      [ ../modules/packages.nix
+        ../modules/issuer.nix
+        ../modules/ssh.nix
       ];
       services.private-storage.sshUsers = sshUsers;
 

nixos/modules/tests/run-client.py → nixos/tests/run-client.py

@@ -33,6 +33,10 @@ def main():
     config.add_section(u"storageclient.plugins.privatestorageio-zkapauthz-v1")
     config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v1", u"redeemer", u"ristretto")
     config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v1", u"ristretto-issuer-root-url", issuerURL)
+    # This has to agree with the PaymentServer configuration at the configured
+    # issuer location.  Presently PaymentServer has 50000 hard-coded as the
+    # correct value.
+    config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v1", u"default-token-count", u"50000")
 
     with open("/tmp/client/tahoe.cfg", "wt") as cfg:
         config.write(cfg)