Merge branch '399.dedicated-voucher-database-volume' into 'develop'

use a different state directory for the issuer's voucher database

Closes privatestorageops#399

See merge request !226

DEPLOYMENT-NOTES.rst

 Deployment notes
 ================
 
+- 2021-12-20
+
+  `https://whetstone.privatestorage.io/privatestorage/privatestorageops/-/issues/399`_ requires moving the PaymentServer database on the ``payments`` host onto a new dedicated filesystem.
+
+  Follow these steps *before* deploying this version of PrivateStorageio:
+
+  0. Deploy the `PrivateStorageOps change <https://whetstone.privatestorage.io/privatestorage/privatestorageops/-/merge_requests/169>`_ that creates a new dedicated volume.
+
+  1. Put a disk label on the new dedicated volume ::
+
+     nix-shell -p parted --run 'parted /dev/nvme1n1 mklabel msdos'
+
+  2. Put a properly aligned partition in the new disk label ::
+
+     nix-shell -p parted --run 'parted /dev/nvme1n1 mkpart primary ext2 4096s 4G'
+
+  3. Create a labeled filesystem on the partition ::
+
+     mkfs.ext4 -L zkapissuer-data /dev/nvme1n1p1
+
+  4. Deploy the PrivateStorageio update.
+
+  5. Move the database file to the new location ::
+
+     mv -iv /var/lib/zkapissuer/vouchers.sqlite3 /var/lib/zkapissuer-v2
+
+  6. Clean up the old state directory ::
+
+     rm -ir /var/lib/zkapissuer
+
+  7. Start the PaymentServer service (not running because its path assertions were not met earlier) ::
+
+     systemctl start zkapissuer
+
 - 2021-10-12 The secret in ``private-keys/grafana-slack-url`` needs to be changed to remove the ``SLACKURL=`` prefix.
 
 - 2021-09-30 `Enable alerting <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/185>`_ needs a secret in ``private-keys/grafana-slack-url`` looking like the template in ``morph/grid/local/private-keys/grafana-slack-url`` and pointing to the secret API endpoint URL saved in `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ (or create a new secret URL at https://www.slack.com/apps/A0F7XDUAZ).
@@ -15,4 +49,3 @@ Deployment notes
 
    chmod 750 /var/lib/zkapissuer
    chmod 640 /var/lib/zkapissuer/vouchers.sqlite3
-

morph/lib/base.nix

@@ -20,8 +20,12 @@
     };
   };
 
+  # Any extra NixOS modules to load on all our servers.  Note that just
+  # because they're loaded doesn't *necessarily* mean they're turned on.
   imports = [
-    ../../nixos/modules/packages.nix
+    # This brings in various other modules that define options for different
+    # areas of the service.
+    ../../nixos/modules/default.nix
   ];
 
   config = {

morph/lib/hardware-vagrant.nix

@@ -31,7 +31,21 @@
       prefixLength = 24;
     }];
 
+    # The issuer configuration wants to read the location of its database
+    # directory from the filesystem configuration.  Since the Vagrant
+    # environment doesn't have separate volume-as-infrastructure management
+    # (maybe it could?  but why bother?) we do a bind-mount here so there is a
+    # configured value readable.  The database won't really have a dedicated
+    # volume but it will sort of appear as if it does.
+    services.private-storage-issuer.databaseFileSystem = {
+      device = "/var/lib/origin-zkapissuer-v2";
+      options = ["bind"];
+    };
+
+    # XXX This should be handled by the storage module like the zkap
+    # filesystem above is handled by the issuer module.
     fileSystems."/storage" = { fsType = "tmpfs"; };
+
     fileSystems."/" =
       { device = "/dev/sda1";
         fsType = "ext4";

morph/lib/issuer-aws.nix

@@ -18,6 +18,14 @@
   # <https://github.com/DBCDK/morph/issues/146>.
   networking.hostName = name;
 
+  # Mount a dedicated filesystem (ideally on a dedicated volume, but that's
+  # beyond control of this particular part of the system) for the
+  # PaymentServer voucher database.  This makes it easier to manage for
+  # tasks like backup/recovery and encryption.
+  services.private-storage-issuer.databaseFileSystem = {
+    label = "zkapissuer-data";
+  };
+
   # Clean up packages after a while
   nix.gc = {
     automatic = true;

morph/lib/issuer.nix

@@ -45,7 +45,6 @@ in {
   };
 
   imports = [
-    ../../nixos/modules/issuer.nix
     ../../nixos/modules/monitoring/vpn/client.nix
     ../../nixos/modules/monitoring/exporters/node.nix
   ];
@@ -56,6 +55,6 @@ in {
     ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination;
     stripeSecretKeyPath = config.deployment.secrets.stripe-secret-key.destination;
     database = "SQLite3";
-    databasePath = "/var/lib/zkapissuer/vouchers.sqlite3";
+    databasePath = "${config.fileSystems."zkapissuer-data".mountPoint}/vouchers.sqlite3";
   };
 }

nixos/modules/default.nix

+{
+  # Load modules that are sometimes universally useful and other times useful
+  # only for a specific service.  Where functionality is not universally
+  # useful, it needs to be enabled by a node's configuration.  By loading more
+  # modules (and therefore defining more options) than is strictly necessary
+  # for any single node the logic for supplying conditional configuration
+  # elsewhere is much simplified.  For example, a Vagrant module can
+  # unconditionally set up a filesystem for PaymentServer.  If PaymentServer
+  # is running on that node then it will get a Vagrant-appropriate
+  # configuration.  If PaymentServer hasn't been enabled then the
+  # configuration will just be ignored.
+  imports = [
+    ./packages.nix
+    ./issuer.nix
+  ];
+}

nixos/modules/issuer.nix

@@ -81,6 +81,15 @@ in {
         The kind of voucher database to use.
       '';
     };
+    services.private-storage-issuer.databaseFileSystem = lib.mkOption {
+      # Logically, the type is the type of an entry in fileSystems - but we'll
+      # just let the type system enforce that when we pass the value on to
+      # fileSystems.
+      description = ''
+        Configuration for a filesystem to mount which will hold the issuer's
+        internal state database.
+      '';
+    };
     services.private-storage-issuer.databasePath = lib.mkOption {
       default = null;
       type = lib.types.str;
@@ -114,7 +123,23 @@ in {
       # Payment server internal http port (arbitrary, non-priviledged):
       internalHttpPort = "1061";
 
+      # The "-vN" suffix indicates that this Nth incompatible version of on
+      # disk state as managed by this deployment system.  This does not have
+      # anything to do with what's inside the PaymentServer-managed state.
+      # Instead it's about things like the type of filesystem used or options
+      # having to do with the backing volume behind the filesystem.  In
+      # general I expect that to get from "-vN" to "-v(N+1)" some manual
+      # upgrade steps will be required.
+      stateDirectory = "zkapissuer-v2";
+
     in lib.mkIf cfg.enable {
+    # Make sure the voucher database filesystem is mounted.
+    fileSystems = {
+      "zkapissuer-data" = cfg.databaseFileSystem // {
+        mountPoint = "/var/lib/${stateDirectory}";
+      };
+    };
+
     # Add a systemd service to run PaymentServer.
     systemd.services.zkapissuer = {
       enable = true;
@@ -137,15 +162,30 @@ in {
       # Make systemd create a User/Group owned directory for PaymentServer
       # state. According to the docs at
       # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory=
-      # "The specified directory names must be relative" ... this
-      # makes systemd create /var/lib/zkapissuer/ for us:
-      serviceConfig.StateDirectory = "zkapissuer";
+      # "The specified directory names must be relative" ... this makes
+      # systemd create this directory in /var/lib/ for us.
+      serviceConfig.StateDirectory = stateDirectory;
       serviceConfig.StateDirectoryMode = "0750";
 
-      # Bail if there is still an old (root-owned) DB file on this system.
-      # If you hit this, and this /var/db/ file is indeed current, move it to
-      # /var/lib/zkapissuer/vouchers.sqlite3 and chown it to zkapissuer:zkapissuer.
-      unitConfig.AssertPathExists = "!/var/db/vouchers.sqlite3";
+      unitConfig.AssertPathExists = [
+        # Bail if there is still an old (root-owned) DB file on this system.
+        # If you hit this, and this /var/db/ file is indeed current, move it
+        # to /var/lib/zkapissuer/vouchers.sqlite3 and chown it to
+        # zkapissuer:zkapissuer.
+        "!/var/db/vouchers.sqlite3"
+
+        # Similarly, bail if the newer path you were just told to create --
+        # /var/lib/zkapissuer/vouchers.sqlite3 -- exists.  It needs to be
+        # moved /var/lib/zkapissuer-v2 where a dedicated filesystem has been
+        # created for it.
+        "!/var/lib/zkapissuer/vouchers.sqlite3"
+      ];
+
+      # Only start if the dedicated vouchers database filesystem is mounted so
+      # that we know we're going to find our vouchers database there (or that
+      # we will create it in the right place).
+      unitConfig.Requires = ["local-fs.target"];
+      unitConfig.After = ["local-fs.target"];
 
       script =
         let