diff --git a/DEPLOYMENT-NOTES.rst b/DEPLOYMENT-NOTES.rst
index 5f1b15e933e9c2e98c364bf5969350a494987f61..fbde887b5ecfc5f1fa5b520df71a6a0ecc1980fe 100644
--- a/DEPLOYMENT-NOTES.rst
+++ b/DEPLOYMENT-NOTES.rst
@@ -1,6 +1,40 @@
Deployment notes
================
+- 2021-12-20
+
+ `https://whetstone.privatestorage.io/privatestorage/privatestorageops/-/issues/399`_ requires moving the PaymentServer database on the ``payments`` host onto a new dedicated filesystem.
+
+ Follow these steps *before* deploying this version of PrivateStorageio:
+
+ 0. Deploy the `PrivateStorageOps change <https://whetstone.privatestorage.io/privatestorage/privatestorageops/-/merge_requests/169>`_ that creates a new dedicated volume.
+
+ 1. Put a disk label on the new dedicated volume ::
+
+ nix-shell -p parted --run 'parted /dev/nvme1n1 mklabel msdos'
+
+ 2. Put a properly aligned partition in the new disk label ::
+
+ nix-shell -p parted --run 'parted /dev/nvme1n1 mkpart primary ext2 4096s 4G'
+
+ 3. Create a labeled filesystem on the partition ::
+
+ mkfs.ext4 -L zkapissuer-data /dev/nvme1n1p1
+
+ 4. Deploy the PrivateStorageio update.
+
+ 5. Move the database file to the new location ::
+
+ mv -iv /var/lib/zkapissuer/vouchers.sqlite3 /var/lib/zkapissuer-v2
+
+ 6. Clean up the old state directory ::
+
+ rm -ir /var/lib/zkapissuer
+
+ 7. Start the PaymentServer service (not running because its path assertions were not met earlier) ::
+
+ systemctl start zkapissuer
+
- 2021-10-12 The secret in ``private-keys/grafana-slack-url`` needs to be changed to remove the ``SLACKURL=`` prefix.
- 2021-09-30 `Enable alerting <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/185>`_ needs a secret in ``private-keys/grafana-slack-url`` looking like the template in ``morph/grid/local/private-keys/grafana-slack-url`` and pointing to the secret API endpoint URL saved in `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ (or create a new secret URL at https://www.slack.com/apps/A0F7XDUAZ).
@@ -15,4 +49,3 @@ Deployment notes
chmod 750 /var/lib/zkapissuer
chmod 640 /var/lib/zkapissuer/vouchers.sqlite3
-
diff --git a/morph/lib/base.nix b/morph/lib/base.nix
index f6a9a5f008681df23487f0f065ec4b9b8f982067..7390654ac167909149b0a6f4dfae897b8f3f43a3 100644
--- a/morph/lib/base.nix
+++ b/morph/lib/base.nix
@@ -20,8 +20,12 @@
};
};
+ # Any extra NixOS modules to load on all our servers. Note that just
+ # because they're loaded doesn't *necessarily* mean they're turned on.
imports = [
- ../../nixos/modules/packages.nix
+ # This brings in various other modules that define options for different
+ # areas of the service.
+ ../../nixos/modules/default.nix
];
config = {
diff --git a/morph/lib/hardware-vagrant.nix b/morph/lib/hardware-vagrant.nix
index 150944cd5b64b7a3eb620cd40ea39e00544779a7..6c41af4923861e89d144303d129d7babde494363 100644
--- a/morph/lib/hardware-vagrant.nix
+++ b/morph/lib/hardware-vagrant.nix
@@ -31,7 +31,21 @@
prefixLength = 24;
}];
+ # The issuer configuration wants to read the location of its database
+ # directory from the filesystem configuration. Since the Vagrant
+ # environment doesn't have separate volume-as-infrastructure management
+ # (maybe it could? but why bother?) we do a bind-mount here so there is a
+ # configured value readable. The database won't really have a dedicated
+ # volume but it will sort of appear as if it does.
+ services.private-storage-issuer.databaseFileSystem = {
+ device = "/var/lib/origin-zkapissuer-v2";
+ options = ["bind"];
+ };
+
+ # XXX This should be handled by the storage module like the zkap
+ # filesystem above is handled by the issuer module.
fileSystems."/storage" = { fsType = "tmpfs"; };
+
fileSystems."/" =
{ device = "/dev/sda1";
fsType = "ext4";
diff --git a/morph/lib/issuer-aws.nix b/morph/lib/issuer-aws.nix
index bf7de56cfb570857da32c34ebfcc9b21c91e702e..8ff172803eda784898aba2d96636df1afcee36e5 100644
--- a/morph/lib/issuer-aws.nix
+++ b/morph/lib/issuer-aws.nix
@@ -18,6 +18,14 @@
# <https://github.com/DBCDK/morph/issues/146>.
networking.hostName = name;
+ # Mount a dedicated filesystem (ideally on a dedicated volume, but that's
+ # beyond control of this particular part of the system) for the
+ # PaymentServer voucher database. This makes it easier to manage for
+ # tasks like backup/recovery and encryption.
+ services.private-storage-issuer.databaseFileSystem = {
+ label = "zkapissuer-data";
+ };
+
# Clean up packages after a while
nix.gc = {
automatic = true;
diff --git a/morph/lib/issuer.nix b/morph/lib/issuer.nix
index d60af799888c97ec8f97a061d40b54d3f2db82a7..d3ee812e865f741b01eb811589262ae01ece824f 100644
--- a/morph/lib/issuer.nix
+++ b/morph/lib/issuer.nix
@@ -45,7 +45,6 @@ in {
};
imports = [
- ../../nixos/modules/issuer.nix
../../nixos/modules/monitoring/vpn/client.nix
../../nixos/modules/monitoring/exporters/node.nix
];
@@ -56,6 +55,6 @@ in {
ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination;
stripeSecretKeyPath = config.deployment.secrets.stripe-secret-key.destination;
database = "SQLite3";
- databasePath = "/var/lib/zkapissuer/vouchers.sqlite3";
+ databasePath = "${config.fileSystems."zkapissuer-data".mountPoint}/vouchers.sqlite3";
};
}
diff --git a/nixos/modules/default.nix b/nixos/modules/default.nix
new file mode 100644
index 0000000000000000000000000000000000000000..1772d399639aa8b4ec2c9ac6a218c4dd8a6169da
--- /dev/null
+++ b/nixos/modules/default.nix
@@ -0,0 +1,16 @@
+{
+ # Load modules that are sometimes universally useful and other times useful
+ # only for a specific service. Where functionality is not universally
+ # useful, it needs to be enabled by a node's configuration. By loading more
+ # modules (and therefore defining more options) than is strictly necessary
+ # for any single node the logic for supplying conditional configuration
+ # elsewhere is much simplified. For example, a Vagrant module can
+ # unconditionally set up a filesystem for PaymentServer. If PaymentServer
+ # is running on that node then it will get a Vagrant-appropriate
+ # configuration. If PaymentServer hasn't been enabled then the
+ # configuration will just be ignored.
+ imports = [
+ ./packages.nix
+ ./issuer.nix
+ ];
+}
diff --git a/nixos/modules/issuer.nix b/nixos/modules/issuer.nix
index e0d1f6560e48748854ab5bcd4613b50c2f24cbcd..67bc3c5029c290676a777115179b273b2e8851ef 100644
--- a/nixos/modules/issuer.nix
+++ b/nixos/modules/issuer.nix
@@ -81,6 +81,15 @@ in {
The kind of voucher database to use.
'';
};
+ services.private-storage-issuer.databaseFileSystem = lib.mkOption {
+ # Logically, the type is the type of an entry in fileSystems - but we'll
+ # just let the type system enforce that when we pass the value on to
+ # fileSystems.
+ description = ''
+ Configuration for a filesystem to mount which will hold the issuer's
+ internal state database.
+ '';
+ };
services.private-storage-issuer.databasePath = lib.mkOption {
default = null;
type = lib.types.str;
@@ -114,7 +123,23 @@ in {
# Payment server internal http port (arbitrary, non-priviledged):
internalHttpPort = "1061";
+ # The "-vN" suffix indicates that this Nth incompatible version of on
+ # disk state as managed by this deployment system. This does not have
+ # anything to do with what's inside the PaymentServer-managed state.
+ # Instead it's about things like the type of filesystem used or options
+ # having to do with the backing volume behind the filesystem. In
+ # general I expect that to get from "-vN" to "-v(N+1)" some manual
+ # upgrade steps will be required.
+ stateDirectory = "zkapissuer-v2";
+
in lib.mkIf cfg.enable {
+ # Make sure the voucher database filesystem is mounted.
+ fileSystems = {
+ "zkapissuer-data" = cfg.databaseFileSystem // {
+ mountPoint = "/var/lib/${stateDirectory}";
+ };
+ };
+
# Add a systemd service to run PaymentServer.
systemd.services.zkapissuer = {
enable = true;
@@ -137,15 +162,30 @@ in {
# Make systemd create a User/Group owned directory for PaymentServer
# state. According to the docs at
# https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory=
- # "The specified directory names must be relative" ... this
- # makes systemd create /var/lib/zkapissuer/ for us:
- serviceConfig.StateDirectory = "zkapissuer";
+ # "The specified directory names must be relative" ... this makes
+ # systemd create this directory in /var/lib/ for us.
+ serviceConfig.StateDirectory = stateDirectory;
serviceConfig.StateDirectoryMode = "0750";
- # Bail if there is still an old (root-owned) DB file on this system.
- # If you hit this, and this /var/db/ file is indeed current, move it to
- # /var/lib/zkapissuer/vouchers.sqlite3 and chown it to zkapissuer:zkapissuer.
- unitConfig.AssertPathExists = "!/var/db/vouchers.sqlite3";
+ unitConfig.AssertPathExists = [
+ # Bail if there is still an old (root-owned) DB file on this system.
+ # If you hit this, and this /var/db/ file is indeed current, move it
+ # to /var/lib/zkapissuer/vouchers.sqlite3 and chown it to
+ # zkapissuer:zkapissuer.
+ "!/var/db/vouchers.sqlite3"
+
+ # Similarly, bail if the newer path you were just told to create --
+ # /var/lib/zkapissuer/vouchers.sqlite3 -- exists. It needs to be
+ # moved /var/lib/zkapissuer-v2 where a dedicated filesystem has been
+ # created for it.
+ "!/var/lib/zkapissuer/vouchers.sqlite3"
+ ];
+
+ # Only start if the dedicated vouchers database filesystem is mounted so
+ # that we know we're going to find our vouchers database there (or that
+ # we will create it in the right place).
+ unitConfig.Requires = ["local-fs.target"];
+ unitConfig.After = ["local-fs.target"];
script =
let