Tests for and improvements to the Tahoe-LAFS systemd definition

This removes the code that might delete a Tahoe-LAFS nodedir and replaces it
with code that will try to save an unexpected copy of it out of the way
instead.

nixos/modules/tahoe.nix

@@ -241,22 +241,36 @@ in
               created = "${nodedir}.created";
               atomic = "${nodedir}.atomic";
             in ''
+              set -eo pipefail
               if [ ! -e ${created} ]; then
                 mkdir -p /var/db/tahoe-lafs/
 
                 # Get rid of any prior partial efforts.  It might not exist.
                 # Don't let this tank us.
-                rm -rv ${atomic} && [ ! -e ${atomic} ]
+                rm -rv ${atomic} || [ ! -e ${atomic} ]
 
                 # Really create the node.
                 tahoe create-node --hostname=localhost ${atomic}
 
-                # Move it to the real location.  We don't create it in-place
-                # because we might fail partway through and leave inconsistent
-                # state.  Also, systemd probably created logs/incidents/ already and
-                # `create-node` complains if it finds these exist already.
-                rm -rv ${nodedir} && [ ! -e ${nodedir} ]
+                # Get rid of any existing partially created node directory
+                # that might be in the way.
+                if [ -e ${nodedir} ]; then
+                  for backup in $(seq 1 100); do
+                    if [ ! -e ${nodedir}.$backup ]; then
+                      mv ${nodedir} ${nodedir}.$backup
+                      break
+                    fi
+                  done
+                fi
+
+                # Move the new thing to the real location.  We don't create it
+                # in-place because we might fail partway through and leave
+                # inconsistent state.  Also, systemd probably created
+                # logs/incidents/ already and `create-node` complains if it
+                # finds these exist already.
                 mv ${atomic} ${nodedir}
+
+                # Record our complete, consistent success.
                 touch ${created}
               fi
 

nixos/modules/tests/private-storage.nix

+{ pkgs }:
 let
-  pkgs = import <nixpkgs> { };
   pspkgs = import ../pspkgs.nix { inherit pkgs; };
 
   sshPrivateKey = ./probeuser_ed25519;
@@ -89,10 +89,9 @@ let
       # succeed() is not success but 1 is.
       1;
       ";
-in
+in {
   # https://nixos.org/nixos/manual/index.html#sec-nixos-tests
-import <nixpkgs/nixos/tests/make-test.nix> {
-
+  # https://nixos.mayflower.consulting/blog/2019/07/11/leveraging-nixos-tests-in-your-project/
   nodes = rec {
     # Get a machine where we can run a Tahoe-LAFS client node.
     client =

nixos/modules/tests/tahoe.nix

+{ ... }: {
+  nodes = {
+    storage = { config, pkgs, ... }: {
+      imports = [
+        ../tahoe.nix
+      ];
+
+      services.tahoe.nodes.storage = {
+        package = (pkgs.callPackage ../pspkgs.nix { }).privatestorage;
+        sections = {
+          node = {
+            nickname = "storage";
+            "web.port" = "tcp:4000:interface=127.0.0.1";
+            "tub.port" = "tcp:4001";
+            "tub.location" = "tcp:127.0.0.1:4001";
+          };
+          storage = {
+            enabled = true;
+          };
+        };
+      };
+    };
+  };
+  testScript = ''
+  startAll;
+
+  # After the service starts, destroy the "created" marker to force it to
+  # re-create its internal state.
+  $storage->waitForOpenPort(4001);
+  $storage->succeed("systemctl stop tahoe.storage");
+  $storage->succeed("rm /var/db/tahoe-lafs/storage.created");
+  $storage->succeed("systemctl start tahoe.storage");
+
+  # After it starts up again, verify it has consistent internal state and a
+  # backup of the prior state.
+  $storage->waitForOpenPort(4001);
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.created ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.1 ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.privkey ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.pem ]");
+  $storage->succeed("[ ! -e /var/db/tahoe-lafs/storage.2 ]");
+
+  # Stop it again, once again destroy the "created" marker, and this time also
+  # jam some partial state in the way that will need cleanup.
+  $storage->succeed("systemctl stop tahoe.storage");
+  $storage->succeed("rm /var/db/tahoe-lafs/storage.created");
+  $storage->succeed("mkdir -p /var/db/tahoe-lafs/storage.atomic/partial");
+  eval {
+    $storage->succeed("systemctl start tahoe.storage");
+    1;
+  } or do {
+    my ($x, $y) = $storage->execute("journalctl -u tahoe.storage");
+    $storage->log($y);
+    die $@;
+  };
+
+  # After it starts up again, verify it has consistent internal state and
+  # backups of the prior two states.  It also has no copy of the inconsistent
+  # state because it could never have been used.
+  $storage->waitForOpenPort(4001);
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.created ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.1 ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.2 ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.privkey ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.pem ]");
+  $storage->succeed("[ ! -e /var/db/tahoe-lafs/storage.atomic ]");
+  $storage->succeed("[ ! -e /var/db/tahoe-lafs/storage/partial ]");
+  $storage->succeed("[ ! -e /var/db/tahoe-lafs/storage.3 ]");
+  '';
+}

nixos/system-tests.nix

 # The overall system test suite for PrivateStorageio NixOS configuration.
-
-# There is only one system test so far so I don't have to do anything to
-# aggregate multiple tests...
-import ./modules/tests/private-storage.nix
+let
+  pkgs = import <nixpkgs> { };
+in {
+  private-storage = pkgs.nixosTest ./modules/tests/private-storage.nix;
+  tahoe = pkgs.nixosTest ./modules/tests/tahoe.nix;
+}