Merge pull request #43 from PrivateStorageio/40.tahoe-lafs-preStart

Improved Tahoe-LAFS systemd unit ExecStartPre

nixos/modules/private-storage.nix

@@ -21,14 +21,6 @@ let
   max-incident-age = "29d";
 in
 {
-  # Upstream tahoe-lafs module conflicts with ours (since ours is a
-  # copy/paste/edit of upstream's...).  Disable it.
-  #
-  # https://nixos.org/nixos/manual/#sec-replace-modules
-  disabledModules =
-  [ "services/network-filesystems/tahoe.nix"
-  ];
-
   imports = [
     # Give it a good SSH configuration.
     ./ssh.nix

nixos/modules/tahoe.nix

@@ -9,6 +9,14 @@ let
   ini = pkgs.callPackage ../lib/ini.nix { };
 in
   {
+    # Upstream tahoe-lafs module conflicts with ours (since ours is a
+    # copy/paste/edit of upstream's...).  Disable it.
+    #
+    # https://nixos.org/nixos/manual/#sec-replace-modules
+    disabledModules =
+    [ "services/network-filesystems/tahoe.nix"
+    ];
+
     options.services.tahoe = {
       introducers = mkOption {
         default = {};
@@ -233,22 +241,36 @@ in
               created = "${nodedir}.created";
               atomic = "${nodedir}.atomic";
             in ''
+              set -eo pipefail
               if [ ! -e ${created} ]; then
                 mkdir -p /var/db/tahoe-lafs/
 
                 # Get rid of any prior partial efforts.  It might not exist.
                 # Don't let this tank us.
-                rm -rv ${atomic} && [ ! -e ${atomic} ]
+                rm -rv ${atomic} || [ ! -e ${atomic} ]
 
                 # Really create the node.
                 tahoe create-node --hostname=localhost ${atomic}
 
-                # Move it to the real location.  We don't create it in-place
-                # because we might fail partway through and leave inconsistent
-                # state.  Also, systemd probably created logs/incidents/ already and
-                # `create-node` complains if it finds these exist already.
-                rm -rv ${nodedir} && [ ! -e ${nodedir} ]
+                # Get rid of any existing partially created node directory
+                # that might be in the way.
+                if [ -e ${nodedir} ]; then
+                  for backup in $(seq 1 100); do
+                    if [ ! -e ${nodedir}.$backup ]; then
+                      mv ${nodedir} ${nodedir}.$backup
+                      break
+                    fi
+                  done
+                fi
+
+                # Move the new thing to the real location.  We don't create it
+                # in-place because we might fail partway through and leave
+                # inconsistent state.  Also, systemd probably created
+                # logs/incidents/ already and `create-node` complains if it
+                # finds these exist already.
                 mv ${atomic} ${nodedir}
+
+                # Record our complete, consistent success.
                 touch ${created}
               fi
 

nixos/modules/tests/exercise-storage.py

@@ -10,6 +10,8 @@ from sys import argv
 from os import urandom
 from subprocess import check_output
 from io import BytesIO
+from time import sleep, ctime
+from pprint import pprint
 
 import requests
 import hyperlink
@@ -21,10 +23,45 @@ def main():
 
     api_root = get_api_root(clientDir)
 
+    block_until_connected(api_root)
+
     subject_cap = exercise_immutable(api_root, someData)
     newDir = exercise_mkdir(api_root)
     exercise_link_unlink(api_root, newDir, subject_cap)
 
+def block_until_connected(api_root):
+    """
+    Block until the Tahoe-LAFS node at the given API root reports it has
+    connected to at least one storage server.
+    """
+    while True:
+        response = requests.get(
+            api_root.replace(query={u"t": u"json"}),
+        )
+        response.raise_for_status()
+        welcome = response.json()
+        servers = welcome["servers"]
+        connected = list(
+            server
+            for server
+            in servers
+            if server["connection_status"].startswith("Connected to ")
+        )
+        if len(connected) >= 1:
+            print(
+                "Connected to a server:\n"
+                "\t{nodeid}\n"
+                "\t{status}\n"
+                "\t{last_received_data}\n".format(
+                    nodeid=connected[0]["nodeid"],
+                    status=connected[0]["connection_status"],
+                    last_received_data=ctime(connected[0]["last_received_data"]),
+                ),
+            )
+            return
+        pprint(welcome)
+        sleep(0.1)
+
 def exercise_immutable(api_root, someData):
     cap = tahoe_put(api_root, someData)
     dataReadBack = tahoe_get(api_root, cap)

nixos/modules/tests/private-storage.nix

+{ pkgs }:
 let
-  pkgs = import <nixpkgs> { };
   pspkgs = import ../pspkgs.nix { inherit pkgs; };
 
   sshPrivateKey = ./probeuser_ed25519;
@@ -89,10 +89,9 @@ let
       # succeed() is not success but 1 is.
       1;
       ";
-in
-# https://nixos.org/nixos/manual/index.html#sec-nixos-tests
-import <nixpkgs/nixos/tests/make-test.nix> {
-
+in {
+  # https://nixos.org/nixos/manual/index.html#sec-nixos-tests
+  # https://nixos.mayflower.consulting/blog/2019/07/11/leveraging-nixos-tests-in-your-project/
   nodes = rec {
     # Get a machine where we can run a Tahoe-LAFS client node.
     client =
@@ -202,7 +201,6 @@ import <nixpkgs/nixos/tests/make-test.nix> {
       eval {
       ${runOnNode "introducer" [ run-introducer "/tmp/node.pem" (toString introducerPort) introducerFURL ]}
       } or do {
-        my $error = $@ || 'Unknown failure';
         my ($code, $log) = $introducer->execute('cat /tmp/stdout /tmp/stderr');
         $introducer->log($log);
         die $@;
@@ -246,7 +244,6 @@ import <nixpkgs/nixos/tests/make-test.nix> {
       eval {
         ${runOnNode "client" [ get-passes "http://127.0.0.1:3456" issuerURL voucher ]}
       } or do {
-        my $error = $@ || 'Unknown failure';
         my ($code, $log) = $client->execute('cat /tmp/stdout /tmp/stderr');
         $client->log($log);
 
@@ -261,10 +258,34 @@ import <nixpkgs/nixos/tests/make-test.nix> {
       eval {
         ${runOnNode "client" [ exercise-storage "/tmp/client" ]}
       } or do {
-        my $error = $@ || 'Unknown failure';
         my ($code, $log) = $client->execute('cat /tmp/stdout /tmp/stderr');
         $client->log($log);
         die $@;
       };
-      '';
-}
+
+      # It should be possible to restart the storage service without the
+      # storage node fURL changing.
+      eval {
+        my $furlfile = '/var/db/tahoe-lafs/storage/private/storage-plugin.privatestorageio-zkapauthz-v1.furl';
+        my $before = $storage->execute('cat ' . $furlfile);
+        ${runOnNode "storage" [ "systemctl" "restart" "tahoe.storage" ]}
+        my $after = $storage->execute('cat ' . $furlfile);
+        if ($before != $after) {
+          die 'fURL changes after storage node restart';
+        }
+        1;
+      } or do {
+        my ($code, $log) = $storage->execute('cat /tmp/stdout /tmp/stderr');
+        $storage->log($log);
+        die $@;
+      };
+
+      # The client should actually still work, too.
+      eval {
+        ${runOnNode "client" [ exercise-storage "/tmp/client" ]}
+      } or do {
+        my ($code, $log) = $client->execute('cat /tmp/stdout /tmp/stderr');
+        $client->log($log);
+        die $@;
+      };
+      ''; }

nixos/modules/tests/tahoe.nix

+{ ... }: {
+  nodes = {
+    storage = { config, pkgs, ... }: {
+      imports = [
+        ../tahoe.nix
+      ];
+
+      services.tahoe.nodes.storage = {
+        package = (pkgs.callPackage ../pspkgs.nix { }).privatestorage;
+        sections = {
+          node = {
+            nickname = "storage";
+            "web.port" = "tcp:4000:interface=127.0.0.1";
+            "tub.port" = "tcp:4001";
+            "tub.location" = "tcp:127.0.0.1:4001";
+          };
+          storage = {
+            enabled = true;
+          };
+        };
+      };
+    };
+  };
+  testScript = ''
+  startAll;
+
+  # After the service starts, destroy the "created" marker to force it to
+  # re-create its internal state.
+  $storage->waitForOpenPort(4001);
+  $storage->succeed("systemctl stop tahoe.storage");
+  $storage->succeed("rm /var/db/tahoe-lafs/storage.created");
+  $storage->succeed("systemctl start tahoe.storage");
+
+  # After it starts up again, verify it has consistent internal state and a
+  # backup of the prior state.
+  $storage->waitForOpenPort(4001);
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.created ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.1 ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.privkey ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.pem ]");
+  $storage->succeed("[ ! -e /var/db/tahoe-lafs/storage.2 ]");
+
+  # Stop it again, once again destroy the "created" marker, and this time also
+  # jam some partial state in the way that will need cleanup.
+  $storage->succeed("systemctl stop tahoe.storage");
+  $storage->succeed("rm /var/db/tahoe-lafs/storage.created");
+  $storage->succeed("mkdir -p /var/db/tahoe-lafs/storage.atomic/partial");
+  eval {
+    $storage->succeed("systemctl start tahoe.storage");
+    1;
+  } or do {
+    my ($x, $y) = $storage->execute("journalctl -u tahoe.storage");
+    $storage->log($y);
+    die $@;
+  };
+
+  # After it starts up again, verify it has consistent internal state and
+  # backups of the prior two states.  It also has no copy of the inconsistent
+  # state because it could never have been used.
+  $storage->waitForOpenPort(4001);
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.created ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.1 ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.2 ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.privkey ]");
+  $storage->succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.pem ]");
+  $storage->succeed("[ ! -e /var/db/tahoe-lafs/storage.atomic ]");
+  $storage->succeed("[ ! -e /var/db/tahoe-lafs/storage/partial ]");
+  $storage->succeed("[ ! -e /var/db/tahoe-lafs/storage.3 ]");
+  '';
+}

nixos/system-tests.nix

 # The overall system test suite for PrivateStorageio NixOS configuration.
-
-# There is only one system test so far so I don't have to do anything to
-# aggregate multiple tests...
-import ./modules/tests/private-storage.nix
+let
+  pkgs = import <nixpkgs> { };
+in {
+  private-storage = pkgs.nixosTest ./modules/tests/private-storage.nix;
+  tahoe = pkgs.nixosTest ./modules/tests/tahoe.nix;
+}