diff --git a/morph/README.rst b/morph/README.rst
index 7f8bb655b36010ae2bcb152632b39387de0a412e..d4a89a373a1bef767ad26859d495f1528a4fb7ca 100644
--- a/morph/README.rst
+++ b/morph/README.rst
@@ -3,10 +3,35 @@ Morph
This directory contains Nix-based configuration for the grid.
This takes the form of Nix expressions in ``.nix`` files
-and some JSON-based configuration in ``.config.json`` files.
+and some JSON-based configuration in ``.json`` files.
This configuration is fed to `morph`_ to make changes to the deployment.
+Deploying
+`````````
+
+The deployment consists of the public software packages and the private secrets.
+You can deploy these together::
+
+ morph deploy --upload-secrets morph/grid/<testing|production|...>/grid.nix test
+
+Or separately::
+
+ morph deploy morph/grid/<testing|production|...>/grid.nix test
+ morph upload-secrets morph/grid/<testing|production|...>/grid.nix
+
+Separate deployment is useful when the software deploy is done from system which may not be sufficiently secure to host the secrets
+(such as a cloud build machine).
+Secrets should only be hosted on an extremely secure system
+(XXX write the document for what this means).
+
+Note secrets only need to be uploaded after a host in the grid has been rebooted or when the secrets have changed.
+
+See the ``morph`` and ``nixos-rebuild`` documentation for more details about these commands.
+
+Filesystem Layout
+`````````````````
+
lib
---
@@ -49,7 +74,7 @@ Each such file contains a minimal Nix expression supplying critical system confi
These files are referenced by the corresponding ``<hostname>.nix`` files.
Configuring New Storage Nodes
------------------------------
+`````````````````````````````
Storage nodes are brought into the grid in a multi-step process.
Here are the steps to configure a new node,
diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix
index a65b77bd9a873c659ee9bb9f6f16aac4c2d45d6d..7b056146c300952f014da6fc57e71ee022ac58a5 100644
--- a/morph/grid/production/grid.nix
+++ b/morph/grid/production/grid.nix
@@ -4,7 +4,10 @@
import ../../lib/make-grid.nix {
name = "Production";
config = ./config.json;
- nodes = cfg: {
+ nodes = cfg:
+ let
+ sshUsers = import ../../../../PrivateStorageSecrets/production-users.nix;
+ in {
# Here are the hosts that are in this morph network. This is sort of like
# a server manifest. We try to keep as many of the specific details as
# possible out of *this* file so that this file only grows as server count
@@ -18,32 +21,38 @@ import ../../lib/make-grid.nix {
#
# The names must be unique!
"payments.privatestorage.io" = import ../../lib/issuer.nix ({
+ inherit sshUsers;
hardware = ../../lib/issuer-aws.nix;
stateVersion = "19.03";
} // cfg);
"storage001" = import ../../lib/make-storage.nix ({
cfg = import ./storage001-config.nix;
+ inherit sshUsers;
hardware = ./storage001-hardware.nix;
stateVersion = "19.09";
} // cfg);
"storage002" = import ../../lib/make-storage.nix ({
cfg = import ./storage002-config.nix;
+ inherit sshUsers;
hardware = ./storage002-hardware.nix;
stateVersion = "19.09";
} // cfg);
"storage003" = import ../../lib/make-storage.nix ({
cfg = import ./storage003-config.nix;
+ inherit sshUsers;
hardware = ./storage003-hardware.nix;
stateVersion = "19.09";
} // cfg);
"storage004" = import ../../lib/make-storage.nix ({
cfg = import ./storage004-config.nix;
+ inherit sshUsers;
hardware = ./storage004-hardware.nix;
stateVersion = "19.09";
} // cfg);
"storage005" = import ../../lib/make-storage.nix ({
cfg = import ./storage005-config.nix;
+ inherit sshUsers;
hardware = ./storage005-hardware.nix;
stateVersion = "19.03";
} // cfg);
diff --git a/morph/grid/production/storage000-config.nix b/morph/grid/production/storage000-config.nix
index b1d38ecb1f896b740f0a392df14da670d0156a48..2a056a5489245e7e334a26fd9b784097512f6196 100644
--- a/morph/grid/production/storage000-config.nix
+++ b/morph/grid/production/storage000-config.nix
@@ -4,5 +4,4 @@
"gateway" = "69.36.183.1";
"gatewayInterface" = "eno1";
"grubDeviceID" = "wwn-0x5000c500936410b9";
- "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
}
diff --git a/morph/grid/production/storage001-config.nix b/morph/grid/production/storage001-config.nix
index 2e6e21548c978f8d41e5856af03caa64124864de..63b0e876c32995e10e24e7afa5b9aa9c41025b51 100644
--- a/morph/grid/production/storage001-config.nix
+++ b/morph/grid/production/storage001-config.nix
@@ -5,5 +5,4 @@
"gateway" = "176.113.72.37";
"gatewayInterface" = "eno1";
"grubDeviceID" = "wwn-0x5000cca25cc08d27";
- "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
}
diff --git a/morph/grid/production/storage002-config.nix b/morph/grid/production/storage002-config.nix
index c9ddf1779d13d5fe67f03088007fcb11085b8bd7..d74a99ec1824d798564cd4c469684af1dcf989d4 100644
--- a/morph/grid/production/storage002-config.nix
+++ b/morph/grid/production/storage002-config.nix
@@ -5,5 +5,4 @@
"gateway" = "37.120.214.109";
"gatewayInterface" = "eno1";
"grubDeviceID" = "wwn-0x5000cca25dccb3dc";
- "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
}
diff --git a/morph/grid/production/storage003-config.nix b/morph/grid/production/storage003-config.nix
index 2feb5eec1460913d1c4ed049cee4c52c50bbbb4c..e83546adbcdab2fd35d990a13550dd3907d7226b 100644
--- a/morph/grid/production/storage003-config.nix
+++ b/morph/grid/production/storage003-config.nix
@@ -5,5 +5,4 @@
"gateway" = "45.83.89.185";
"gatewayInterface" = "eno1";
"grubDeviceID" = "wwn-0x5000cca248c31469";
- "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
}
diff --git a/morph/grid/production/storage004-config.nix b/morph/grid/production/storage004-config.nix
index 8143e30e39140c1fe3f4b5b76560562488b1c31f..8201391c433281092044a284fb5c15a9933929cb 100644
--- a/morph/grid/production/storage004-config.nix
+++ b/morph/grid/production/storage004-config.nix
@@ -5,5 +5,4 @@
"gateway" = "87.101.93.197";
"gatewayInterface" = "eno1";
"grubDeviceID" = "wwn-0x5000cca249d45533";
- "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
}
diff --git a/morph/grid/production/storage005-config.nix b/morph/grid/production/storage005-config.nix
index 75eeece531b22fc84370a82e598128f672470616..42ac495af0064b52acdd0cbd287d84aba1f0bd0c 100644
--- a/morph/grid/production/storage005-config.nix
+++ b/morph/grid/production/storage005-config.nix
@@ -5,5 +5,4 @@
"gateway" = "193.148.18.205";
"gatewayInterface" = "eno1";
"grubDeviceID" = "wwn-0x5000cca25dcc78b5";
- "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
}
diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix
index 4db91363105ab9f35d21ccb037427cbfb7226003..3a1c5f3921c196843b0a4cd1b18f20388a75edde 100644
--- a/morph/grid/testing/grid.nix
+++ b/morph/grid/testing/grid.nix
@@ -4,14 +4,19 @@
import ../../lib/make-grid.nix {
name = "Testing";
config = ./config.json;
- nodes = cfg: {
+ nodes = cfg:
+ let
+ sshUsers = import ../../../../PrivateStorageSecrets/staging-users.nix;
+ in {
"payments.privatestorage-staging.com" = import ../../lib/issuer.nix ({
+ inherit sshUsers;
hardware = ../../lib/issuer-aws.nix;
stateVersion = "19.03";
} // cfg);
"3.120.26.190" = import ../../lib/make-testing.nix (cfg // {
publicIPv4 = "3.120.26.190";
+ inherit sshUsers;
hardware = ./testing001-hardware.nix;
stateVersion = "19.03";
});
diff --git a/morph/lib/issuer.nix b/morph/lib/issuer.nix
index 6e1eca671b0774775cbecdeed55703c6e7b02ea1..33c87f70d81e0ad064977ab7204c5484f4d9e7e6 100644
--- a/morph/lib/issuer.nix
+++ b/morph/lib/issuer.nix
@@ -4,6 +4,7 @@
, issuerDomain
, letsEncryptAdminEmail
, allowedChargeOrigins
+, sshUsers
, stateVersion
, ...
}: {
@@ -33,6 +34,7 @@
../../nixos/modules/issuer.nix
];
+ services.private-storage.sshUsers = sshUsers;
services.private-storage-issuer = {
enable = true;
tls = true;
diff --git a/morph/lib/make-storage.nix b/morph/lib/make-storage.nix
index da206572566b1b997d4160ba46a98aa4fd547430..0c556befc096f0fb8fa4476e6bad234e3e9231a2 100644
--- a/morph/lib/make-storage.nix
+++ b/morph/lib/make-storage.nix
@@ -3,6 +3,7 @@
, hardware # The path to the hardware configuration for this node.
, publicStoragePort # The storage port number on which to accept connections.
, ristrettoSigningKeyPath # The *local* path to the Ristretto signing key file.
+, sshUsers # Users for which to configure SSH access to this node.
, stateVersion # The value for system.stateVersion on this node.
# This value determines the NixOS release with
# which your system is to be compatible, in order
@@ -56,6 +57,8 @@
inherit publicStoragePort;
# Give it the Ristretto signing key, too, to support authorization.
ristrettoSigningKeyPath = deployment.secrets.ristretto-signing-key.destination;
+ # It gets the users, too.
+ inherit sshUsers;
};
system.stateVersion = stateVersion;
diff --git a/morph/lib/make-testing.nix b/morph/lib/make-testing.nix
index 24de7ea40ad778d0ca2418be063a4a1528965839..df8eb006b3234d5e127a334a401c9079d5adc2fb 100644
--- a/morph/lib/make-testing.nix
+++ b/morph/lib/make-testing.nix
@@ -1,4 +1,4 @@
-{ publicIPv4, hardware, publicStoragePort, ristrettoSigningKeyPath, stateVersion, ... }: rec {
+{ publicIPv4, hardware, publicStoragePort, ristrettoSigningKeyPath, sshUsers, stateVersion, ... }: rec {
deployment = {
secrets = {
@@ -26,6 +26,7 @@
inherit publicIPv4;
inherit publicStoragePort;
ristrettoSigningKeyPath = deployment.secrets.ristretto-signing-key.destination;
+ inherit sshUsers;
};
system.stateVersion = stateVersion;
diff --git a/nixos/modules/100tb.nix b/nixos/modules/100tb.nix
index ec4bf66579d4d9b89065f02df3ee454f5b647294..1bcb6ba176714ca3cfa5c527450dd9a3bc60684d 100644
--- a/nixos/modules/100tb.nix
+++ b/nixos/modules/100tb.nix
@@ -69,11 +69,6 @@ let
example = lib.literalExample "wwn-0x5000c500936410b9";
description = "The ID of the disk on which to install grub.";
};
- rootPublicKey = lib.mkOption
- { type = lib.types.str;
- example = lib.literalExample "ssh-ed25519 AAAA... username@host";
- description = "The public key to install for the root user.";
- };
};
in {
# Here we actually define the module's options. They're what we said they
@@ -112,11 +107,6 @@ in {
boot.loader.timeout = 1;
networking.firewall.enable = false;
- services.openssh.enable = true;
-
- users.users.root.openssh.authorizedKeys.keys = [
- cfg.rootPublicKey
- ];
networking.hostId = cfg.hostId;
networking.dhcpcd.enable = false;
diff --git a/nixos/modules/issuer.nix b/nixos/modules/issuer.nix
index a65eed5a3539f2d5ba67870782635a1ae7209582..7654bf1fc7082afcdc2056a9373deea89bdc4f19 100644
--- a/nixos/modules/issuer.nix
+++ b/nixos/modules/issuer.nix
@@ -1,10 +1,15 @@
-# A NixOS module which can run a Ristretto-based issuer for PrivacyStorage
+# A NixOS module which can run a Ristretto-based issuer for PrivateStorage
# ZKAPs.
{ lib, pkgs, config, ... }: let
pspkgs = pkgs.callPackage ./pspkgs.nix { };
zkapissuer = pspkgs.callPackage ../pkgs/zkapissuer.nix { };
cfg = config.services.private-storage-issuer;
in {
+ imports = [
+ # Give it a good SSH configuration.
+ ../../nixos/modules/ssh.nix
+ ];
+
options = {
services.private-storage-issuer.enable = lib.mkEnableOption "PrivateStorage ZKAP Issuer Service";
services.private-storage-issuer.package = lib.mkOption {
diff --git a/nixos/modules/private-storage.nix b/nixos/modules/private-storage.nix
index 687c9e35d609601f85a57d2ac85c7aac7b66cbf9..cada491e04a49ce1e4931b58ffae6527f8cf77c5 100644
--- a/nixos/modules/private-storage.nix
+++ b/nixos/modules/private-storage.nix
@@ -29,10 +29,12 @@ in
[ "services/network-filesystems/tahoe.nix"
];
- # Load our tahoe-lafs module. It is configurable in the way I want it to be
- # configurable.
- imports =
- [ ./tahoe.nix
+ imports = [
+ # Give it a good SSH configuration.
+ ./ssh.nix
+ # Load our tahoe-lafs module. It is configurable in the way I want it to
+ # be configurable.
+ ./tahoe.nix
];
options =
diff --git a/nixos/modules/ssh.nix b/nixos/modules/ssh.nix
new file mode 100644
index 0000000000000000000000000000000000000000..59ee2fec949be247143041379626f35b7d8bf657
--- /dev/null
+++ b/nixos/modules/ssh.nix
@@ -0,0 +1,55 @@
+# A NixOS module which configures SSH access to a system.
+{
+ lib,
+ config,
+ ...
+}: {
+ options = {
+ services.private-storage.sshUsers = lib.mkOption {
+ type = lib.types.attrsOf lib.types.str;
+ example = lib.literalExample { root = "ssh-ed25519 AAA..."; };
+ description = ''
+ Users to configure on the issuer server and the storage servers and
+ the SSH public keys to use to authenticate them.
+ '';
+ };
+ };
+ config =
+ let
+ cfg = config.services."private-storage";
+ in {
+ # An attempt at a properly secure SSH configuration. This is informed by
+ # personal experience as well as various web resources:
+ #
+ # https://www.cyberciti.biz/tips/linux-unix-bsd-openssh-server-best-practices.html
+ services.openssh = {
+ enable = true;
+
+ # We don't use SFTP for anything. No reason to expose it.
+ allowSFTP = false;
+
+ # We only allow key-based authentication.
+ challengeResponseAuthentication = false;
+ passwordAuthentication = false;
+
+ extraConfig = ''
+ # Possibly this is superfluous considering we don't allow
+ # password-based authentication at all.
+ PermitEmptyPasswords no
+
+ # Only allow authentication as one of the configured users, not random
+ # other (often system-managed) users. Possibly this is also
+ # superfluous! NixOS system users have nologin as their shell ... so they
+ # cannot log in anyway.
+ AllowUsers ${builtins.concatStringsSep " " (builtins.attrNames cfg.sshUsers)}
+ '';
+ };
+
+ users.users =
+ let makeUserConfig = username: sshPublicKey: {
+ isNormalUser = username != "root";
+ openssh.authorizedKeys.keys = [ sshPublicKey ];
+ };
+ in builtins.mapAttrs makeUserConfig cfg.sshUsers;
+ };
+}
diff --git a/nixos/modules/tahoe.nix b/nixos/modules/tahoe.nix
index 05e68d4fd8c8bb83be85afefc6f1de66439cad8e..f1274534795fc53e27467ac9e28c34c6ae1e8be2 100644
--- a/nixos/modules/tahoe.nix
+++ b/nixos/modules/tahoe.nix
@@ -172,10 +172,10 @@ in
# (node: settings: settings.tub.port);
systemd.services = flip mapAttrs' cfg.nodes (node: settings:
let
- pidfile = "/run/tahoe.${node}.pid";
+ pidfile = "/run/tahoe.${lib.escapeShellArg node}.pid";
# This is a directory, but it has no trailing slash. Tahoe commands
# get antsy when there's a trailing slash.
- nodedir = "/var/db/tahoe-lafs/${node}";
+ nodedir = "/var/db/tahoe-lafs/${lib.escapeShellArg node}";
in nameValuePair "tahoe.${node}" {
description = "Tahoe LAFS node ${node}";
wantedBy = [ "multi-user.target" ];
@@ -189,13 +189,31 @@ in
# arguments to $(tahoe run). The node directory must come first,
# and arguments which alter Twisted's behavior come afterwards.
ExecStart = ''
- ${settings.package}/bin/tahoe run ${lib.escapeShellArg nodedir} -n -l- --pidfile=${lib.escapeShellArg pidfile}
+ ${settings.package}/bin/tahoe run ${nodedir} -n -l- --pidfile=${pidfile}
'';
};
- preStart = ''
- if [ ! -d ${lib.escapeShellArg nodedir} ]; then
- mkdir -p /var/db/tahoe-lafs
- tahoe create-node --hostname=localhost ${lib.escapeShellArg nodedir}
+ preStart =
+ let
+ created = "${nodedir}.created";
+ atomic = "${nodedir}.atomic";
+ in ''
+ if [ ! -e ${created} ]; then
+ mkdir -p /var/db/tahoe-lafs/
+
+ # Get rid of any prior partial efforts. It might not exist.
+ # Don't let this tank us.
+ rm -rv ${atomic} && [ ! -e ${atomic} ]
+
+ # Really create the node.
+ tahoe create-node --hostname=localhost ${atomic}
+
+ # Move it to the real location. We don't create it in-place
+ # because we might fail partway through and leave inconsistent
+ # state. Also, systemd probably created logs/incidents/ already and
+ # `create-node` complains if it finds these exist already.
+ rm -rv ${nodedir} && [ ! -e ${nodedir} ]
+ mv ${atomic} ${nodedir}
+ touch ${created}
fi
# Tahoe has created a predefined tahoe.cfg which we must now
@@ -204,7 +222,7 @@ in
# we must do this on every prestart. Fixes welcome.
# rm ${nodedir}/tahoe.cfg
# ln -s /etc/tahoe-lafs/${lib.escapeShellArg node}.cfg ${nodedir}/tahoe.cfg
- cp /etc/tahoe-lafs/${lib.escapeShellArg node}.cfg ${lib.escapeShellArg nodedir}/tahoe.cfg
+ cp /etc/tahoe-lafs/${lib.escapeShellArg node}.cfg ${nodedir}/tahoe.cfg
'';
});
users.users = flip mapAttrs' cfg.nodes (node: _:
diff --git a/nixos/modules/tests/get-passes.py b/nixos/modules/tests/get-passes.py
index 96875713233b7c46abae0cfc3f0d946628bf8cc2..39690ad8a41edd161c4ff1f6a682669c15828266 100755
--- a/nixos/modules/tests/get-passes.py
+++ b/nixos/modules/tests/get-passes.py
@@ -42,13 +42,23 @@ def main():
response.raise_for_status()
# Poll the vouchers list for a while to see it get redeemed.
- expected = {"version": 1, "number": voucher, "redeemed": True}
def find_redeemed_voucher():
response = get(zkapauthz + "/voucher/" + voucher)
response.raise_for_status()
actual = response.json()
print("Actual response: {}".format(actual))
- return expected == actual
+ try:
+ check = (
+ actual["version"],
+ actual["number"],
+ actual["state"]["name"],
+ )
+ except Exception as e:
+ print("Check failed: {}".format(e))
+ return False
+ else:
+ print("Checking {}".format(check))
+ return check == (1, voucher, "redeemed")
retry(
"find redeemed voucher",
diff --git a/nixos/modules/tests/private-storage.nix b/nixos/modules/tests/private-storage.nix
index 1fe55c1302b35aeb9e5645bbc1cb60053fc6e97c..47acfbf475a5d029b5f2800d4a11e2ff18eaa9d2 100644
--- a/nixos/modules/tests/private-storage.nix
+++ b/nixos/modules/tests/private-storage.nix
@@ -2,6 +2,20 @@ let
pkgs = import <nixpkgs> { };
pspkgs = import ../pspkgs.nix { inherit pkgs; };
+ sshPrivateKey = ./probeuser_ed25519;
+ sshPublicKey = ./probeuser_ed25519.pub;
+ sshUsers = {
+ root = (builtins.readFile sshPublicKey);
+ probeuser = (builtins.readFile sshPublicKey);
+ };
+ # Generate a command which can be used with runOnNode to ssh to the given
+ # host.
+ ssh = username: hostname: [
+ "cp" sshPrivateKey "/tmp/ssh_key" ";"
+ "chmod" "0400" "/tmp/ssh_key" ";"
+ "ssh" "-oStrictHostKeyChecking=no" "-i" "/tmp/ssh_key" "${username}@${hostname}" ":"
+ ];
+
# Separate helper programs so we can write as little perl inside a string
# inside a nix expression as possible.
run-introducer = ./run-introducer.py;
@@ -108,6 +122,8 @@ import <nixpkgs/nixos/tests/make-test.nix> {
introducerFURL = introducerFURL;
issuerRootURL = issuerURL;
inherit ristrettoSigningKeyPath;
+ inherit sshUsers;
+
};
} // networkConfig;
@@ -117,6 +133,8 @@ import <nixpkgs/nixos/tests/make-test.nix> {
{ imports =
[ ../issuer.nix
];
+ services.private-storage.sshUsers = sshUsers;
+
services.private-storage-issuer = {
enable = true;
domain = "issuer";
@@ -165,6 +183,16 @@ import <nixpkgs/nixos/tests/make-test.nix> {
# Start booting all the VMs in parallel to speed up operations down below.
startAll;
+ # The issuer and the storage server should accept SSH connections. This
+ # doesn't prove it is so but if it fails it's a pretty good indication
+ # it isn't so.
+ $storage->waitForOpenPort(22);
+ ${runOnNode "issuer" (ssh "probeuser" "storage")}
+ ${runOnNode "issuer" (ssh "root" "storage")}
+ $issuer->waitForOpenPort(22);
+ ${runOnNode "storage" (ssh "probeuser" "issuer")}
+ ${runOnNode "storage" (ssh "root" "issuer")}
+
# Set up a Tahoe-LAFS introducer.
$introducer->copyFileFromHost(
'${pemFile}',
diff --git a/nixos/modules/tests/probeuser_ed25519 b/nixos/modules/tests/probeuser_ed25519
new file mode 100644
index 0000000000000000000000000000000000000000..09c734d087d2e550163f11c088f7262e8a4b0e8b
--- /dev/null
+++ b/nixos/modules/tests/probeuser_ed25519
@@ -0,0 +1,38 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEA9RqVFIjOI9Wwwhmlrt3HIeom4fgKcdd6DW8vFR25aynXGILwIFFJ
+9BW1IhrEAdSUdwtwdDDKB1nV3DxvYrS4tty+BdbKl8ZbqyflHy87sNnt027LpvBa9AReZp
+7eNXZ4myIReg7lQnQeOyVWXfF7y2OCzBUb089IO+2AdoFBDrDrvnJpDXURiBpj8oA9FOvd
+4BzkfRuSqTOODTrjXSilhDSU0N5kko5bny07tWDsI+obJFVGALzO5rt86I8T+brWevAwA9
+5DPQAwrIK7cl9O9eH/59cVBYwdQMtgGfjqsSHbP2YqxROjkK9BFSAgNib2IXyJ0OkzZN0s
+If292lUkkDfKppfJ8I+z1Wc37/E8LkW0B5KSxal79cftUxqJIT1sfeVDo31r5UdK8V9kkA
+tsJFNqJrETyMDZeboJF5x8YWXCgQM++Ts24P3vBcwOJwcvam+BcmtZhay2+0jrBFVqA8Mk
+w8zNEViGaV/zlKSft3ZetYyj5lk/JrNqHSl5T9j5AAAFiO+v0gLvr9ICAAAAB3NzaC1yc2
+EAAAGBAPUalRSIziPVsMIZpa7dxyHqJuH4CnHXeg1vLxUduWsp1xiC8CBRSfQVtSIaxAHU
+lHcLcHQwygdZ1dw8b2K0uLbcvgXWypfGW6sn5R8vO7DZ7dNuy6bwWvQEXmae3jV2eJsiEX
+oO5UJ0HjslVl3xe8tjgswVG9PPSDvtgHaBQQ6w675yaQ11EYgaY/KAPRTr3eAc5H0bkqkz
+jg06410opYQ0lNDeZJKOW58tO7Vg7CPqGyRVRgC8zua7fOiPE/m61nrwMAPeQz0AMKyCu3
+JfTvXh/+fXFQWMHUDLYBn46rEh2z9mKsUTo5CvQRUgIDYm9iF8idDpM2TdLCH9vdpVJJA3
+yqaXyfCPs9VnN+/xPC5FtAeSksWpe/XH7VMaiSE9bH3lQ6N9a+VHSvFfZJALbCRTaiaxE8
+jA2Xm6CRecfGFlwoEDPvk7NuD97wXMDicHL2pvgXJrWYWstvtI6wRVagPDJMPMzRFYhmlf
+85Skn7d2XrWMo+ZZPyazah0peU/Y+QAAAAMBAAEAAAGBAKkeomb8zl/jfocvcybpWBGKoz
+GgGHTcnRbP9Mi5LctHn2cGUfG6pTCKGeViWoR4zcgmWH2TfJL95ZaFDMYqtJlYTrVws3Fu
+KKo8aNfPm3w3ouYUuOiDR/6/VPOyAtkY2bcRFsYFqSLlREbDSIihqy13iDSRDBZmHA1dnx
+olrWIZqVBLWTkz4djrfLNC0OKyrPGKfg3lDJk7PXTbgS4ycaJ7NYO5L+P/3jBC4cQREF7n
+lbrIK/kuTgTesR02PC5AP4v2nU0nMvdI8v4mYx+Ybji7dO93ZbZkI2FRxp/cl50FSKjc6N
+9im3CWqb0I07+h0cy9rNAhVT221Kfjypv0SZp/1xE+VpVx4KyLkTenvhzhLfTNdbWHn/kv
+GSWX8BvQEiUvUORExfjvmo2/y+o/ca8DRTN/KuT/mUkjFdSFdlFWkMD5xF6YkPaFarhxbt
+yD670DpqV21P8xoXZCP9DGKefFAdfemkhpSxyZmKimb4vHUQD4Ddkc3CNqzw/sJdLQQQAA
+AMEAma/GNKl599PkWA58AtEQUWAcVKUljSOgyzRcncOiSKJ1K5RYHYnOPLjBBKb9yOF4Ss
+5WVvozqByJSowrXHWPj5qAlGCfNCzkJ9pKDA6BCcxbQmAzPXl8TFx1v25daYMDTMH29Sxq
+3UjzxthlP9GMovqe7BtFdd1Ep+V60wnHUcbBKIUQmlIhB9T4GyOyPwn6FlXkaz2hvk5jMM
+pTrB2DdFmIgyQzNuuPauJ8Krjx4gTVbi1teHZLljRwokMXsOsxAAAAwQD9vmyjKZhbWgkL
+/DDQp983yCcvF/ywnqHRQZijHRyh5QT48DE5xCQV9MQtidbNQgP6z6Z8BK6kXIy8CSA8zx
+tXu2aAz8q3XCn3X8Tp1sbNKtQs/d/bf9AfOgXbhRKSE3P/1jYQVdlnyZht/Rvjxflc2KQZ
+QV7RtLE4TAQF1HuwnppCc2tI2yT24LUNBS/IzsvgT+CaatpeOblWwAiIN/OYpf4A5vjcFd
+omNYiKuKywhPI6e+UX9BzL0F4rDQ9MsEUAAADBAPdIfdBo8UUsctAjzYNrlq9lG81CxKYf
+Id34d61eg5JNCozPOx86yNpvs5Itxr8SXQUvxDUWTour7ceByssRkJNllDdp4s4t3Ya7Wk
+cFTFfkfPPbFmY0JbVFimpgMVulWfLgE2tnSDpiqYnHc55e+DBR4DbaCJdtRKAbE9vYYamd
+/+qgMQdqsWJ7D0aVao0f6IcGUDL7Bn2qd4phrLtCpyOhV8hOr17WukTQlDHCU7VFAF9jLe
+Uu3rxmzZD94QRTJQAAAA5leGFya3VuQGJhcnlvbgECAw==
+-----END OPENSSH PRIVATE KEY-----
diff --git a/nixos/modules/tests/probeuser_ed25519.pub b/nixos/modules/tests/probeuser_ed25519.pub
new file mode 100644
index 0000000000000000000000000000000000000000..e9ba1bbd5cf64d7448845d1408846e201085be4d
--- /dev/null
+++ b/nixos/modules/tests/probeuser_ed25519.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD1GpUUiM4j1bDCGaWu3cch6ibh+Apx13oNby8VHblrKdcYgvAgUUn0FbUiGsQB1JR3C3B0MMoHWdXcPG9itLi23L4F1sqXxlurJ+UfLzuw2e3Tbsum8Fr0BF5mnt41dnibIhF6DuVCdB47JVZd8XvLY4LMFRvTz0g77YB2gUEOsOu+cmkNdRGIGmPygD0U693gHOR9G5KpM44NOuNdKKWENJTQ3mSSjlufLTu1YOwj6hskVUYAvM7mu3zojxP5utZ68DAD3kM9ADCsgrtyX0714f/n1xUFjB1Ay2AZ+OqxIds/ZirFE6OQr0EVICA2JvYhfInQ6TNk3Swh/b3aVSSQN8qml8nwj7PVZzfv8TwuRbQHkpLFqXv1x+1TGokhPWx95UOjfWvlR0rxX2SQC2wkU2omsRPIwNl5ugkXnHxhZcKBAz75Ozbg/e8FzA4nBy9qb4Fya1mFrLb7SOsEVWoDwyTDzM0RWIZpX/OUpJ+3dl61jKPmWT8ms2odKXlP2Pk= probeuser