diff --git a/ci-tools/known_hosts.staging b/ci-tools/known_hosts.staging
index 2a015656e4c21498f2fe152723888046d9341c1a..1a8e6245e569fdda6daa81cc67c4c8cc33b8e8f8 100644
--- a/ci-tools/known_hosts.staging
+++ b/ci-tools/known_hosts.staging
@@ -1,3 +1,3 @@
monitoring.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINI9kvEBaOMvpWqcFH+6nFvRriBECKB4RFShdPiIMkk9
payments.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK0eO/01VFwdoZzpclrmu656eaMkE19BaxtDdkkFHMa8
-storage001.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFP8L6OHCxq9XFd8ME8ZrCbmO5dGZDPH8I5dm0AwSGiN
+storage001.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA6iWHO9/4s3h9VIpaxgD+rgj/OQh8+jupxBoOmie3St
diff --git a/default.nix b/default.nix
index 6441675a243e22e6154267c656652c8d8575940e..8b605281715be3f74c375a6b5532a4a87a6b4993 100644
--- a/default.nix
+++ b/default.nix
@@ -1,4 +1,4 @@
-{ pkgs ? import ./nixpkgs-2105.nix { } }:
+{ pkgs ? import ./nixpkgs.nix { } }:
{
# Render the project documentation source to some presentation format (ie,
# html) with Sphinx.
diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix
index e672f3d14ef5f7389511a818921a9a75c6e948fe..da8a83812ceba910280bfc61210487b2f217113f 100644
--- a/morph/grid/local/grid.nix
+++ b/morph/grid/local/grid.nix
@@ -75,7 +75,7 @@ let
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.11";
- grid.publicIPv4 = "192.168.67.21";
+ grid.publicIPv4 = "192.168.56.21";
grid.issuer = {
inherit (grid-config) issuerDomains allowedChargeOrigins;
};
@@ -89,7 +89,7 @@ let
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.12";
- grid.publicIPv4 = "192.168.67.22";
+ grid.publicIPv4 = "192.168.56.22";
grid.storage = {
inherit (grid-config) passValue publicStoragePort;
};
@@ -104,7 +104,7 @@ let
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.13";
- grid.publicIPv4 = "192.168.67.23";
+ grid.publicIPv4 = "192.168.56.23";
grid.storage = {
inherit (grid-config) passValue publicStoragePort;
};
@@ -119,7 +119,7 @@ let
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.1";
- grid.publicIPv4 = "192.168.67.24";
+ grid.publicIPv4 = "192.168.56.24";
grid.monitoring = {
inherit paymentExporterTargets blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
diff --git a/morph/lib/default.nix b/morph/lib/default.nix
index 88c83bc2211da2e00e69f95d7d2110d3ee636cc3..c43fa6ea1ebc6b3159f3f2f6872c23f0da62b776 100644
--- a/morph/lib/default.nix
+++ b/morph/lib/default.nix
@@ -17,7 +17,7 @@
# installed, as well as the NixOS module set that is used.
# This is intended to be used in a grid definition like:
# network = { ... ; inherit (gridlib) pkgs; ... }
- pkgs = import ../../nixpkgs-2105.nix {
+ pkgs = import ../../nixpkgs.nix {
# Ensure that configuration of the system where this runs
# doesn't leak into what we build.
# See https://github.com/NixOS/nixpkgs/issues/62513
diff --git a/morph/lib/monitoring.nix b/morph/lib/monitoring.nix
index 64f26c92c4e0e8f7cc961b98972ec6c3ce8f2c89..3cf1680e1e1d0d1399b3e5741cc8b3850f68c2e5 100644
--- a/morph/lib/monitoring.nix
+++ b/morph/lib/monitoring.nix
@@ -111,19 +111,19 @@ in {
permissions = "0400";
action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
};
- }
- (lib.mkIf (cfg.googleOAuthClientID != "") {
- "grafana-google-sso-secret" = {
- source = "${privateKeyPath}/grafana-google-sso.secret";
- destination = "/run/keys/grafana-google-sso.secret";
+ "grafana-admin-password" = {
+ source = "${privateKeyPath}/grafana-admin.password";
+ destination = "/run/keys/grafana-admin.password";
owner.user = config.systemd.services.grafana.serviceConfig.User;
owner.group = config.users.users.grafana.group;
permissions = "0400";
action = ["sudo" "systemctl" "restart" "grafana.service"];
};
- "grafana-admin-password" = {
- source = "${privateKeyPath}/grafana-admin.password";
- destination = "/run/keys/grafana-admin.password";
+ }
+ (lib.mkIf (cfg.googleOAuthClientID != "") {
+ "grafana-google-sso-secret" = {
+ source = "${privateKeyPath}/grafana-google-sso.secret";
+ destination = "/run/keys/grafana-google-sso.secret";
owner.user = config.systemd.services.grafana.serviceConfig.User;
owner.group = config.users.users.grafana.group;
permissions = "0400";
diff --git a/nixos/lib/default.nix b/nixos/lib/default.nix
new file mode 100644
index 0000000000000000000000000000000000000000..3ebaf60b7d536589d46a745d2945cbef96b2b554
--- /dev/null
+++ b/nixos/lib/default.nix
@@ -0,0 +1,6 @@
+{ callPackage }:
+{
+ /* A library of tools useful for writing tests with Nix.
+ */
+ testing = callPackage ./testing.nix { };
+}
diff --git a/nixos/lib/testing.nix b/nixos/lib/testing.nix
new file mode 100644
index 0000000000000000000000000000000000000000..d89717a0a76f93bb6062ad63c6cfdbb91c12c746
--- /dev/null
+++ b/nixos/lib/testing.nix
@@ -0,0 +1,23 @@
+{ ...}:
+{
+ /* Returns a string that runs tests from the Python code at the given path.
+
+ The Python code is loaded using *execfile* and the *test* global it
+ defines is called with the given keyword arguments.
+
+ Type: makeTestScript :: Path -> AttrSet -> String
+
+ Example:
+ testScript = (makeTestScript ./test_foo.py { x = "y"; });
+ */
+ makeTestScript = { testpath, kwargs ? {} }:
+ ''
+ # The driver runs pyflakes on this script before letting it
+ # run... Convince pyflakes that there is a `test` name.
+ test = None
+ with open("${testpath}") as testfile:
+ exec(testfile.read(), globals())
+ # For simple types, JSON is compatible with Python syntax!
+ test(**${builtins.toJSON kwargs})
+ '';
+}
diff --git a/nixos/modules/monitoring/exporters/megacli2prom.nix b/nixos/modules/monitoring/exporters/megacli2prom.nix
index a38f1ccc18b59073ff835e50babeb565f79a20b8..c1f10aef7821343e2f0a572d006782858dcf7ef4 100644
--- a/nixos/modules/monitoring/exporters/megacli2prom.nix
+++ b/nixos/modules/monitoring/exporters/megacli2prom.nix
@@ -35,11 +35,10 @@ in {
config =
lib.mkIf cfg.enable {
- environment.systemPackages = [ ourpkgs.megacli2prom ];
+ environment.systemPackages = [ ourpkgs.megacli2prom pkgs.megacli ];
systemd.services.megacli2prom = {
enable = true;
description = "MegaCli2Prom metrics gathering service";
- wantedBy = [ "multi-user.target" ];
startAt = cfg.interval;
path = [ pkgs.megacli ];
# Save to a temp file and then move atomically so the
diff --git a/nixos/modules/tahoe.nix b/nixos/modules/tahoe.nix
index e0b6eb4d8be3c5359de1d391c42b2ba83f7a1ba4..44c381e6b6dfa6039d1dd6a49d44f1afaf51ab10 100644
--- a/nixos/modules/tahoe.nix
+++ b/nixos/modules/tahoe.nix
@@ -156,6 +156,10 @@ in
nameValuePair "tahoe.introducer-${node}" {
description = "Tahoe node user for introducer ${node}";
isSystemUser = true;
+ group = "tahoe.introducer-${node}";
+ });
+ users.groups = flip mapAttrs' cfg.introducers (node: _:
+ nameValuePair "tahoe.introducer-${node}" {
});
})
(mkIf (cfg.nodes != {}) {
@@ -287,6 +291,10 @@ in
nameValuePair "tahoe.${node}" {
description = "Tahoe node user for node ${node}";
isSystemUser = true;
+ group = "tahoe.${node}";
+ });
+ users.groups = flip mapAttrs' cfg.introducers (node: _:
+ nameValuePair "tahoe.${node}" {
});
})
];
diff --git a/nixos/modules/update-deployment b/nixos/modules/update-deployment
index 1c8960588f418e57eeaadb7ad29db4285369cbdd..a8efffa062ad8f8dc6b6dc22827e4f0087b4d618 100755
--- a/nixos/modules/update-deployment
+++ b/nixos/modules/update-deployment
@@ -72,14 +72,14 @@ EOF
ssh -o StrictHostKeyChecking=no "$(hostname).$(domainname)" ":"
# Set nixpkgs to our preferred version for the morph build. Annoyingly, we
-# can't just use nixpkgs-2105.nix as our nixpkgs because some code (in morph,
+# can't just use nixpkgs.nix as our nixpkgs because some code (in morph,
# at least) wants <nixpkgs> to be a fully-resolved path to a nixpkgs tree.
# For example, morph evaluated `import <nixpkgs/lib>` which would turn into
-# something like `import nixpkgs-2105.nix/lib` which is nonsense.
+# something like `import nixpkgs.nix/lib` which is nonsense.
#
# So instead, import our nixpkgs which forces it to be instantiated in the
# store, then ask for its path, then set NIX_PATH to that.
-export NIX_PATH="nixpkgs=$(nix eval "(import ${CHECKOUT}/nixpkgs-2105.nix { }).path")"
+export NIX_PATH="nixpkgs=$(nix eval "(import ${CHECKOUT}/nixpkgs.nix { }).path")"
# Attempt to update just this host. Choose the morph grid definition matching
# the grid we belong to and limit the morph deployment update to the host
diff --git a/nixos/pkgs/default.nix b/nixos/pkgs/default.nix
index bfc30b36101c220434606832127a7e8ca0a70490..435095f7890b7ac41afaebe050a756c4b4887641 100644
--- a/nixos/pkgs/default.nix
+++ b/nixos/pkgs/default.nix
@@ -5,6 +5,8 @@
# pkgs.callPackage ./nixos/pkgs
{buildPlatform, hostPlatform, callPackage}:
{
+ lib = callPackage ../lib {};
+
leasereport = callPackage ./leasereport {};
# `privatestorage` is a derivation with a good Tahoe+ZKAP environment
# that is exposed by ZKAPAuthorizer.
diff --git a/nixos/pkgs/megacli2prom/repo.json b/nixos/pkgs/megacli2prom/repo.json
index 3c8cd0af95adf95e22def4e727b8c2c5d12044aa..daa4c7b8a0a7dba8ca382c9e1913e2c7ff2d36cd 100644
--- a/nixos/pkgs/megacli2prom/repo.json
+++ b/nixos/pkgs/megacli2prom/repo.json
@@ -2,7 +2,7 @@
"owner": "PrivateStorageio",
"repo": "megacli2prom",
"branch": "main",
- "rev": "9536933d325c843b2662f80486660bf81d73941e",
+ "rev": "e76300ca0a723bf0ed105d805f166976162d58d3",
"outputHashAlgo": "sha512",
- "outputHash": "1xrsv0bkmazbhqarx84lhvmrzzdv1bm04xvr0hw1yrw1f4xb450f4pwgapnkjczy0l4c6rp3pmh64cblgbs3ki30wacbv1bqzv5745g"
+ "outputHash": "256di1f4bw5a0kqm37wr5dk9yg0cxhgqaflrhk0p3azimml3pd1gr4rh54mj4vsrw17iyziajmilx98fsvc9w70y14rh7kgxcam9vwp"
}
\ No newline at end of file
diff --git a/nixos/pkgs/privatestorage/default.nix b/nixos/pkgs/privatestorage/default.nix
index bd487af32941f6db920ea2d43ec89e9eded38201..3bbbd3dbcf0b974e6e1997e20773cddbd9ea59c0 100644
--- a/nixos/pkgs/privatestorage/default.nix
+++ b/nixos/pkgs/privatestorage/default.nix
@@ -2,7 +2,7 @@
let
repo-data = lib.importJSON ./repo.json;
repo = fetchFromGitHub (builtins.removeAttrs repo-data [ "branch" ]);
- privatestorage = callPackage repo {};
+ privatestorage = callPackage repo { python = "python39"; };
in
privatestorage.privatestorage
diff --git a/nixos/pkgs/privatestorage/repo.json b/nixos/pkgs/privatestorage/repo.json
index 81f6e18ba4bbec657a5a5ba543ef05408bf472ad..20fb749ffdbdcdb86e2af21c63ac1c756c6157a1 100644
--- a/nixos/pkgs/privatestorage/repo.json
+++ b/nixos/pkgs/privatestorage/repo.json
@@ -2,7 +2,7 @@
"owner": "PrivateStorageio",
"branch": "main",
"repo": "ZKAPAuthorizer",
- "rev": "b61f3d4a3f5eb72cb600dd83796a1aaca2931e07",
+ "rev": "a263d171aa20d6b34926a6af51b849cd127b7190",
"outputHashAlgo": "sha512",
- "outputHash": "2d7a9m34jx1k38fmiwskgwd1ryyhrb56m9nam12fd66shl8qzmlfcr1lwf063qi1wqdzb2g7998vxbv3c2bmvw7g6iqwzjmsck2czpn"
-}
\ No newline at end of file
+ "outputHash": "0sxxhmrfbag5ksis0abvwnlqzfiqlwdyfnhav2h7hpn62r81l7k2gk4jhs3blw8r6bv6di978lx6rv56r1vmjnpmlyx2l33afzf9bf3"
+}
diff --git a/nixos/pkgs/zkap-spending-service/repo.json b/nixos/pkgs/zkap-spending-service/repo.json
index 69f7a30053de661f2c7829384e9496e49077cfd9..eafc2e2e96592926a40535ee370db9bda9f10cc4 100644
--- a/nixos/pkgs/zkap-spending-service/repo.json
+++ b/nixos/pkgs/zkap-spending-service/repo.json
@@ -1,9 +1,9 @@
-{
- "owner": "privatestorage",
- "repo": "zkap-spending-service",
- "rev": "cbf7509f429ffd6e6cf37a73e4ff84a9c5ce1141",
- "branch": "main",
- "domain": "whetstone.privatestorage.io",
- "outputHash": "04g7pcykc2525cg3z7wg5834s7vqn82xaqjvf52l6dnxv3mb9xr93kk505dvxcwhgfbqpim5i479s9kqd8gi7q3lq5wn5fq7rf7lkrj",
- "outputHashAlgo": "sha512"
-}
+{
+ "owner": "privatestorage",
+ "repo": "zkap-spending-service",
+ "rev": "66fd395268b466d4c7bb0a740fb758a5acccd1c4",
+ "branch": "main",
+ "domain": "whetstone.privatestorage.io",
+ "outputHash": "1nryvsccncrka25kzrwqkif4x68ib0cs2vbw1ngfmzw86gjgqx01a7acgspmrpfs62p4q8zw0f2ynl8jr3ygyypjrl8v7w8g49y0y0y",
+ "outputHashAlgo": "sha512"
+}
diff --git a/nixos/tests/private-storage.nix b/nixos/tests/private-storage.nix
index a208ce249f1f1227f966e38a1c62ab6166d187f8..eaff1ed5320607e6aabc94226804aea4b7186b0a 100644
--- a/nixos/tests/private-storage.nix
+++ b/nixos/tests/private-storage.nix
@@ -1,25 +1,14 @@
{ pkgs }:
let
+ ourpkgs = pkgs.callPackage ../pkgs { };
+
sshPrivateKey = ./probeuser_ed25519;
sshPublicKey = ./probeuser_ed25519.pub;
+
sshUsers = {
root = (builtins.readFile sshPublicKey);
probeuser = (builtins.readFile sshPublicKey);
};
- # Generate a command which can be used with runOnNode to ssh to the given
- # host.
- ssh = username: hostname: [
- "cp" sshPrivateKey "/tmp/ssh_key" ";"
- "chmod" "0400" "/tmp/ssh_key" ";"
- "ssh" "-oStrictHostKeyChecking=no" "-i" "/tmp/ssh_key" "${username}@${hostname}" ":"
- ];
-
- # Separate helper programs so we can write as little python inside a string
- # inside a nix expression as possible.
- run-introducer = ./run-introducer.py;
- run-client = ./run-client.py;
- get-passes = ./get-passes.py;
- exercise-storage = ./exercise-storage.py;
# This is a test double of the Stripe API server. It is extremely simple.
# It barely knows how to respond to exactly the API endpoints we use,
@@ -72,18 +61,6 @@ let
networking.firewall.enable = false;
networking.dhcpcd.enable = false;
};
-
- # Return a python program fragment to run a shell command on one of the nodes.
- # The first argument is the name of the node. The second is a list of the
- # argv to run.
- #
- # The program's output is piped to systemd-cat and the python fragment
- # evaluates to success if the command exits with a success status.
- runOnNode = node: argv:
- let
- command = builtins.concatStringsSep " " argv;
- in
- "${node}.succeed('set -eo pipefail; ${command} | systemd-cat')";
in {
# https://nixos.org/nixos/manual/index.html#sec-nixos-tests
# https://nixos.mayflower.consulting/blog/2019/07/11/leveraging-nixos-tests-in-your-project/
@@ -177,134 +154,16 @@ in {
};
# Test the machines with a Python program.
- testScript = ''
- # Boot the VMs. We used to do them all in parallel but the boot
- # sequence got flaky at some point for some reason I don't
- # understand. :/ It might be related to this:
- #
- # https://discourse.nixos.org/t/nixos-ppc64le-vm-does-not-have-dev-vda-device/11548/9
- #
- # See <nixpkgs/nixos/modules/virtualisation/qemu-vm.nix> for the Nix
- # that constructs the QEMU command that gets run.
- #
- # Boot them one at a time for now.
- issuer.connect()
- introducer.connect()
- storage.connect()
- client.connect()
- api_stripe_com.connect()
-
- # The issuer and the storage server should accept SSH connections. This
- # doesn't prove it is so but if it fails it's a pretty good indication
- # it isn't so.
- storage.wait_for_open_port(22)
- ${runOnNode "issuer" (ssh "probeuser" "storage")}
- ${runOnNode "issuer" (ssh "root" "storage")}
- issuer.wait_for_open_port(22)
- ${runOnNode "storage" (ssh "probeuser" "issuer")}
- ${runOnNode "storage" (ssh "root" "issuer")}
-
- # Set up a Tahoe-LAFS introducer.
- introducer.copy_from_host('${pemFile}', '/tmp/node.pem')
-
- try:
- ${runOnNode "introducer" [ run-introducer "/tmp/node.pem" (toString introducerPort) introducerFURL ]}
- except:
- code, output = introducer.execute('cat /tmp/stdout /tmp/stderr')
- introducer.log(output)
- raise
-
- #
- # Get a Tahoe-LAFS storage server up.
- #
- code, version = storage.execute('tahoe --version')
- storage.log(version)
-
- # The systemd unit should reach the running state.
- storage.wait_for_unit('tahoe.storage.service')
-
- # Some while after that the Tahoe-LAFS node should listen on the web API
- # port. The port number here has to agree with the port number set in
- # the private-storage.nix module.
- storage.wait_for_open_port(3456)
-
- # Once the web API is listening it should be possible to scrape some
- # status from the node if it is really working.
- storage.succeed('tahoe -d /var/db/tahoe-lafs/storage status')
-
- # It should have Eliot logging turned on as well.
- storage.succeed('[ -e /var/db/tahoe-lafs/storage/logs/eliot.json ]')
-
- #
- # Storage appears to be working so try to get a client to speak with it.
- #
- ${runOnNode "client" [ run-client "/tmp/client" introducerFURL issuerURL ristrettoPublicKey ]}
- client.wait_for_open_port(3456)
-
- # Make sure the fake Stripe API server is ready for requests.
- try:
- api_stripe_com.wait_for_unit("api.stripe.com")
- except:
- code, output = api_stripe_com.execute('journalctl -u api.stripe.com')
- api_stripe_com.log(output)
- raise
-
- # Get some ZKAPs from the issuer.
- try:
- ${runOnNode "client" [
- get-passes
- "http://127.0.0.1:3456"
- "/tmp/client/private/api_auth_token"
- issuerURL
- voucher
- ]}
- except:
- code, output = client.execute('cat /tmp/stdout /tmp/stderr');
- client.log(output)
-
- # Dump the fake Stripe API server logs, too, since the error may arise
- # from a PaymentServer/Stripe interaction.
- code, output = api_stripe_com.execute('journalctl -u api.stripe.com')
- api_stripe_com.log(output)
- raise
-
- # The client should be prepped now. Make it try to use some storage.
- try:
- ${runOnNode "client" [ exercise-storage "/tmp/client" ]}
- except:
- code, output = client.execute('cat /tmp/stdout /tmp/stderr')
- client.log(output)
- raise
-
- # It should be possible to restart the storage service without the
- # storage node fURL changing.
- try:
- furlfile = '/var/db/tahoe-lafs/storage/private/storage-plugin.privatestorageio-zkapauthz-v1.furl'
- before = storage.execute('cat ' + furlfile)
- ${runOnNode "storage" [ "systemctl" "restart" "tahoe.storage" ]}
- after = storage.execute('cat ' + furlfile)
- if (before != after):
- raise Exception('fURL changes after storage node restart')
- except:
- code, output = storage.execute('cat /tmp/stdout /tmp/stderr')
- storage.log(output)
- raise
-
- # The client should actually still work, too.
- try:
- ${runOnNode "client" [ exercise-storage "/tmp/client" ]}
- except:
- code, output = client.execute('cat /tmp/stdout /tmp/stderr')
- client.log(output)
- raise
-
- # The issuer metrics should be accessible from the monitoring network.
- issuer.execute('ifconfig lo:fauxvpn 172.23.23.2/24')
- issuer.wait_until_succeeds("nc -z 172.23.23.2 80")
- issuer.succeed('curl --silent --insecure --fail --output /dev/null http://172.23.23.2/metrics')
- # The issuer metrics should NOT be accessible from any other network.
- issuer.fail('curl --silent --insecure --fail --output /dev/null http://localhost/metrics')
- client.fail('curl --silent --insecure --fail --output /dev/null http://issuer/metrics')
- issuer.execute('ifconfig lo:fauxvpn down')
- '';
+ testScript = ourpkgs.lib.testing.makeTestScript {
+ testpath = ./test_privatestorage.py;
+ kwargs = {
+ inherit sshPrivateKey pemFile introducerPort introducerFURL issuerURL ristrettoPublicKey voucher;
+
+ # Supply some helper programs to help the tests stay a bit higher level.
+ run_introducer = ./run-introducer.py;
+ run_client = ./run-client.py;
+ get_passes = ./get-passes.py;
+ exercise_storage = ./exercise-storage.py;
+ };
+ };
}
diff --git a/nixos/tests/tahoe.nix b/nixos/tests/tahoe.nix
index e39fd6d3fcb776e8e5215bb1264e08e2b7306c1f..a007e65efd2d6bee8ab4adba9df3cb2901f53526 100644
--- a/nixos/tests/tahoe.nix
+++ b/nixos/tests/tahoe.nix
@@ -1,5 +1,8 @@
-{ ... }:
- {
+{ pkgs }:
+let
+ ourpkgs = pkgs.callPackage ../pkgs { };
+in
+{
nodes = {
storage = { config, pkgs, ourpkgs, ... }: {
imports = [
@@ -23,50 +26,7 @@
};
};
};
- testScript = ''
- start_all()
-
- # After the service starts, destroy the "created" marker to force it to
- # re-create its internal state.
- storage.wait_for_open_port(4001)
- storage.succeed("systemctl stop tahoe.storage")
- storage.succeed("rm /var/db/tahoe-lafs/storage.created")
- storage.succeed("systemctl start tahoe.storage")
-
- # After it starts up again, verify it has consistent internal state and a
- # backup of the prior state.
- storage.wait_for_open_port(4001)
- storage.succeed("[ -e /var/db/tahoe-lafs/storage ]")
- storage.succeed("[ -e /var/db/tahoe-lafs/storage.created ]")
- storage.succeed("[ -e /var/db/tahoe-lafs/storage.1 ]")
- storage.succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.privkey ]")
- storage.succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.pem ]")
- storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.2 ]")
-
- # Stop it again, once again destroy the "created" marker, and this time also
- # jam some partial state in the way that will need cleanup.
- storage.succeed("systemctl stop tahoe.storage")
- storage.succeed("rm /var/db/tahoe-lafs/storage.created")
- storage.succeed("mkdir -p /var/db/tahoe-lafs/storage.atomic/partial")
- try:
- storage.succeed("systemctl start tahoe.storage")
- except:
- x, y = storage.execute("journalctl -u tahoe.storage")
- storage.log(y)
- raise
-
- # After it starts up again, verify it has consistent internal state and
- # backups of the prior two states. It also has no copy of the inconsistent
- # state because it could never have been used.
- storage.wait_for_open_port(4001)
- storage.succeed("[ -e /var/db/tahoe-lafs/storage ]")
- storage.succeed("[ -e /var/db/tahoe-lafs/storage.created ]")
- storage.succeed("[ -e /var/db/tahoe-lafs/storage.1 ]")
- storage.succeed("[ -e /var/db/tahoe-lafs/storage.2 ]")
- storage.succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.privkey ]")
- storage.succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.pem ]")
- storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.atomic ]")
- storage.succeed("[ ! -e /var/db/tahoe-lafs/storage/partial ]")
- storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.3 ]")
- '';
+ testScript = ourpkgs.lib.testing.makeTestScript {
+ testpath = ./test_tahoe.py;
+ };
}
diff --git a/nixos/tests/test_privatestorage.py b/nixos/tests/test_privatestorage.py
new file mode 100644
index 0000000000000000000000000000000000000000..dc060d51f1815f549485d3415b3b3af97d5c79af
--- /dev/null
+++ b/nixos/tests/test_privatestorage.py
@@ -0,0 +1,148 @@
+def runOnNode(node, argv):
+ """
+ Run a shell command on one of the nodes. The first argument is the name
+ of the node. The second is a list of the argv to run.
+
+ The program's output is piped to systemd-cat and the python fragment
+ evaluates to success if the command exits with a success status.
+ """
+ try:
+ node.succeed('set -eo pipefail; {} | systemd-cat'.format(" ".join(argv)))
+ except Exception as e:
+ code, output = node.execute('cat /tmp/stdout /tmp/stderr')
+ introducer.log(output)
+ raise
+
+def ssh(username, sshPrivateKey, hostname):
+ """
+ Generate a command which can be used with runOnNode to ssh to the given
+ host.
+ """
+ return [
+ "cp", sshPrivateKey, "/tmp/ssh_key", ";",
+ "chmod", "0400", "/tmp/ssh_key", ";",
+ "ssh", "-oStrictHostKeyChecking=no", "-i", "/tmp/ssh_key",
+ "{username}@{hostname}".format(username=username, hostname=hostname), ":",
+ ]
+
+def test(
+ sshPrivateKey,
+ pemFile,
+ run_introducer,
+ run_client,
+ get_passes,
+ exercise_storage,
+ introducerPort,
+ introducerFURL,
+ issuerURL,
+ ristrettoPublicKey,
+ voucher,
+):
+ """
+ """
+ # Boot the VMs. We used to do them all in parallel but the boot
+ # sequence got flaky at some point for some reason I don't
+ # understand. :/ It might be related to this:
+ #
+ # https://discourse.nixos.org/t/nixos-ppc64le-vm-does-not-have-dev-vda-device/11548/9
+ #
+ # See <nixpkgs/nixos/modules/virtualisation/qemu-vm.nix> for the Nix
+ # that constructs the QEMU command that gets run.
+ #
+ # Boot them one at a time for now.
+ issuer.connect()
+ introducer.connect()
+ storage.connect()
+ client.connect()
+ api_stripe_com.connect()
+
+ # The issuer and the storage server should accept SSH connections. This
+ # doesn't prove it is so but if it fails it's a pretty good indication
+ # it isn't so.
+ storage.wait_for_open_port(22)
+ runOnNode(issuer, ssh("probeuser", sshPrivateKey, "storage"))
+ runOnNode(issuer, ssh("root", sshPrivateKey, "storage"))
+ issuer.wait_for_open_port(22)
+ runOnNode(storage, ssh("probeuser", sshPrivateKey, "issuer"))
+ runOnNode(storage, ssh("root", sshPrivateKey, "issuer"))
+
+ # Set up a Tahoe-LAFS introducer.
+ introducer.copy_from_host(pemFile, '/tmp/node.pem')
+
+ runOnNode(introducer, [run_introducer, "/tmp/node.pem", str(introducerPort), introducerFURL])
+
+ #
+ # Get a Tahoe-LAFS storage server up.
+ #
+ code, version = storage.execute('tahoe --version')
+ storage.log(version)
+
+ # The systemd unit should reach the running state.
+ storage.wait_for_unit('tahoe.storage.service')
+
+ # Some while after that the Tahoe-LAFS node should listen on the web API
+ # port. The port number here has to agree with the port number set in
+ # the private-storage.nix module.
+ storage.wait_for_open_port(3456)
+
+ # Once the web API is listening it should be possible to scrape some
+ # status from the node if it is really working.
+ storage.succeed('tahoe -d /var/db/tahoe-lafs/storage status')
+
+ # It should have Eliot logging turned on as well.
+ storage.succeed('[ -e /var/db/tahoe-lafs/storage/logs/eliot.json ]')
+
+ #
+ # Storage appears to be working so try to get a client to speak with it.
+ #
+ runOnNode(client, [run_client, "/tmp/client", introducerFURL, issuerURL, ristrettoPublicKey])
+ client.wait_for_open_port(3456)
+
+ # Make sure the fake Stripe API server is ready for requests.
+ try:
+ api_stripe_com.wait_for_unit("api.stripe.com")
+ except:
+ code, output = api_stripe_com.execute('journalctl -u api.stripe.com')
+ api_stripe_com.log(output)
+ raise
+
+ # Get some ZKAPs from the issuer.
+ try:
+ runOnNode(client, [
+ get_passes,
+ "http://127.0.0.1:3456",
+ "/tmp/client/private/api_auth_token",
+ issuerURL,
+ voucher,
+ ])
+ except:
+ # Dump the fake Stripe API server logs, too, since the error may arise
+ # from a PaymentServer/Stripe interaction.
+ code, output = api_stripe_com.execute('journalctl -u api.stripe.com')
+ api_stripe_com.log(output)
+
+ raise
+
+ # The client should be prepped now. Make it try to use some storage.
+ runOnNode(client, [exercise_storage, "/tmp/client"])
+
+ # It should be possible to restart the storage service without the
+ # storage node fURL changing.
+ furlfile = '/var/db/tahoe-lafs/storage/private/storage-plugin.privatestorageio-zkapauthz-v1.furl'
+ before = storage.execute('cat ' + furlfile)
+ runOnNode(storage, ["systemctl", "restart", "tahoe.storage"])
+ after = storage.execute('cat ' + furlfile)
+ if (before != after):
+ raise Exception('fURL changes after storage node restart')
+
+ # The client should actually still work, too.
+ runOnNode(client, [exercise_storage, "/tmp/client"])
+
+ # The issuer metrics should be accessible from the monitoring network.
+ issuer.execute('ifconfig lo:fauxvpn 172.23.23.2/24')
+ issuer.wait_until_succeeds("nc -z 172.23.23.2 80")
+ issuer.succeed('curl --silent --insecure --fail --output /dev/null http://172.23.23.2/metrics')
+ # The issuer metrics should NOT be accessible from any other network.
+ issuer.fail('curl --silent --insecure --fail --output /dev/null http://localhost/metrics')
+ client.fail('curl --silent --insecure --fail --output /dev/null http://issuer/metrics')
+ issuer.execute('ifconfig lo:fauxvpn down')
diff --git a/nixos/tests/test_tahoe.py b/nixos/tests/test_tahoe.py
new file mode 100644
index 0000000000000000000000000000000000000000..c5190c78b04cb2cc59b5275e45869dd53e0e81c3
--- /dev/null
+++ b/nixos/tests/test_tahoe.py
@@ -0,0 +1,45 @@
+def test():
+ start_all()
+
+ # After the service starts, destroy the "created" marker to force it to
+ # re-create its internal state.
+ storage.wait_for_open_port(4001)
+ storage.succeed("systemctl stop tahoe.storage")
+ storage.succeed("rm /var/db/tahoe-lafs/storage.created")
+ storage.succeed("systemctl start tahoe.storage")
+
+ # After it starts up again, verify it has consistent internal state and a
+ # backup of the prior state.
+ storage.wait_for_open_port(4001)
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage ]")
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage.created ]")
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage.1 ]")
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.privkey ]")
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.pem ]")
+ storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.2 ]")
+
+ # Stop it again, once again destroy the "created" marker, and this time also
+ # jam some partial state in the way that will need cleanup.
+ storage.succeed("systemctl stop tahoe.storage")
+ storage.succeed("rm /var/db/tahoe-lafs/storage.created")
+ storage.succeed("mkdir -p /var/db/tahoe-lafs/storage.atomic/partial")
+ try:
+ storage.succeed("systemctl start tahoe.storage")
+ except:
+ x, y = storage.execute("journalctl -u tahoe.storage")
+ storage.log(y)
+ raise
+
+ # After it starts up again, verify it has consistent internal state and
+ # backups of the prior two states. It also has no copy of the inconsistent
+ # state because it could never have been used.
+ storage.wait_for_open_port(4001)
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage ]")
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage.created ]")
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage.1 ]")
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage.2 ]")
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.privkey ]")
+ storage.succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.pem ]")
+ storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.atomic ]")
+ storage.succeed("[ ! -e /var/db/tahoe-lafs/storage/partial ]")
+ storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.3 ]")
diff --git a/nixpkgs-2105.json b/nixpkgs-2105.json
deleted file mode 100644
index 523c1468f35019c6685f3a8486603d0936732dbe..0000000000000000000000000000000000000000
--- a/nixpkgs-2105.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "name": "release2105",
- "url": "https://releases.nixos.org/nixos/21.05/nixos-21.05.4547.2949ed36539/nixexprs.tar.xz",
- "sha256": "0nm5znl7lh3qws29ppzpzsqscyw3hk7q0128xqmga2g86qcmy38x"
-}
\ No newline at end of file
diff --git a/nixpkgs-2105.nix b/nixpkgs-2105.nix
index 536d913b89ba6a57d8d683381ea1c8f40e026b4f..fbd7ca592bfd4e9d1b941f437758f8da1b8bcb5a 100644
--- a/nixpkgs-2105.nix
+++ b/nixpkgs-2105.nix
@@ -1 +1,6 @@
-import (builtins.fetchTarball (builtins.fromJSON (builtins.readFile ./nixpkgs-2105.json)))
+# This actually imports nixos-21.11 but we need to keep this file around so that
+# upgrades work, as the on-node deployment script expects this file in the checkout.
+# See https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/222#note_18600
+# This file can be removed once all nodes have been updated to point to the new file.
+
+import ./nixpkgs.nix
diff --git a/nixpkgs.json b/nixpkgs.json
new file mode 100644
index 0000000000000000000000000000000000000000..3066c2260d204d86ef3b50ef3deac3619cd14145
--- /dev/null
+++ b/nixpkgs.json
@@ -0,0 +1,5 @@
+{
+ "name": "source",
+ "url": "https://releases.nixos.org/nixos/21.11/nixos-21.11.335130.386234e2a61/nixexprs.tar.xz",
+ "sha256": "05lw8w4mbpzxsam09s22q7fm21ayhzh9w8g74vhhhmigr18ggxc7"
+}
diff --git a/nixpkgs.nix b/nixpkgs.nix
new file mode 100644
index 0000000000000000000000000000000000000000..a49c447874e45bea0804185636468568f5bd5035
--- /dev/null
+++ b/nixpkgs.nix
@@ -0,0 +1 @@
+import (builtins.fetchTarball (builtins.fromJSON (builtins.readFile ./nixpkgs.json)))
diff --git a/shell.nix b/shell.nix
index a5741377eec5ebd4b8862a0ea47e15edfdac2731..b8be3a3a6088a987468329ad29919a6957313c6a 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,7 +1,7 @@
let
- release2105 = import ./nixpkgs-2105.nix { };
+ pinned-pkgs = import ./nixpkgs.nix { };
in
-{ pkgs ? release2105, lib ? pkgs.lib, python ? pkgs.python3 }:
+{ pkgs ? pinned-pkgs, lib ? pkgs.lib, python ? pkgs.python3 }:
let
tools = pkgs.callPackage ./tools {};
in
@@ -10,7 +10,7 @@ pkgs.mkShell {
# first adds that path to the store, and then interpolates the store path
# into the string. We use `builtins.toString` to convert the path to a
# string without copying it to the store before interpolating. Either the
- # path is already in the store (e.g. when `pkgs` is `release2105`) so we
+ # path is already in the store (e.g. when `pkgs` is `pinned-pkgs`) so we
# avoid making a second copy with a longer name, or the user passed in local
# path (e.g. a checkout of nixpkgs) and we point at it directly, rather than
# a snapshot of it.
diff --git a/tools/update-nixpkgs b/tools/update-nixpkgs
index 09c823b0a419b5937d4953337b94a26c4b502e32..3c6832c95cee09632318f7a4ef4efc7099e317e1 100755
--- a/tools/update-nixpkgs
+++ b/tools/update-nixpkgs
@@ -10,7 +10,7 @@ from ps_tools import get_url_hash
# We pass this to builtins.fetchTarball which only supports sha256
HASH_TYPE = "sha256"
-DEFAULT_CHANNEL = "nixos-21.05"
+DEFAULT_CHANNEL = "nixos-21.11"
CHANNEL_URL_TEMPLATE = "https://channels.nixos.org/{channel}/nixexprs.tar.xz"
@@ -37,7 +37,7 @@ def main():
"repo_file",
metavar="repo-file",
nargs="?",
- default=Path(__file__).parent.with_name("nixpkgs-2105.json"),
+ default=Path(__file__).parent.with_name("nixpkgs.json"),
type=Path,
help="JSON file with pinned configuration.",
)