diff --git a/DEPLOYMENT-NOTES.rst b/DEPLOYMENT-NOTES.rst index fbde887b5ecfc5f1fa5b520df71a6a0ecc1980fe..4337b43b8cf365ba37c50ced218cb49e42708a27 100644 --- a/DEPLOYMENT-NOTES.rst +++ b/DEPLOYMENT-NOTES.rst @@ -3,11 +3,11 @@ Deployment notes - 2021-12-20 - `https://whetstone.privatestorage.io/privatestorage/privatestorageops/-/issues/399`_ requires moving the PaymentServer database on the ``payments`` host onto a new dedicated filesystem. + `https://whetstone.private.storage/privatestorage/privatestorageops/-/issues/399`_ requires moving the PaymentServer database on the ``payments`` host onto a new dedicated filesystem. Follow these steps *before* deploying this version of PrivateStorageio: - 0. Deploy the `PrivateStorageOps change <https://whetstone.privatestorage.io/privatestorage/privatestorageops/-/merge_requests/169>`_ that creates a new dedicated volume. + 0. Deploy the `PrivateStorageOps change <https://whetstone.private.storage/privatestorage/privatestorageops/-/merge_requests/169>`_ that creates a new dedicated volume. 1. Put a disk label on the new dedicated volume :: @@ -37,9 +37,9 @@ Deployment notes - 2021-10-12 The secret in ``private-keys/grafana-slack-url`` needs to be changed to remove the ``SLACKURL=`` prefix. -- 2021-09-30 `Enable alerting <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/185>`_ needs a secret in ``private-keys/grafana-slack-url`` looking like the template in ``morph/grid/local/private-keys/grafana-slack-url`` and pointing to the secret API endpoint URL saved in `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ (or create a new secret URL at https://www.slack.com/apps/A0F7XDUAZ). +- 2021-09-30 `Enable alerting <https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/185>`_ needs a secret in ``private-keys/grafana-slack-url`` looking like the template in ``morph/grid/local/private-keys/grafana-slack-url`` and pointing to the secret API endpoint URL saved in `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ (or create a new secret URL at https://www.slack.com/apps/A0F7XDUAZ). -- 2021-09-07 `Manage access to payment metrics <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/146>`_ requires moving and chown'ing the PaymentServer database on the ``payments`` host:: +- 2021-09-07 `Manage access to payment metrics <https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/146>`_ requires moving and chown'ing the PaymentServer database on the ``payments`` host:: mkdir /var/lib/zkapissuer diff --git a/README.rst b/README.rst index 3a2b2d8ecfcdba6ffaa4f3f2e275cb85feb709de..46511c81f24136615f0513674f8091aad1201262 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ Project Hosting Moved ===================== -This project can now be found at https://whetstone.privatestorage.io/privatestorage/PrivateStorageio +This project can now be found at https://whetstone.private.storage/privatestorage/PrivateStorageio PrivateStorageio ================ diff --git a/ci-tools/known_hosts.staging b/ci-tools/known_hosts.staging index 2a015656e4c21498f2fe152723888046d9341c1a..1a8e6245e569fdda6daa81cc67c4c8cc33b8e8f8 100644 --- a/ci-tools/known_hosts.staging +++ b/ci-tools/known_hosts.staging @@ -1,3 +1,3 @@ monitoring.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINI9kvEBaOMvpWqcFH+6nFvRriBECKB4RFShdPiIMkk9 payments.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK0eO/01VFwdoZzpclrmu656eaMkE19BaxtDdkkFHMa8 -storage001.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFP8L6OHCxq9XFd8ME8ZrCbmO5dGZDPH8I5dm0AwSGiN +storage001.privatestorage-staging.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA6iWHO9/4s3h9VIpaxgD+rgj/OQh8+jupxBoOmie3St diff --git a/default.nix b/default.nix index 6441675a243e22e6154267c656652c8d8575940e..35bd691f60b222140f66a7db023678a15fbf144d 100644 --- a/default.nix +++ b/default.nix @@ -1,4 +1,4 @@ -{ pkgs ? import ./nixpkgs-2105.nix { } }: +{ pkgs ? import ./nixpkgs.nix { } }: { # Render the project documentation source to some presentation format (ie, # html) with Sphinx. @@ -11,4 +11,9 @@ # Run some unit tests of the Nix that ties all of these things together (ie, # PrivateStorageio-internal library functionality). unit-tests = pkgs.callPackage ./nixos/unit-tests.nix { }; + + # Build all grids into a single derivation. The derivation also has several + # attributes that are useful for exploring the configuration in a repl or + # with eval. + morph = pkgs.callPackage ./morph {}; } diff --git a/docs/dev/README.rst b/docs/dev/README.rst index 29eb38e1f1695084d3276d41d4a063be4a53a015..29300baaba96bac2cc00038c4dfe73f595491731 100644 --- a/docs/dev/README.rst +++ b/docs/dev/README.rst @@ -28,7 +28,7 @@ The system tests boot QEMU VMs which prevents them from running on CI at this ti The build requires > 10 GB of disk space, and the VMs might be timing out on slow or busy machines. If you run into timeouts, -try `raising the number of retries <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/e8233d2/nixos/modules/tests/run-introducer.py#L55-62>`_. +try `raising the number of retries <https://whetstone.private.storage/privatestorage/PrivateStorageio/-/blob/e8233d2/nixos/modules/tests/run-introducer.py#L55-62>`_. It is also possible go through the testing script interactively - useful for debugging:: @@ -36,7 +36,7 @@ It is also possible go through the testing script interactively - useful for deb This will give you a result symlink in the current directory. Inside that is bin/nixos-test-driver which gives you a kind of REPL for interacting with the VMs. -The kind of `Python in this testScript <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/78881a3/nixos/modules/tests/private-storage.nix#L180>`_ is what you can enter into this REPL. +The kind of `Python in this testScript <https://whetstone.private.storage/privatestorage/PrivateStorageio/-/blob/78881a3/nixos/modules/tests/private-storage.nix#L180>`_ is what you can enter into this REPL. Consult the `official documentation on NixOS Tests <https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests>`_ for more information. Updatings Pins diff --git a/morph/README.rst b/morph/README.rst index 96d03eb3cf522af6f1b0065105a2d57ab5c78f6a..bdfe48454529c0ff13f7b30aa275b5a1726a2096 100644 --- a/morph/README.rst +++ b/morph/README.rst @@ -36,11 +36,17 @@ lib --- This contains Nix library code for defining the grids. +It has all the details of how each type of node in our grid is configured. +It knows about morph (so defines ``deployment.secrets`` and has the logic for collecting data defined by other nodes). +It defines options (i.e. ``grid.*``) for things specific to how we configure grids (e.g. ``grid.publicKeyPath``). +It defines metadata about nodes that we use on other nodes (e.g. ``grid.monitoringvpnIPv4`` which is used to define various things on the monitoring node). +Each top-level module here defines one type of node with all (or at least most) of the configuration necessary for that node. grid ---- Specific grid definitions live in subdirectories beneath this directory. +They consist almost exclusively setting options defined in ``morph/lib`` (and few options defined elsewhere) and then delegating to the ``morph/lib`` modules. private-keys ~~~~~~~~~~~~ diff --git a/morph/default.nix b/morph/default.nix new file mode 100644 index 0000000000000000000000000000000000000000..52cf7d107a1b4c9e7a146d35e413ec6e53a3041b --- /dev/null +++ b/morph/default.nix @@ -0,0 +1,32 @@ +{ pkgs ? import ../nixpkgs.nix {} }: +let + lib = pkgs.lib; + gridlib = import ./lib; + inherit (gridlib.pkgs) ourpkgs; + grids-path = "${builtins.toString ./.}/grid"; + grid-configs = lib.mapAttrs (n: v: grids-path + "/${n}/grid.nix") (lib.filterAttrs (n: v: v == "directory") (builtins.readDir grids-path)); + # It would be useful if morph exposed this as a function. + # https://github.com/DBCDK/morph/pull/166 + morph-eval = networkExpr: (import "${pkgs.morph.lib}/eval-machines.nix") { inherit networkExpr; }; + grids = lib.mapAttrs (n: v: (morph-eval v)) grid-configs; + # Derivation with symlinks to the morph output for each grid. + output = pkgs.runCommand "privatestorage-morph" + { preferLocalBuild = true; allowSubstitutes = false; passthru = { inherit gridlib ourpkgs grids; }; } + '' + mkdir $out + ${lib.concatStringsSep "\n" ( + lib.mapAttrsToList ( + name: morph: + let + output = morph.machines { + # It would be nice if we didn't need to write this data to a file. + # https://github.com/DBCDK/morph/pull/186 + argsFile = pkgs.writeText "args" (builtins.toJSON { Names = lib.attrNames morph.nodes; }); + }; + in + '' + ln -s ${output} $out/${lib.escapeShellArg name} + '' + ) grids + )}''; +in output diff --git a/morph/grid/local/Vagrantfile b/morph/grid/local/Vagrantfile index a871cbbe72e410de88a19596ff528391e32ff811..64d4aec5aadc67e48c91cb0b8154b1107c23f1bb 100644 --- a/morph/grid/local/Vagrantfile +++ b/morph/grid/local/Vagrantfile @@ -23,7 +23,12 @@ Vagrant.configure("2") do |config| # v.memory = 4096 # end - config.vm.network "private_network", ip: "192.168.67.21" + # Assign a static IP address inside the VirtualBox host-only (Vagrant + # calls it "private") network. The address must be in the range + # VirtualBox allows. + # https://www.virtualbox.org/manual/ch06.html#network_hostonly says some + # things about this. + config.vm.network "private_network", ip: "192.168.56.21" # Add self signed SSL key for zkap-issuer: config.vm.provision "file", source: "private-keys/payments-localdev-ssl", destination: "/tmp/payments-localdev-ssl" config.vm.provision "shell", inline: "sudo mkdir -p /var/lib/letsencrypt/live/payments.localdev/" @@ -35,7 +40,7 @@ Vagrant.configure("2") do |config| config.vm.box = "esselius/nixos" config.vm.box_version = "20.09" config.vm.box_check_update = false - config.vm.network "private_network", ip: "192.168.67.22" + config.vm.network "private_network", ip: "192.168.56.22" end config.vm.define "storage2.localdev" do |config| @@ -43,7 +48,7 @@ Vagrant.configure("2") do |config| config.vm.box = "esselius/nixos" config.vm.box_version = "20.09" config.vm.box_check_update = false - config.vm.network "private_network", ip: "192.168.67.23" + config.vm.network "private_network", ip: "192.168.56.23" end config.vm.define "monitoring.localdev" do |config| @@ -51,7 +56,7 @@ Vagrant.configure("2") do |config| config.vm.box = "esselius/nixos" config.vm.box_version = "20.09" config.vm.box_check_update = false - config.vm.network "private_network", ip: "192.168.67.24" + config.vm.network "private_network", ip: "192.168.56.24" end # To make the VMs assign the static IPs to the network interfaces we need a rebuild: @@ -60,7 +65,7 @@ Vagrant.configure("2") do |config| config.trigger.after :up do |trigger| trigger.info = "Hostname and IP address this host actually uses:" - trigger.run_remote = {inline: "echo `hostname` `ifconfig | egrep -o '192.168.67.[0-9]* '`"} + trigger.run_remote = {inline: "echo `hostname` `ifconfig | egrep -o '192.168.56.[0-9]* '`"} end end diff --git a/morph/grid/local/config.json b/morph/grid/local/config.json index 8bd686a023b704688c8708b2408d0c3df8287f13..52809842c8877b2e9c5c87a9239d37c61f1b8896 100644 --- a/morph/grid/local/config.json +++ b/morph/grid/local/config.json @@ -2,7 +2,7 @@ , "publicStoragePort": 8898 , "publicKeyPath": "./public-keys" , "privateKeyPath": "./private-keys" -, "monitoringvpnEndpoint": "192.168.67.24:51820" +, "monitoringvpnEndpoint": "192.168.56.24:51820" , "passValue": 1000000 , "issuerDomains": ["payments.localdev"] , "monitoringDomains": ["monitoring.localdev"] diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix index 4a1524c6b6b7f5e085766aec6a79af5b569e72ba..da8a83812ceba910280bfc61210487b2f217113f 100644 --- a/morph/grid/local/grid.nix +++ b/morph/grid/local/grid.nix @@ -59,6 +59,7 @@ let grid = { publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}"; privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}"; + inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail; }; # Configure deployment management authorization for all systems in the grid. services.private-storage.deployment = { @@ -70,74 +71,66 @@ let payments = { imports = [ gridlib.issuer - (gridlib.customize-issuer (grid-config // { - monitoringvpnIPv4 = "172.23.23.11"; - })) grid-module ]; config = { - grid.publicIPv4 = "192.168.67.21"; + grid.monitoringvpnIPv4 = "172.23.23.11"; + grid.publicIPv4 = "192.168.56.21"; + grid.issuer = { + inherit (grid-config) issuerDomains allowedChargeOrigins; + }; }; }; storage1 = { imports = [ gridlib.storage - (gridlib.customize-storage (grid-config // { - monitoringvpnIPv4 = "172.23.23.12"; - stateVersion = "19.09"; - })) grid-module ]; config = { - grid.publicIPv4 = "192.168.67.22"; + grid.monitoringvpnIPv4 = "172.23.23.12"; + grid.publicIPv4 = "192.168.56.22"; + grid.storage = { + inherit (grid-config) passValue publicStoragePort; + }; + system.stateVersion = "19.09"; }; }; storage2 = { imports = [ gridlib.storage - (gridlib.customize-storage (grid-config // { - monitoringvpnIPv4 = "172.23.23.13"; - stateVersion = "19.09"; - })) grid-module ]; config = { - grid.publicIPv4 = "192.168.67.23"; + grid.monitoringvpnIPv4 = "172.23.23.13"; + grid.publicIPv4 = "192.168.56.23"; + grid.storage = { + inherit (grid-config) passValue publicStoragePort; + }; + system.stateVersion = "19.09"; }; }; monitoring = { imports = [ gridlib.monitoring - (gridlib.customize-monitoring { - inherit hostsMap vpnClientIPs - nodeExporterTargets - paymentExporterTargets - blackboxExporterHttpsTargets; - inherit (grid-config) letsEncryptAdminEmail monitoringDomains; - googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; - enableSlackAlert = false; - monitoringvpnIPv4 = "172.23.23.1"; - stateVersion = "19.09"; - }) grid-module ]; config = { - grid.publicIPv4 = "192.168.67.24"; + grid.monitoringvpnIPv4 = "172.23.23.1"; + grid.publicIPv4 = "192.168.56.24"; + grid.monitoring = { + inherit paymentExporterTargets blackboxExporterHttpsTargets; + inherit (grid-config) monitoringDomains; + googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; + enableSlackAlert = false; + }; + system.stateVersion = "19.09"; }; }; # TBD: derive these automatically: - hostsMap = { - "172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ]; - "172.23.23.11" = [ "payments" "payments.monitoringvpn" ]; - "172.23.23.12" = [ "storage1" "storage1.monitoringvpn" ]; - "172.23.23.13" = [ "storage2" "storage2.monitoringvpn" ]; - }; - vpnClientIPs = [ "172.23.23.11" "172.23.23.12" "172.23.23.13" ]; - nodeExporterTargets = [ "monitoring" "payments" "storage1" "storage2" ]; paymentExporterTargets = [ "payments" ]; blackboxExporterHttpsTargets = [ # "https://private.storage/" diff --git a/morph/grid/local/public-keys/users.nix.example b/morph/grid/local/public-keys/users.nix.example index 10a60be1f7b8760e81f7fdb6ecd1d177913e05af..4e4794de770437fb14e666d7b538a3d481c38eb7 100644 --- a/morph/grid/local/public-keys/users.nix.example +++ b/morph/grid/local/public-keys/users.nix.example @@ -1,6 +1,11 @@ +let # Add your public key. Example: -# let key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la"; +# key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la"; # You can use the following to get key from the local machine. -# let key = builtins.readFile ~/.ssh/id_ed25519.pub; -let key = undefined; -in { "root" = key; "vagrant" = key; } +# key = builtins.readFile ~/.ssh/id_ed25519.pub; + key = undefined; + keys = [key] +in { + "root" = keys; + "vagrant" = keys; +} diff --git a/morph/grid/production/config.json b/morph/grid/production/config.json index 1696b5fb3c45df94b8bf69aae9ca323e6bac2266..8cdeaab993fd894783953e7c8f51cd9ea3bed96d 100644 --- a/morph/grid/production/config.json +++ b/morph/grid/production/config.json @@ -5,8 +5,8 @@ , "monitoringvpnEndpoint": "monitoring.private.storage:51820" , "passValue": 1000000 , "issuerDomains": [ - "payments.privatestorage.io" - , "payments.private.storage" + "payments.private.storage" + , "payments.privatestorage.io" ] , "monitoringDomains": [ "monitoring.privatestorage.io" @@ -14,10 +14,7 @@ ] , "letsEncryptAdminEmail": "jean-paul@privatestorage.io" , "allowedChargeOrigins": [ - "https://privatestorage.io" - , "https://www.privatestorage.io" - , "https://private.storage" - , "https://www.private.storage" + "https://private.storage" ] , "monitoringGoogleOAuthClientID": "802959152038-klpkk38sfnqmknn1ucg7pvs4hcc2k8ae.apps.googleusercontent.com" } diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix index 950282f5573560f76355bcdcf4d6da51dacedd7d..ab45d4ba7f67e71383d28120bd925ac3a05f04ef 100644 --- a/morph/grid/production/grid.nix +++ b/morph/grid/production/grid.nix @@ -21,6 +21,7 @@ let grid = { publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}"; privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}"; + inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail; }; # Configure deployment management authorization for all systems in the grid. services.private-storage.deployment = { @@ -33,30 +34,32 @@ let imports = [ gridlib.issuer gridlib.hardware-aws - (gridlib.customize-issuer (grid-config // { - monitoringvpnIPv4 = "172.23.23.11"; - })) grid-module ]; + config = { + grid.monitoringvpnIPv4 = "172.23.23.11"; + grid.issuer = { + inherit (grid-config) issuerDomains allowedChargeOrigins; + }; + }; }; monitoring = { imports = [ gridlib.monitoring gridlib.hardware-aws - (gridlib.customize-monitoring { - inherit hostsMap vpnClientIPs - nodeExporterTargets - paymentExporterTargets - blackboxExporterHttpsTargets; - inherit (grid-config) letsEncryptAdminEmail monitoringDomains; - googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; - enableSlackAlert = true; - monitoringvpnIPv4 = "172.23.23.1"; - stateVersion = "19.09"; - }) grid-module ]; + config = { + grid.monitoringvpnIPv4 = "172.23.23.1"; + grid.monitoring = { + inherit paymentExporterTargets blackboxExporterHttpsTargets; + inherit (grid-config) monitoringDomains; + googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; + enableSlackAlert = true; + }; + system.stateVersion = "19.09"; + }; }; defineStorageNode = name: { vpnIP, stateVersion }: @@ -79,26 +82,27 @@ let # Get all of the configuration that is common across all storage nodes. gridlib.storage - - # Then customize the storage system a little bit based on this node's particulars. - (gridlib.customize-storage (grid-config // nodecfg // { - monitoringvpnIPv4 = vpnIP; - inherit stateVersion; - })) - # Also configure deployment management authorization grid-module ]; - # And supply configuration for those hardware / network / bootloader - # options. See the 100tb module for handling of this value. The module - # name is quoted because `1` makes `100tb` look an awful lot like a - # number. - "100tb".config = nodecfg; + config = { + grid.monitoringvpnIPv4 = vpnIP; + grid.storage = { + inherit (grid-config) passValue publicStoragePort; + }; + system.stateVersion = stateVersion; + + # And supply configuration for those hardware / network / bootloader + # options. See the 100tb module for handling of this value. The module + # name is quoted because `1` makes `100tb` look an awful lot like a + # number. + "100tb".config = nodecfg; - # Enable statistics gathering for MegaRAID cards. - # TODO would be nice to enable only on machines that have such a device. - services.private-storage.monitoring.megacli2prom.enable = true; + # Enable statistics gathering for MegaRAID cards. + # TODO would be nice to enable only on machines that have such a device. + services.private-storage.monitoring.megacli2prom.enable = true; + }; }; # Define all of the storage nodes for this grid. @@ -110,33 +114,6 @@ let storage005 = { vpnIP = "172.23.23.25"; stateVersion = "19.03"; }; }; - # TBD: derive these automatically: - hostsMap = { - "172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ]; - "172.23.23.11" = [ "payments" "payments.monitoringvpn" ]; - "172.23.23.21" = [ "storage001" "storage001.monitoringvpn" ]; - "172.23.23.22" = [ "storage002" "storage002.monitoringvpn" ]; - "172.23.23.23" = [ "storage003" "storage003.monitoringvpn" ]; - "172.23.23.24" = [ "storage004" "storage004.monitoringvpn" ]; - "172.23.23.25" = [ "storage005" "storage005.monitoringvpn" ]; - }; - vpnClientIPs = [ - "172.23.23.11" - "172.23.23.21" - "172.23.23.22" - "172.23.23.23" - "172.23.23.24" - "172.23.23.25" - ]; - nodeExporterTargets = [ - "monitoring" - "payments" - "storage001" - "storage002" - "storage003" - "storage004" - "storage005" - ]; paymentExporterTargets = [ "payments" ]; blackboxExporterHttpsTargets = [ "https://private.storage/" diff --git a/morph/grid/production/public-keys/users.nix b/morph/grid/production/public-keys/users.nix index 8b586703740765b7a3d462e74ca3ef3cced68da7..9dcc90ea0efb3c927915d441e77c9af2459303e4 100644 --- a/morph/grid/production/public-keys/users.nix +++ b/morph/grid/production/public-keys/users.nix @@ -1,2 +1,6 @@ -let key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGN4VQm3BIQKEFTw6aPrEwNuShf640N+Py2LOKznFCRT exarkun@bottom"; -in { "root" = key; "jcalderone" = key; } +let + jcalderone = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGN4VQm3BIQKEFTw6aPrEwNuShf640N+Py2LOKznFCRT exarkun@bottom"]; +in { + "root" = jcalderone; + "jcalderone" = jcalderone; +} diff --git a/morph/grid/testing/config.json b/morph/grid/testing/config.json index 7c3775df55ce76cf6048712e644a3f2669b6f07c..ba48a27deea9d35150b1834727b659e4972bd2e5 100644 --- a/morph/grid/testing/config.json +++ b/morph/grid/testing/config.json @@ -16,7 +16,6 @@ , "allowedChargeOrigins": [ "http://localhost:5000" , "https://privatestorage-staging.com" - , "https://www.privatestorage-staging.com" ] , "monitoringGoogleOAuthClientID": "802959152038-6esn1c6u2lm3j82lf29jvmn8s63hi8dc.apps.googleusercontent.com" } diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix index 334518774851c22738c93b323223f255d871a394..19839ae83fa16c31adf0fcd9e3727a8304f8dd6c 100644 --- a/morph/grid/testing/grid.nix +++ b/morph/grid/testing/grid.nix @@ -21,6 +21,7 @@ let grid = { publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}"; privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}"; + inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail; }; # Configure deployment management authorization for all systems in the grid. services.private-storage.deployment = { @@ -33,11 +34,14 @@ let imports = [ gridlib.issuer gridlib.hardware-aws - (gridlib.customize-issuer (grid-config // { - monitoringvpnIPv4 = "172.23.23.11"; - })) grid-module ]; + config = { + grid.monitoringvpnIPv4 = "172.23.23.11"; + grid.issuer = { + inherit (grid-config) issuerDomains allowedChargeOrigins; + }; + }; }; storage001 = { @@ -45,41 +49,36 @@ let gridlib.storage gridlib.hardware-aws ./testing001-hardware.nix - (gridlib.customize-storage (grid-config // { - monitoringvpnIPv4 = "172.23.23.12"; - stateVersion = "19.03"; - })) grid-module ]; + config = { + grid.monitoringvpnIPv4 = "172.23.23.12"; + grid.storage = { + inherit (grid-config) passValue publicStoragePort; + }; + system.stateVersion = "19.03"; + }; }; monitoring = { imports = [ gridlib.monitoring gridlib.hardware-aws - (gridlib.customize-monitoring { - inherit hostsMap vpnClientIPs - nodeExporterTargets - paymentExporterTargets - blackboxExporterHttpsTargets; - inherit (grid-config) letsEncryptAdminEmail monitoringDomains; - googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; - enableSlackAlert = true; - monitoringvpnIPv4 = "172.23.23.1"; - stateVersion = "19.09"; - }) grid-module ]; + config = { + grid.monitoringvpnIPv4 = "172.23.23.1"; + grid.monitoring = { + inherit paymentExporterTargets blackboxExporterHttpsTargets; + inherit (grid-config) monitoringDomains; + googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; + enableSlackAlert = true; + }; + system.stateVersion = "19.09"; + }; }; # TBD: derive these automatically: - hostsMap = { - "172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ]; - "172.23.23.11" = [ "payments" "payments.monitoringvpn" ]; - "172.23.23.12" = [ "storage001" "storage001.monitoringvpn" ]; - }; - vpnClientIPs = [ "172.23.23.11" "172.23.23.12" ]; - nodeExporterTargets = [ "monitoring" "payments" "storage001" ]; paymentExporterTargets = [ "payments" ]; blackboxExporterHttpsTargets = [ "https://privatestorage-staging.com/" diff --git a/morph/grid/testing/public-keys/users.nix b/morph/grid/testing/public-keys/users.nix index d6a965011065cfe39713adfb797c190eb8dd1ecd..14647efb7d04d39f8201c03b542191f7e86f35c2 100644 --- a/morph/grid/testing/public-keys/users.nix +++ b/morph/grid/testing/public-keys/users.nix @@ -1,6 +1,6 @@ let - jcalderone = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon"; - flo = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la"; + jcalderone = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon"]; + flo = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la"]; in { "root" = jcalderone; diff --git a/morph/lib/base.nix b/morph/lib/base.nix index 7390654ac167909149b0a6f4dfae897b8f3f43a3..71d4dfb57c8723a384a3dfb9d08a6069914de3d2 100644 --- a/morph/lib/base.nix +++ b/morph/lib/base.nix @@ -18,6 +18,26 @@ corresponding private keys for the system. ''; }; + monitoringvpnIPv4 = lib.mkOption { + type = lib.types.str; + description = '' + The IPv4 address of this node on the monitoring VPN. + ''; + }; + monitoringvpnEndpoint = lib.mkOption { + type = lib.types.str; + description = '' + The domain name and port of the monitoring VPN endpoint. + ''; + }; + + letsEncryptAdminEmail = lib.mkOption { + type = lib.types.str; + description = '' + A string giving an email address to use for Let's Encrypt registration and + certificate issuance. + ''; + }; }; # Any extra NixOS modules to load on all our servers. Note that just @@ -37,6 +57,8 @@ # qualified domain name. deployment.targetHost = config.networking.fqdn; + services.private-storage.monitoring.exporters.promtail.enable = true; + assertions = [ # This is a check to save somebody in the future trying to debug why # setting `nixpkgs.config` is not having an effect. @@ -45,7 +67,7 @@ assertion = config.nixpkgs.config == {}; message = '' Since we set `nixpkgs.pkgs` via morph's `network.pkgs`, the value for `nixpkgs.config` is ignored. - See https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/issues/85#note_15876 for details. + See https://whetstone.private.storage/privatestorage/PrivateStorageio/-/issues/85#note_15876 for details. ''; } ]; diff --git a/morph/lib/bootstrap-configuration.nix b/morph/lib/bootstrap-configuration.nix index 531f867572f3bd46963fc850384f6280f11531a1..f59385d1fd0ec21997d21c16be5ed5e937611acc 100644 --- a/morph/lib/bootstrap-configuration.nix +++ b/morph/lib/bootstrap-configuration.nix @@ -67,7 +67,7 @@ let # Stop! I hope you're done when you get here. If you have to modify # anything below this point the expression should probably be refactored and # another variable added controlling whatever new thing you need to control. - # Open an issue: https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/issues/new?issue + # Open an issue: https://whetstone.private.storage/privatestorage/PrivateStorageio/-/issues/new?issue in # Define a function that ignores all its arguments. We don't need any of them # for now. diff --git a/morph/lib/customize-issuer.nix b/morph/lib/customize-issuer.nix deleted file mode 100644 index 0686556cdf6abe79f0ac9e16586c9c219f3cddb1..0000000000000000000000000000000000000000 --- a/morph/lib/customize-issuer.nix +++ /dev/null @@ -1,53 +0,0 @@ -# Define a function which returns a value which fills in all the holes left by -# ``issuer.nix``. -{ - # A string giving the IP address and port number (":"-separated) of the VPN - # server. - monitoringvpnEndpoint - - # A string giving the VPN IPv4 address for this system. -, monitoringvpnIPv4 - - # A string giving an email address to use for Let's Encrypt registration and - # certificate issuance. -, letsEncryptAdminEmail - - # A list of strings giving the domain names that point at this issuer - # system. These will all be included in Let's Encrypt certificate. -, issuerDomains - - # A list of strings giving CORS Origins will the issuer will be configured - # to allow. -, allowedChargeOrigins -, ... -}: -{ config, ... }: -let - inherit (config.grid) publicKeyPath privateKeyPath; -in { - deployment.secrets = { - # ``.../monitoringvpn`` is a path on the deployment system of a directory - # containing a number of VPN-related secrets. This is expected to contain - # a number of files named like ``<VPN IPv4 address>.key`` containing the - # VPN private key for the corresponding host. It must also contain - # ``server.pub`` and ``preshared.key`` holding the VPN server's public key - # and the pre-shared key, respectively. All of these things are used as - # the sources of various VPN-related morph secrets. - "monitoringvpn-secret-key".source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key"; - "monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key"; - }; - - services.private-storage.monitoring.vpn.client = { - enable = true; - ip = monitoringvpnIPv4; - endpoint = monitoringvpnEndpoint; - endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub"; - }; - - services.private-storage-issuer = { - inherit letsEncryptAdminEmail allowedChargeOrigins; - domains = issuerDomains; - }; - - system.stateVersion = "19.03"; -} diff --git a/morph/lib/customize-monitoring.nix b/morph/lib/customize-monitoring.nix deleted file mode 100644 index 2899d9940d4309b81a31f96590f0d3df1d632dc4..0000000000000000000000000000000000000000 --- a/morph/lib/customize-monitoring.nix +++ /dev/null @@ -1,127 +0,0 @@ -# Define a function which returns a value which fills in all the holes left by -# ``monitoring.nix``. -{ - # A set mapping VPN IP addresses as strings to lists of hostnames as - # strings. The system's ``/etc/hosts`` will be populated with this - # information. Apart from helping with normal forward resolution, this - # *also* gives us reverse resolution from the VPN IPs to hostnames which - # allows Grafana to show us hostnames instead of VPN IP addresses. - hostsMap - - # See ``customize-issuer.nix``. -, monitoringvpnIPv4 -, letsEncryptAdminEmail -, monitoringDomains - - # A list of VPN IP addresses as strings indicating which clients will be - # allowed onto the VPN. -, vpnClientIPs - - # A list of VPN clients (IP addresses or hostnames) as strings indicating - # which nodes to scrape "nodeExporter" metrics from. -, nodeExporterTargets - - # A list of VPN clients (IP addresses or hostnames) as strings indicating - # which nodes to scrape "nginxExporter" metrics from. -, nginxExporterTargets ? [] - - # A list of VPN clients (IP addresses or hostnames) as strings indicating - # which nodes to scrape PaymentServer metrics from. -, paymentExporterTargets ? [] - - # A list of HTTPS servers (URLs, IP addresses or hostnames) as strings indicating - # which nodes the BlackboxExporter should scrape HTTP and TLS metrics from. -, blackboxExporterHttpsTargets ? [] - - # A string containing the GSuite OAuth2 ClientID to use to authenticate - # logins to Grafana. -, googleOAuthClientID - - # Whether to enable alerting via Slack. - # When true requires a grafana-slack-url file (see private-keys/README.rst). -, enableSlackAlert ? false - - # A string giving the NixOS state version for the system. -, stateVersion -, ... -}: -{ config, ... }: -let - inherit (config.grid) publicKeyPath privateKeyPath; -in { - deployment.secrets = let - # When Grafana SSO is disabled there is not necessarily any client secret - # available. Avoid telling morph that there is one in this case (so it - # avoids trying to read it and then failing). Even if the secret did - # exist, if SSO is disabled there's no point sending the secret to the - # server. - # - # Also, we have to define this whole secret here so that we can configure - # it completely or not at all. morph gets angry if we half configure it - # (say, by just omitting the "source" value). - grafanaSSO = - if googleOAuthClientID == "" - then { } - else { - "grafana-google-sso-secret" = { - source = "${privateKeyPath}/grafana-google-sso.secret"; - destination = "/run/keys/grafana-google-sso.secret"; - owner.user = config.systemd.services.grafana.serviceConfig.User; - owner.group = config.users.users.grafana.group; - permissions = "0400"; - action = ["sudo" "systemctl" "restart" "grafana.service"]; - }; - "grafana-admin-password" = { - source = "${privateKeyPath}/grafana-admin.password"; - destination = "/run/keys/grafana-admin.password"; - owner.user = config.systemd.services.grafana.serviceConfig.User; - owner.group = config.users.users.grafana.group; - permissions = "0400"; - action = ["sudo" "systemctl" "restart" "grafana.service"]; - }; - }; - grafanaSlackUrl = - if !enableSlackAlert - then { } - else { - "grafana-slack-url" = { - source = "${privateKeyPath}/grafana-slack-url"; - destination = "/run/keys/grafana-slack-url"; - owner.user = config.systemd.services.grafana.serviceConfig.User; - owner.group = config.users.users.grafana.group; - permissions = "0400"; - action = ["sudo" "systemctl" "restart" "grafana.service"]; - }; - }; - monitoringvpn = { - "monitoringvpn-private-key".source = "${privateKeyPath}/monitoringvpn/server.key"; - "monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key"; - }; - in - grafanaSSO // grafanaSlackUrl // monitoringvpn; - - networking.hosts = hostsMap; - - services.private-storage.monitoring.vpn.server = { - enable = true; - ip = monitoringvpnIPv4; - inherit vpnClientIPs; - pubKeysPath = "${publicKeyPath}/monitoringvpn"; - }; - - services.private-storage.monitoring.prometheus = { - inherit nodeExporterTargets; - inherit nginxExporterTargets; - inherit paymentExporterTargets; - inherit blackboxExporterHttpsTargets; - }; - - services.private-storage.monitoring.grafana = { - inherit letsEncryptAdminEmail; - inherit googleOAuthClientID; - inherit enableSlackAlert; - domains = monitoringDomains; - }; - - system.stateVersion = stateVersion; -} diff --git a/morph/lib/customize-storage.nix b/morph/lib/customize-storage.nix deleted file mode 100644 index 6a288213c3f117309b697e44304be9a7d5620bcb..0000000000000000000000000000000000000000 --- a/morph/lib/customize-storage.nix +++ /dev/null @@ -1,40 +0,0 @@ -# Define a function which returns a value which fills in all the holes left by -# ``storage.nix``. -{ - # See ``customize-issuer.nix`` - monitoringvpnEndpoint -, monitoringvpnIPv4 - - # An integer giving the value of a single pass in byte×months. -, passValue - - # An integer giving the port number to include in Tahoe storage service - # advertisements and on which to listen for storage connections. -, publicStoragePort - - # A string giving the NixOS state version for the system. -, stateVersion -, ... -}: -{ config, ... }: -let - inherit (config.grid) publicKeyPath privateKeyPath; -in { - deployment.secrets = { - "monitoringvpn-secret-key".source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key"; - "monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key"; - }; - - services.private-storage = { - inherit passValue publicStoragePort; - }; - - services.private-storage.monitoring.vpn.client = { - enable = true; - ip = monitoringvpnIPv4; - endpoint = monitoringvpnEndpoint; - endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub"; - }; - - system.stateVersion = stateVersion; -} diff --git a/morph/lib/default.nix b/morph/lib/default.nix index a820cc559b6b2da78c06bcb84282e392c3a1ebc7..f236b8cada99b71cd1c5ab851f3c081421c4b717 100644 --- a/morph/lib/default.nix +++ b/morph/lib/default.nix @@ -8,13 +8,8 @@ hardware-vagrant = import ./hardware-vagrant.nix; issuer = import ./issuer.nix; - customize-issuer = import ./customize-issuer.nix; - storage = import ./storage.nix; - customize-storage = import ./customize-storage.nix; - monitoring = import ./monitoring.nix; - customize-monitoring = import ./customize-monitoring.nix; modules = builtins.toString ../../nixos/modules; @@ -22,7 +17,7 @@ # installed, as well as the NixOS module set that is used. # This is intended to be used in a grid definition like: # network = { ... ; inherit (gridlib) pkgs; ... } - pkgs = import ../../nixpkgs-2105.nix { + pkgs = import ../../nixpkgs.nix { # Ensure that configuration of the system where this runs # doesn't leak into what we build. # See https://github.com/NixOS/nixpkgs/issues/62513 @@ -31,6 +26,13 @@ "megacli" ]; }; - overlays = []; + # Expose `nixos/pkgs` as an attribute of our package set. + # This is is primarly consumed by `nixos/modules/packages.nix`, which + # then exposes it as a module argument. We do this here, so that + # the package set only needs to be evaluted once for the grid, rather + # than once for each host. + overlays = [ + (self: super: { ourpkgs = self.callPackage ../../nixos/pkgs {}; }) + ]; }; } diff --git a/morph/lib/issuer-aws.nix b/morph/lib/issuer-aws.nix index 8ff172803eda784898aba2d96636df1afcee36e5..80495e2dc7bafbc9bfbbe174e1a2f75f66942dfe 100644 --- a/morph/lib/issuer-aws.nix +++ b/morph/lib/issuer-aws.nix @@ -32,4 +32,8 @@ dates = "weekly"; options = "--delete-older-than 30d"; }; + + # Turn on automatic optimization of nix store + # https://nixos.wiki/wiki/Storage_optimization + nix.autoOptimiseStore = true; } diff --git a/morph/lib/issuer.nix b/morph/lib/issuer.nix index d3ee812e865f741b01eb811589262ae01ece824f..69b0527cd74e0752ded6ffbe7513db126f0613f5 100644 --- a/morph/lib/issuer.nix +++ b/morph/lib/issuer.nix @@ -1,60 +1,91 @@ -# This, along with `customize-issuer.nix, contains all of the NixOS system -# configuration necessary to specify an "issuer"-type system. Originally, this -# file has all the static configuration, and `customize-issuer.nix` was a function -# that filled in the holes. We are in the process of merging the modules, using settings -# instead of function arguments. -# See https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/issues/80 -{ config, ...}: +# This contains all of the NixOS system configuration necessary to specify an +# "issuer"-type system. +{ lib, config, ...}: let - inherit (config.grid) publicKeyPath privateKeyPath; + inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnEndpoint monitoringvpnIPv4; + inherit (config.grid.issuer) issuerDomains allowedChargeOrigins; in { - deployment = { - secrets = { - "ristretto-signing-key" = { - destination = "/run/keys/ristretto.signing-key"; - source = "${privateKeyPath}/ristretto.signing-key"; - owner.user = "zkapissuer"; - owner.group = "zkapissuer"; - permissions = "0400"; - action = ["sudo" "systemctl" "restart" "zkapissuer.service"]; - }; - "stripe-secret-key" = { - destination = "/run/keys/stripe.secret-key"; - source = "${privateKeyPath}/stripe.secret"; - owner.user = "zkapissuer"; - owner.group = "zkapissuer"; - permissions = "0400"; - action = ["sudo" "systemctl" "restart" "zkapissuer.service"]; - }; - - "monitoringvpn-secret-key" = { - destination = "/run/keys/monitoringvpn/client.key"; - owner.user = "root"; - owner.group = "root"; - permissions = "0400"; - action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; - }; - "monitoringvpn-preshared-key" = { - destination = "/run/keys/monitoringvpn/preshared.key"; - owner.user = "root"; - owner.group = "root"; - permissions = "0400"; - action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; - }; - }; - }; - imports = [ ../../nixos/modules/monitoring/vpn/client.nix ../../nixos/modules/monitoring/exporters/node.nix ]; - services.private-storage-issuer = { - enable = true; - tls = true; - ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination; - stripeSecretKeyPath = config.deployment.secrets.stripe-secret-key.destination; - database = "SQLite3"; - databasePath = "${config.fileSystems."zkapissuer-data".mountPoint}/vouchers.sqlite3"; + options.grid.issuer = { + issuerDomains = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = '' + A list of strings giving the domain names that point at this issuer + system. These will all be included in Let's Encrypt certificate. + ''; + }; + + allowedChargeOrigins = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = '' + A list of strings giving CORS Origins will the issuer will be configured + to allow. + ''; + }; + }; + + config = { + deployment = { + secrets = { + "ristretto-signing-key" = { + destination = "/run/keys/ristretto.signing-key"; + source = "${privateKeyPath}/ristretto.signing-key"; + owner.user = "zkapissuer"; + owner.group = "zkapissuer"; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "zkapissuer.service"]; + }; + "stripe-secret-key" = { + destination = "/run/keys/stripe.secret-key"; + source = "${privateKeyPath}/stripe.secret"; + owner.user = "zkapissuer"; + owner.group = "zkapissuer"; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "zkapissuer.service"]; + }; + + "monitoringvpn-secret-key" = { + destination = "/run/keys/monitoringvpn/client.key"; + source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key"; + owner.user = "root"; + owner.group = "root"; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; + }; + "monitoringvpn-preshared-key" = { + destination = "/run/keys/monitoringvpn/preshared.key"; + source = "${privateKeyPath}/monitoringvpn/preshared.key"; + owner.user = "root"; + owner.group = "root"; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; + }; + }; + }; + + services.private-storage-issuer = { + enable = true; + tls = true; + ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination; + stripeSecretKeyPath = config.deployment.secrets.stripe-secret-key.destination; + database = "SQLite3"; + databasePath = "${config.fileSystems."zkapissuer-data".mountPoint}/vouchers.sqlite3"; + inherit (config.grid) letsEncryptAdminEmail; + inherit allowedChargeOrigins; + domains = issuerDomains; + }; + + services.private-storage.monitoring.vpn.client = { + enable = true; + ip = monitoringvpnIPv4; + endpoint = monitoringvpnEndpoint; + endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub"; + }; + + system.stateVersion = "19.03"; }; } diff --git a/morph/lib/monitoring.nix b/morph/lib/monitoring.nix index 89a328e89a799b445dff7180dff552350b9629cf..d299d62ae7997511897517f9574e33c6de94b7a5 100644 --- a/morph/lib/monitoring.nix +++ b/morph/lib/monitoring.nix @@ -1,32 +1,165 @@ -# Similar to ``issuer.nix`` but for a "monitoring"-type system. Holes are -# filled by ``customize-monitoring.nix``. -{ - deployment = { - secrets = { - "monitoringvpn-private-key" = { +# This contains all of the NixOS system configuration necessary to specify an +# "monitoring"-type system. +{ lib, config, nodes, ...}: +let + cfg = config.grid.monitoring; + inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4 letsEncryptAdminEmail; + + # This collects information about monitored hosts from their configuration for use below. + monitoringHosts = lib.mapAttrsToList (name: node: rec { + inherit name; + vpnIPv4 = node.config.grid.monitoringvpnIPv4; + vpnHostName = "${name}.monitoringvpn"; + hostNames = [name vpnHostName]; + }) nodes; + + # A set mapping VPN IP addresses as strings to lists of hostnames as + # strings. The system's ``/etc/hosts`` will be populated with this + # information. Apart from helping with normal forward resolution, this + # *also* gives us reverse resolution from the VPN IPs to hostnames which + # allows Grafana to show us hostnames instead of VPN IP addresses. + hostsMap = lib.listToAttrs (map (node: lib.nameValuePair node.vpnIPv4 node.hostNames) monitoringHosts); + # A list of VPN IP addresses as strings indicating which clients will be + # allowed onto the VPN. + vpnClientIPs = lib.remove monitoringvpnIPv4 (map (node: node.vpnIPv4) monitoringHosts); + # A list of VPN clients (IP addresses or hostnames) as strings indicating + # which nodes to scrape "nodeExporter" metrics from. + nodeExporterTargets = map (node: node.vpnHostName) monitoringHosts; +in { + imports = [ + ../../nixos/modules/monitoring/vpn/server.nix + ../../nixos/modules/monitoring/server/grafana.nix + ../../nixos/modules/monitoring/server/prometheus.nix + ../../nixos/modules/monitoring/server/loki.nix + ../../nixos/modules/monitoring/exporters/node.nix + ../../nixos/modules/monitoring/exporters/blackbox.nix + ]; + + options.grid.monitoring = { + paymentExporterTargets = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = '' + A list of VPN clients (IP addresses or hostnames) as strings indicating + which nodes to scrape PaymentServer metrics from. + ''; + }; + + blackboxExporterHttpsTargets = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = '' + A list of HTTPS servers (URLs, IP addresses or hostnames) as strings indicating + which nodes the BlackboxExporter should scrape HTTP and TLS metrics from. + ''; + }; + + monitoringDomains = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = '' + A list of strings giving the domain names that point at this monitoring + system. These will all be included in Let's Encrypt certificate. + ''; + }; + + + googleOAuthClientID = lib.mkOption { + type = lib.types.str; + default = ""; + description = '' + A string containing the GSuite OAuth2 ClientID to use to authenticate + logins to Grafana. + ''; + }; + + enableSlackAlert = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to enable alerting via Slack. + When true requires a grafana-slack-url file (see private-keys/README.rst). + ''; + }; + }; + + config = { + assertions = [ + { + assertion = let + vpnIPs = (map (node: node.vpnIPv4) monitoringHosts); + in vpnIPs == lib.unique vpnIPs; + message = '' + Duplicate grid.monitoringvpnIPv4 values specified for different nodes. + ''; + } + ]; + + deployment.secrets = lib.mkMerge [ + { + "monitoringvpn-private-key" = { destination = "/run/keys/monitoringvpn/server.key"; + source = "${privateKeyPath}/monitoringvpn/server.key"; owner.user = "root"; owner.group = "root"; permissions = "0400"; action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; - }; - "monitoringvpn-preshared-key" = { + }; + "monitoringvpn-preshared-key" = { destination = "/run/keys/monitoringvpn/preshared.key"; + source = "${privateKeyPath}/monitoringvpn/preshared.key"; owner.user = "root"; owner.group = "root"; permissions = "0400"; action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; - }; + }; + "grafana-admin-password" = { + source = "${privateKeyPath}/grafana-admin.password"; + destination = "/run/keys/grafana-admin.password"; + owner.user = config.systemd.services.grafana.serviceConfig.User; + owner.group = config.users.users.grafana.group; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "grafana.service"]; + }; + } + (lib.mkIf (cfg.googleOAuthClientID != "") { + "grafana-google-sso-secret" = { + source = "${privateKeyPath}/grafana-google-sso.secret"; + destination = "/run/keys/grafana-google-sso.secret"; + owner.user = config.systemd.services.grafana.serviceConfig.User; + owner.group = config.users.users.grafana.group; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "grafana.service"]; + }; + }) + (lib.mkIf cfg.enableSlackAlert { + "grafana-slack-url" = { + source = "${privateKeyPath}/grafana-slack-url"; + destination = "/run/keys/grafana-slack-url"; + owner.user = config.systemd.services.grafana.serviceConfig.User; + owner.group = config.users.users.grafana.group; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "grafana.service"]; + }; + }) + ]; + + networking.hosts = hostsMap; + + services.private-storage.monitoring.vpn.server = { + enable = true; + ip = monitoringvpnIPv4; + inherit vpnClientIPs; + pubKeysPath = "${publicKeyPath}/monitoringvpn"; }; - }; - imports = [ - ../../nixos/modules/monitoring/vpn/server.nix - ../../nixos/modules/monitoring/server/grafana.nix - ../../nixos/modules/monitoring/server/prometheus.nix - ../../nixos/modules/monitoring/exporters/node.nix - ../../nixos/modules/monitoring/exporters/blackbox.nix - # Loki 0.3.0 from Nixpkgs 19.09 is too old and does not work: - # ../../nixos/modules/monitoring/server/loki.nix - ]; + services.private-storage.monitoring.prometheus = { + inherit nodeExporterTargets; + inherit (cfg) paymentExporterTargets blackboxExporterHttpsTargets; + nginxExporterTargets = []; + }; + + services.private-storage.monitoring.grafana = { + inherit (cfg) googleOAuthClientID enableSlackAlert ; + inherit letsEncryptAdminEmail; + domains = cfg.monitoringDomains; + }; + }; } diff --git a/morph/lib/storage.nix b/morph/lib/storage.nix index 15e2373737a7ff2f1efe8cf2c41b59de606f0a1a..71e3c22371ad042c4ddbc5d8cd87db5cb05923af 100644 --- a/morph/lib/storage.nix +++ b/morph/lib/storage.nix @@ -1,39 +1,9 @@ -# Similar to ``issuer.nix`` but for a "storage"-type system. Holes are filled -# by ``customize-storage.nix``. -{ config, ...} : +# This contains all of the NixOS system configuration necessary to specify an +# "storage"-type system. +{ lib, config, ...} : let - inherit (config.grid) publicKeyPath privateKeyPath; + inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4 monitoringvpnEndpoint; in { - deployment = { - secrets = { - "ristretto-signing-key" = { - destination = "/run/keys/ristretto.signing-key"; - source = "${privateKeyPath}/ristretto.signing-key"; - owner.user = "root"; - owner.group = "root"; - permissions = "0400"; - # Service name here matches the name defined by our tahoe-lafs nixos - # module. It would be nice to not have to hard-code it here. Can we - # extract it from the tahoe-lafs nixos module somehow? - action = ["sudo" "systemctl" "restart" "tahoe.storage.service"]; - }; - "monitoringvpn-secret-key" = { - destination = "/run/keys/monitoringvpn/client.key"; - owner.user = "root"; - owner.group = "root"; - permissions = "0400"; - action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; - }; - "monitoringvpn-preshared-key" = { - destination = "/run/keys/monitoringvpn/preshared.key"; - owner.user = "root"; - owner.group = "root"; - permissions = "0400"; - action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; - }; - }; - }; - # Any extra NixOS modules to load on this server. imports = [ # Bring in our module for configuring the Tahoe-LAFS service and other @@ -47,13 +17,72 @@ in { ../../nixos/modules/monitoring/exporters/tahoe.nix ]; - services.private-storage.monitoring.tahoe.enable = true; + options.grid.storage = { + passValue = lib.mkOption { + type = lib.types.int; + description = '' + An integer giving the value of a single pass in byte×months. + ''; + }; - # Turn on the Private Storage (Tahoe-LAFS) service. - services.private-storage = { - # Yep. Turn it on. - enable = true; - # Give it the Ristretto signing key to support authorization. - ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination; + publicStoragePort = lib.mkOption { + type = lib.types.port; + description = '' + An integer giving the port number to include in Tahoe storage service + advertisements and on which to listen for storage connections. + ''; + }; + }; + + config = { + deployment = { + secrets = { + "ristretto-signing-key" = { + destination = "/run/keys/ristretto.signing-key"; + source = "${privateKeyPath}/ristretto.signing-key"; + owner.user = "root"; + owner.group = "root"; + permissions = "0400"; + # Service name here matches the name defined by our tahoe-lafs nixos + # module. It would be nice to not have to hard-code it here. Can we + # extract it from the tahoe-lafs nixos module somehow? + action = ["sudo" "systemctl" "restart" "tahoe.storage.service"]; + }; + "monitoringvpn-secret-key" = { + destination = "/run/keys/monitoringvpn/client.key"; + source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key"; + owner.user = "root"; + owner.group = "root"; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; + }; + "monitoringvpn-preshared-key" = { + destination = "/run/keys/monitoringvpn/preshared.key"; + source = "${privateKeyPath}/monitoringvpn/preshared.key"; + owner.user = "root"; + owner.group = "root"; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; + }; + }; + }; + + services.private-storage.monitoring.tahoe.enable = true; + + # Turn on the Private Storage (Tahoe-LAFS) service. + services.private-storage = { + # Yep. Turn it on. + enable = true; + # Give it the Ristretto signing key to support authorization. + ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination; + inherit (config.grid.storage) passValue publicStoragePort; + }; + + services.private-storage.monitoring.vpn.client = { + enable = true; + ip = monitoringvpnIPv4; + endpoint = monitoringvpnEndpoint; + endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub"; + }; }; } diff --git a/nixos/lib/default.nix b/nixos/lib/default.nix new file mode 100644 index 0000000000000000000000000000000000000000..3ebaf60b7d536589d46a745d2945cbef96b2b554 --- /dev/null +++ b/nixos/lib/default.nix @@ -0,0 +1,6 @@ +{ callPackage }: +{ + /* A library of tools useful for writing tests with Nix. + */ + testing = callPackage ./testing.nix { }; +} diff --git a/nixos/lib/testing.nix b/nixos/lib/testing.nix new file mode 100644 index 0000000000000000000000000000000000000000..d89717a0a76f93bb6062ad63c6cfdbb91c12c746 --- /dev/null +++ b/nixos/lib/testing.nix @@ -0,0 +1,23 @@ +{ ...}: +{ + /* Returns a string that runs tests from the Python code at the given path. + + The Python code is loaded using *execfile* and the *test* global it + defines is called with the given keyword arguments. + + Type: makeTestScript :: Path -> AttrSet -> String + + Example: + testScript = (makeTestScript ./test_foo.py { x = "y"; }); + */ + makeTestScript = { testpath, kwargs ? {} }: + '' + # The driver runs pyflakes on this script before letting it + # run... Convince pyflakes that there is a `test` name. + test = None + with open("${testpath}") as testfile: + exec(testfile.read(), globals()) + # For simple types, JSON is compatible with Python syntax! + test(**${builtins.toJSON kwargs}) + ''; +} diff --git a/nixos/modules/README.rst b/nixos/modules/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..b395eace08655ade8b52262dbdf3bf62664d1c66 --- /dev/null +++ b/nixos/modules/README.rst @@ -0,0 +1,5 @@ +These are mostly modelled on upstream nixos modules. +They are generally fairly configurable (they don't tend to hard-code paths, they can be enabled or disabled). +They don't know anything about morph (e.g. ``deployment.secrets``) or how the different grids are configured (e.g. ``grid.publicKeyPath``). +Each module here tends to define one service (or group of related services) or feature. +Eventually, all of these will be imported automatically and controlled by ``services.private-storage.*.enabled`` options. diff --git a/nixos/modules/default.nix b/nixos/modules/default.nix index 1772d399639aa8b4ec2c9ac6a218c4dd8a6169da..f7e247f99406ad982c3b1e59d8248e2c80a3a658 100644 --- a/nixos/modules/default.nix +++ b/nixos/modules/default.nix @@ -12,5 +12,6 @@ imports = [ ./packages.nix ./issuer.nix + ./monitoring/exporters/promtail.nix ]; } diff --git a/nixos/modules/monitoring/exporters/megacli2prom.nix b/nixos/modules/monitoring/exporters/megacli2prom.nix index a38f1ccc18b59073ff835e50babeb565f79a20b8..c1f10aef7821343e2f0a572d006782858dcf7ef4 100644 --- a/nixos/modules/monitoring/exporters/megacli2prom.nix +++ b/nixos/modules/monitoring/exporters/megacli2prom.nix @@ -35,11 +35,10 @@ in { config = lib.mkIf cfg.enable { - environment.systemPackages = [ ourpkgs.megacli2prom ]; + environment.systemPackages = [ ourpkgs.megacli2prom pkgs.megacli ]; systemd.services.megacli2prom = { enable = true; description = "MegaCli2Prom metrics gathering service"; - wantedBy = [ "multi-user.target" ]; startAt = cfg.interval; path = [ pkgs.megacli ]; # Save to a temp file and then move atomically so the diff --git a/nixos/modules/monitoring/exporters/promtail.nix b/nixos/modules/monitoring/exporters/promtail.nix new file mode 100644 index 0000000000000000000000000000000000000000..83de3250af5f02635ec5c790eedf445b1e38a92e --- /dev/null +++ b/nixos/modules/monitoring/exporters/promtail.nix @@ -0,0 +1,65 @@ +# Promtail log forwarder configuration +# +# Scope: Tail logs on the local system and send them to Loki +# +# Description: This is not strictly an "exporter" like the Prometheus +# exporters, but it is very similar in what it is doing - +# preparing local data and sending it off to a TSDB. + +{ config, options, lib, ... }: + +let + cfg = config.services.private-storage.monitoring.exporters.promtail; + hostName = config.networking.hostName; + +in { + options.services.private-storage.monitoring.exporters.promtail = { + enable = lib.mkEnableOption "Promtail log exporter service"; + lokiUrl = lib.mkOption { + type = lib.types.str; + description = '' + The server URL that logs should be pushed to. + ''; + # Resolving names is hard, let's have breakfast + # If you are curious why there's a plain IP address in here, read all of + # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/251 + # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/257 + # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/258 + default = "http://172.23.23.1:3100/loki/api/v1/push"; + }; + }; + + config = lib.mkIf cfg.enable { + services.promtail.enable = true; + networking.firewall.interfaces.monitoringvpn.allowedTCPPorts = [ 9080 ]; + services.promtail.configuration = { + server = { + http_listen_port = 9080; # Using /metrics for health check + grpc_listen_address = "127.0.0.1"; # unused, but no option to turn it off. + grpc_listen_port = 9094; # unused, but no option to turn it off. + }; + + clients = [{ + url = cfg.lokiUrl; + }]; + + scrape_configs = [{ + job_name = "systemd-journal"; + journal = { + labels = { + job = "systemd-journal"; + host = hostName; + }; + }; + # The journal has many internal labels, that by default will + # be dropped because of their "__" prefix. To keep them, rename them. + # https://grafana.com/docs/loki/latest/clients/promtail/scraping/#journal-scraping-linux-only + # https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html + relabel_configs = [{ + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + }]; + }]; + }; + }; +} diff --git a/nixos/modules/monitoring/server/grafana-dashboards/payments.json b/nixos/modules/monitoring/server/grafana-dashboards/payments.json index 7d6f6bb12bae5c1401b1199a8b2831b39a4ba955..6541c9796059523aff679dc34f451db2feeed85d 100644 --- a/nixos/modules/monitoring/server/grafana-dashboards/payments.json +++ b/nixos/modules/monitoring/server/grafana-dashboards/payments.json @@ -8,19 +8,25 @@ "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, "description": "PaymentServer and related metrics", "editable": true, - "gnetId": null, - "graphTooltip": 0, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, "links": [], + "liveNow": false, "panels": [ { "collapsed": false, - "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -33,53 +39,107 @@ "type": "row" }, { - "aliasColors": { - "Attempts": "yellow", - "Successes": "green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, "description": "Our calls to the Stripe API: Attempted and successful credit card charges.", "fieldConfig": { - "defaults": {}, - "overrides": [] + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Attempts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Successes" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 1 }, - "hiddenSeries": false, "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.4", "targets": [ { "exemplar": true, @@ -97,107 +157,97 @@ "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Stripe", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:350", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:351", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": { - "Redeemed vouchers": "yellow" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, "description": "", "fieldConfig": { - "defaults": {}, - "overrides": [] + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Redeemed vouchers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 1 }, - "hiddenSeries": false, "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:223", - "alias": "Redeemed vouchers", - "yaxis": 1 + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - { - "$$hashKey": "object:230", - "alias": "Issued signatures", - "yaxis": 2 + "tooltip": { + "mode": "single" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "8.3.4", "targets": [ { "exemplar": true, @@ -216,53 +266,11 @@ "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Redemption", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:285", - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:286", - "decimals": null, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { "collapsed": false, - "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -275,52 +283,78 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, "description": "HTTPS responses per second", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 9 }, - "hiddenSeries": false, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.4", "targets": [ { "exemplar": true, @@ -331,96 +365,83 @@ "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Requests per second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:452", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:453", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, "description": "", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 9 }, - "hiddenSeries": false, "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.4", "targets": [ { "exemplar": true, @@ -437,103 +458,144 @@ "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:576", - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "$$hashKey": "object:577", - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": { - "=< 0.1s": "blue", - "=< 1s": "green", - "=< 5s": "yellow", - "> 5s": "orange" - }, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": null, "description": "Request durations, stacked", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "=< 0.1s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "=< 1s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "=< 5s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "> 5s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] }, - "fill": 2, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 9 }, - "hiddenSeries": false, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.4", "targets": [ { "exemplar": true, @@ -580,52 +642,11 @@ "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Durations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:625", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:626", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { "collapsed": false, - "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -638,52 +659,78 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, "description": "HTTPS responses per second", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 17 }, - "hiddenSeries": false, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.4", "targets": [ { "expr": "rate(http_responses_total{path=\"v1/redeem\"}[5m])", @@ -693,95 +740,86 @@ "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Requests per second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:751", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:752", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, + "description": "HTTP 4xx and 5xx errors", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 17 }, - "hiddenSeries": false, "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.4", "targets": [ { "expr": "sum(http_responses_total{path=\"v1/redeem\", status=\"4XX\"}) / sum(http_responses_total{path=\"v1/redeem\"})", @@ -794,103 +832,144 @@ "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:804", - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "$$hashKey": "object:805", - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "type": "timeseries" }, { - "aliasColors": { - "=< 0.1s": "blue", - "=< 1s": "green", - "=< 5s": "yellow", - "> 5s": "orange" - }, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": null, "description": "Request durations, stacked.", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "=< 0.1s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "=< 1s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "=< 5s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "> 5s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] }, - "fill": 2, - "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 17 }, - "hiddenSeries": false, "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.4", "targets": [ { "exemplar": true, @@ -937,59 +1016,160 @@ "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, "title": "Durations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:853", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 26, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "000000002" + }, + "description": "Exercise in counting maybe interesting lines. This can be alerted on.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "000000002" + }, + "expr": "count_over_time({host=\"payments\",unit=\"zkapissuer.service\"} !~ \"200|/metrics|Accept\" [5m])", + "refId": "A" + } + ], + "thresholds": [], + "title": "Number of maybe interesting log lines", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "000000002" + }, + "description": "Exercise in filtering the payment server logs.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 30, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ { - "$$hashKey": "object:854", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "datasource": { + "type": "loki", + "uid": "000000002" + }, + "expr": "{host=\"payments\",unit=\"zkapissuer.service\"} !~ \"200|/metrics|Accept\"", + "refId": "A" } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Maybe interesting lines", + "type": "logs" } ], "refresh": "5m", - "schemaVersion": 27, + "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { - "from": "now-3h", + "from": "now-7d", "to": "now" }, "timepicker": { @@ -1009,5 +1189,6 @@ "timezone": "", "title": "Payments", "uid": "Payments", - "version": 1 + "version": 1, + "weekStart": "" } diff --git a/nixos/modules/monitoring/server/grafana.nix b/nixos/modules/monitoring/server/grafana.nix index d7efd4c7d3f92d0120444374aa5250d68d4764a8..0923885f86d9bcebc4d3df590c71fbcddd8d1df8 100644 --- a/nixos/modules/monitoring/server/grafana.nix +++ b/nixos/modules/monitoring/server/grafana.nix @@ -183,6 +183,17 @@ in { proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}"; proxyWebsockets = true; }; + locations."/metrics" = { + # Only allow our monitoringvpn subnet + # And localhost since we're the monitoring server currently + extraConfig = '' + allow ${config.grid.monitoringvpnIPv4}/24; + allow 127.0.0.1; + allow ::1; + deny all; + ''; + proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}"; + }; }; }; diff --git a/nixos/modules/monitoring/server/loki.nix b/nixos/modules/monitoring/server/loki.nix index 96554523f06d0d86c620db445b2443575a1c3fd3..491d1a4c5edd1100ea17c26bbe8e8799b9424582 100644 --- a/nixos/modules/monitoring/server/loki.nix +++ b/nixos/modules/monitoring/server/loki.nix @@ -1,9 +1,14 @@ # Loki Server # -# Scope: Log aggregator +# Scope: Log ingester and aggregator to be run on the monitoring node +# +# See also: +# - The configuration is adapted from +# https://grafana.com/docs/loki/latest/configuration/examples/#complete-local-configyaml +# { - config.networking.firewall.allowedTCPPorts = [ 3100 ]; + config.networking.firewall.interfaces.monitoringvpn.allowedTCPPorts = [ 3100 ]; config.services.loki = { enable = true; @@ -14,11 +19,12 @@ server = { http_listen_port = 3100; + grpc_listen_port = 9095; # unused, but no option to turn it off. + grpc_listen_address = "127.0.0.1"; # unused, but no option to turn it off. }; ingester = { lifecycler = { - address = "0.0.0.0"; ring = { kvstore = { store = "inmemory"; @@ -27,50 +33,35 @@ }; final_sleep = "0s"; }; - chunk_idle_period = "1h"; # Any chunk not receiving new logs in this time will be flushed - max_chunk_age = "1h"; # All chunks will be flushed when they hit this age, default is 1h - chunk_target_size = 1048576; # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first - chunk_retain_period = "30s"; # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + chunk_target_size = 1536000; # As per https://grafana.com/docs/loki/v2.2.1/best-practices/ max_transfer_retries = 0; # Chunk transfers disabled }; schema_config = { configs = [{ - from = "2020-10-24"; # TODO: Should this be "today"? - store = "boltdb-shipper"; + from = "2020-12-26"; + store = "boltdb"; object_store = "filesystem"; schema = "v11"; index = { prefix = "index_"; - period = "24h"; }; }]; }; storage_config = { - boltdb_shipper = { - active_index_directory = "/var/lib/loki/boltdb-shipper-active"; - cache_location = "/var/lib/loki/boltdb-shipper-cache"; - cache_ttl = "24h"; # Can be increased for faster performance over longer query periods, uses more disk space - shared_store = "filesystem"; + boltdb = { + directory = "/var/lib/loki/index"; }; + filesystem = { directory = "/var/lib/loki/chunks"; }; }; - limits_config = { - reject_old_samples = true; - reject_old_samples_max_age = "168h"; - }; - - chunk_store_config = { - max_look_back_period = "336h"; - }; - table_manager = { retention_deletes_enabled = true; - retention_period = "336h"; + retention_period = "336h"; # two weeks }; }; }; diff --git a/nixos/modules/monitoring/server/prometheus.nix b/nixos/modules/monitoring/server/prometheus.nix index 3bb00a5b95855859e455b5df8fb065b3d70bc855..2a78dd3e797c0b28d14fc9e9e0858811ac86ef76 100644 --- a/nixos/modules/monitoring/server/prometheus.nix +++ b/nixos/modules/monitoring/server/prometheus.nix @@ -10,7 +10,7 @@ let cfg = config.services.private-storage.monitoring.prometheus; dropPortNumber = { source_labels = [ "__address__" ]; - regex = "^(.*):\\d+$"; + regex = "^(.*)(?:\\.monitoringvpn):\\d+$"; target_label = "instance"; }; diff --git a/nixos/modules/packages.nix b/nixos/modules/packages.nix index c4390dc00f3948e04e3e90ef270261cc0dd1cdbb..5879e6d63e6e69c2517127dea94cc058ef9ce76a 100644 --- a/nixos/modules/packages.nix +++ b/nixos/modules/packages.nix @@ -1,7 +1,9 @@ # A NixOS module which exposes custom packages to other modules. { pkgs, ...}: let - ourpkgs = pkgs.callPackage ../../nixos/pkgs {}; + # Get our custom packages; either from the nixpkgs attribute added via an + # overlay in `morph/lib/default.nix`, or by importing them directly. + ourpkgs = pkgs.ourpkgs or (pkgs.callPackage ../pkgs {}); in { config = { # Expose `nixos/pkgs` as a new module argument `ourpkgs`. diff --git a/nixos/modules/ssh.nix b/nixos/modules/ssh.nix index eb55fbf2ee4d3e6c04dd08039a8a9f9012f069b8..8d5d5766ae3b30c4801b6ce200fa58c1460f6ca7 100644 --- a/nixos/modules/ssh.nix +++ b/nixos/modules/ssh.nix @@ -6,7 +6,7 @@ }: { options = { services.private-storage.sshUsers = lib.mkOption { - type = lib.types.attrsOf lib.types.str; + type = lib.types.attrsOf (lib.types.listOf lib.types.str); example = { root = "ssh-ed25519 AAA..."; }; description = '' Users to configure on the issuer server and the storage servers and @@ -44,9 +44,9 @@ }; users.users = - let makeUserConfig = username: sshPublicKey: { + let makeUserConfig = username: sshPublicKeys: { isNormalUser = username != "root"; - openssh.authorizedKeys.keys = [ sshPublicKey ]; + openssh.authorizedKeys.keys = sshPublicKeys; }; in builtins.mapAttrs makeUserConfig cfg.sshUsers; }; diff --git a/nixos/modules/tahoe.nix b/nixos/modules/tahoe.nix index e0b6eb4d8be3c5359de1d391c42b2ba83f7a1ba4..44c381e6b6dfa6039d1dd6a49d44f1afaf51ab10 100644 --- a/nixos/modules/tahoe.nix +++ b/nixos/modules/tahoe.nix @@ -156,6 +156,10 @@ in nameValuePair "tahoe.introducer-${node}" { description = "Tahoe node user for introducer ${node}"; isSystemUser = true; + group = "tahoe.introducer-${node}"; + }); + users.groups = flip mapAttrs' cfg.introducers (node: _: + nameValuePair "tahoe.introducer-${node}" { }); }) (mkIf (cfg.nodes != {}) { @@ -287,6 +291,10 @@ in nameValuePair "tahoe.${node}" { description = "Tahoe node user for node ${node}"; isSystemUser = true; + group = "tahoe.${node}"; + }); + users.groups = flip mapAttrs' cfg.introducers (node: _: + nameValuePair "tahoe.${node}" { }); }) ]; diff --git a/nixos/modules/update-deployment b/nixos/modules/update-deployment index 1c8960588f418e57eeaadb7ad29db4285369cbdd..a0d233a63595a9f838f48243b14ef98fa79a240d 100755 --- a/nixos/modules/update-deployment +++ b/nixos/modules/update-deployment @@ -32,7 +32,7 @@ CHECKOUT="${HOME}/PrivateStorageio" # This is the address of the git remote where we can get the latest # PrivateStorageio. -REPO="https://whetstone.privatestorage.io/privatestorage/PrivateStorageio.git" +REPO="https://whetstone.private.storage/privatestorage/PrivateStorageio.git" if [ -e "${CHECKOUT}" ]; then # It exists already so just make sure it contains the latest changes from @@ -72,14 +72,14 @@ EOF ssh -o StrictHostKeyChecking=no "$(hostname).$(domainname)" ":" # Set nixpkgs to our preferred version for the morph build. Annoyingly, we -# can't just use nixpkgs-2105.nix as our nixpkgs because some code (in morph, +# can't just use nixpkgs.nix as our nixpkgs because some code (in morph, # at least) wants <nixpkgs> to be a fully-resolved path to a nixpkgs tree. # For example, morph evaluated `import <nixpkgs/lib>` which would turn into -# something like `import nixpkgs-2105.nix/lib` which is nonsense. +# something like `import nixpkgs.nix/lib` which is nonsense. # # So instead, import our nixpkgs which forces it to be instantiated in the # store, then ask for its path, then set NIX_PATH to that. -export NIX_PATH="nixpkgs=$(nix eval "(import ${CHECKOUT}/nixpkgs-2105.nix { }).path")" +export NIX_PATH="nixpkgs=$(nix eval "(import ${CHECKOUT}/nixpkgs.nix { }).path")" # Attempt to update just this host. Choose the morph grid definition matching # the grid we belong to and limit the morph deployment update to the host diff --git a/nixos/pkgs/default.nix b/nixos/pkgs/default.nix index bfc30b36101c220434606832127a7e8ca0a70490..435095f7890b7ac41afaebe050a756c4b4887641 100644 --- a/nixos/pkgs/default.nix +++ b/nixos/pkgs/default.nix @@ -5,6 +5,8 @@ # pkgs.callPackage ./nixos/pkgs {buildPlatform, hostPlatform, callPackage}: { + lib = callPackage ../lib {}; + leasereport = callPackage ./leasereport {}; # `privatestorage` is a derivation with a good Tahoe+ZKAP environment # that is exposed by ZKAPAuthorizer. diff --git a/nixos/pkgs/leasereport/repo.json b/nixos/pkgs/leasereport/repo.json index 759814a124d0a4bab23411bebd8de19f5f021060..fc2f8947264a08f3234e8a9372fb1b3da4c79feb 100644 --- a/nixos/pkgs/leasereport/repo.json +++ b/nixos/pkgs/leasereport/repo.json @@ -2,7 +2,7 @@ "owner": "privatestorage", "repo": "LeaseReport", "branch": "main", - "domain": "whetstone.privatestorage.io", + "domain": "whetstone.private.storage", "rev": "3739ffde14e698f56118a444e6946edb736b6983", "outputHashAlgo": "sha512", "outputHash": "37b4hrhjghvza0bqvmngcdapqfjjjiv0gx90y0i4wvj72nf1xsh7g2kwpvjm4prpb5s7fxb50x971xfw4sqpwwsk2zdll4nbl5764ij" diff --git a/nixos/pkgs/megacli2prom/repo.json b/nixos/pkgs/megacli2prom/repo.json index 3c8cd0af95adf95e22def4e727b8c2c5d12044aa..daa4c7b8a0a7dba8ca382c9e1913e2c7ff2d36cd 100644 --- a/nixos/pkgs/megacli2prom/repo.json +++ b/nixos/pkgs/megacli2prom/repo.json @@ -2,7 +2,7 @@ "owner": "PrivateStorageio", "repo": "megacli2prom", "branch": "main", - "rev": "9536933d325c843b2662f80486660bf81d73941e", + "rev": "e76300ca0a723bf0ed105d805f166976162d58d3", "outputHashAlgo": "sha512", - "outputHash": "1xrsv0bkmazbhqarx84lhvmrzzdv1bm04xvr0hw1yrw1f4xb450f4pwgapnkjczy0l4c6rp3pmh64cblgbs3ki30wacbv1bqzv5745g" + "outputHash": "256di1f4bw5a0kqm37wr5dk9yg0cxhgqaflrhk0p3azimml3pd1gr4rh54mj4vsrw17iyziajmilx98fsvc9w70y14rh7kgxcam9vwp" } \ No newline at end of file diff --git a/nixos/pkgs/privatestorage/default.nix b/nixos/pkgs/privatestorage/default.nix index bd487af32941f6db920ea2d43ec89e9eded38201..3bbbd3dbcf0b974e6e1997e20773cddbd9ea59c0 100644 --- a/nixos/pkgs/privatestorage/default.nix +++ b/nixos/pkgs/privatestorage/default.nix @@ -2,7 +2,7 @@ let repo-data = lib.importJSON ./repo.json; repo = fetchFromGitHub (builtins.removeAttrs repo-data [ "branch" ]); - privatestorage = callPackage repo {}; + privatestorage = callPackage repo { python = "python39"; }; in privatestorage.privatestorage diff --git a/nixos/pkgs/privatestorage/repo.json b/nixos/pkgs/privatestorage/repo.json index 81f6e18ba4bbec657a5a5ba543ef05408bf472ad..4113e150a72c907dce4ee0d7345361eadb248041 100644 --- a/nixos/pkgs/privatestorage/repo.json +++ b/nixos/pkgs/privatestorage/repo.json @@ -2,7 +2,7 @@ "owner": "PrivateStorageio", "branch": "main", "repo": "ZKAPAuthorizer", - "rev": "b61f3d4a3f5eb72cb600dd83796a1aaca2931e07", + "rev": "744a063ab76a677b259aa9022711113ffbab2545", "outputHashAlgo": "sha512", - "outputHash": "2d7a9m34jx1k38fmiwskgwd1ryyhrb56m9nam12fd66shl8qzmlfcr1lwf063qi1wqdzb2g7998vxbv3c2bmvw7g6iqwzjmsck2czpn" + "outputHash": "293j4469iy69d2hz3gwxwyj0flqb1cncl938s5w5jmfgbvkm1w0yfg1y06nx89zis1rvwqpcly3vxp94pz1dx28d74wiianqks11p54" } \ No newline at end of file diff --git a/nixos/pkgs/zkap-spending-service/repo.json b/nixos/pkgs/zkap-spending-service/repo.json index 69f7a30053de661f2c7829384e9496e49077cfd9..cc6fac7b5b950ea4ccccf83335b16812ba2b3146 100644 --- a/nixos/pkgs/zkap-spending-service/repo.json +++ b/nixos/pkgs/zkap-spending-service/repo.json @@ -1,9 +1,9 @@ -{ - "owner": "privatestorage", - "repo": "zkap-spending-service", - "rev": "cbf7509f429ffd6e6cf37a73e4ff84a9c5ce1141", - "branch": "main", - "domain": "whetstone.privatestorage.io", - "outputHash": "04g7pcykc2525cg3z7wg5834s7vqn82xaqjvf52l6dnxv3mb9xr93kk505dvxcwhgfbqpim5i479s9kqd8gi7q3lq5wn5fq7rf7lkrj", - "outputHashAlgo": "sha512" -} +{ + "owner": "privatestorage", + "repo": "zkap-spending-service", + "rev": "66fd395268b466d4c7bb0a740fb758a5acccd1c4", + "branch": "main", + "domain": "whetstone.private.storage", + "outputHash": "1nryvsccncrka25kzrwqkif4x68ib0cs2vbw1ngfmzw86gjgqx01a7acgspmrpfs62p4q8zw0f2ynl8jr3ygyypjrl8v7w8g49y0y0y", + "outputHashAlgo": "sha512" +} diff --git a/nixos/pkgs/zkapissuer/repo.json b/nixos/pkgs/zkapissuer/repo.json index 0a003dc61620fd92b1a618e9845763e276c9693a..98ecb9ff70785d8ebd338d6a5e17fe19b8bfebd8 100644 --- a/nixos/pkgs/zkapissuer/repo.json +++ b/nixos/pkgs/zkapissuer/repo.json @@ -1,8 +1,8 @@ -{ - "owner": "PrivateStorageio", - "repo": "PaymentServer", - "rev": "e080beb14ec58ffe8e55c35e6dddd46c5082887f", - "branch": "main", - "outputHashAlgo": "sha256", - "outputHash": "1zck9kawbs2lkr3qjipira9gawa4gxlqijqqjrmlvvyp9mr0fgxm" -} +{ + "owner": "PrivateStorageio", + "repo": "PaymentServer", + "rev": "47478f705332b23219285e9598a69668f2c79aa1", + "branch": "main", + "outputHashAlgo": "sha512", + "outputHash": "3z62dfkyivb0l8yc1l1qm31k8sl8i88m9pzrk9nhs42kmgcqyr7sa10lavj499w9l6zvh1628ss0g5pza5yaji537r1bc51qqfszydl" +} \ No newline at end of file diff --git a/nixos/system-tests.nix b/nixos/system-tests.nix index 218132fe2cd3857f4c201085b4df56a411c794d4..819b5c738eca08b95d3c85b14088a2bf6c000dbf 100644 --- a/nixos/system-tests.nix +++ b/nixos/system-tests.nix @@ -1,7 +1,11 @@ # The overall system test suite for PrivateStorageio NixOS configuration. { pkgs }: -{ - private-storage = pkgs.nixosTest ./tests/private-storage.nix; - spending = pkgs.nixosTest ./tests/spending.nix; - tahoe = pkgs.nixosTest ./tests/tahoe.nix; +let + # Add custom packages as an attribute, so it they only need to be evalutated once. + # See the comment in `morph/lib/default.nix` for details. + pkgs' = pkgs.extend (self: super: { ourpkgs = self.callPackage ./pkgs {}; }); +in { + private-storage = pkgs'.nixosTest ./tests/private-storage.nix; + spending = pkgs'.nixosTest ./tests/spending.nix; + tahoe = pkgs'.nixosTest ./tests/tahoe.nix; } diff --git a/nixos/tests/private-storage.nix b/nixos/tests/private-storage.nix index a208ce249f1f1227f966e38a1c62ab6166d187f8..b17b8f32ed494c0823f349166f9a28f1e3dcb876 100644 --- a/nixos/tests/private-storage.nix +++ b/nixos/tests/private-storage.nix @@ -1,25 +1,14 @@ { pkgs }: let - sshPrivateKey = ./probeuser_ed25519; - sshPublicKey = ./probeuser_ed25519.pub; + ourpkgs = pkgs.callPackage ../pkgs { }; + + sshPrivateKeyFile = ./probeuser_ed25519; + sshPublicKeyFile = ./probeuser_ed25519.pub; + sshUsers = { - root = (builtins.readFile sshPublicKey); - probeuser = (builtins.readFile sshPublicKey); + root = [(builtins.readFile sshPublicKeyFile)]; + probeuser = [(builtins.readFile sshPublicKeyFile)]; }; - # Generate a command which can be used with runOnNode to ssh to the given - # host. - ssh = username: hostname: [ - "cp" sshPrivateKey "/tmp/ssh_key" ";" - "chmod" "0400" "/tmp/ssh_key" ";" - "ssh" "-oStrictHostKeyChecking=no" "-i" "/tmp/ssh_key" "${username}@${hostname}" ":" - ]; - - # Separate helper programs so we can write as little python inside a string - # inside a nix expression as possible. - run-introducer = ./run-introducer.py; - run-client = ./run-client.py; - get-passes = ./get-passes.py; - exercise-storage = ./exercise-storage.py; # This is a test double of the Stripe API server. It is extremely simple. # It barely knows how to respond to exactly the API endpoints we use, @@ -72,18 +61,6 @@ let networking.firewall.enable = false; networking.dhcpcd.enable = false; }; - - # Return a python program fragment to run a shell command on one of the nodes. - # The first argument is the name of the node. The second is a list of the - # argv to run. - # - # The program's output is piped to systemd-cat and the python fragment - # evaluates to success if the command exits with a success status. - runOnNode = node: argv: - let - command = builtins.concatStringsSep " " argv; - in - "${node}.succeed('set -eo pipefail; ${command} | systemd-cat')"; in { # https://nixos.org/nixos/manual/index.html#sec-nixos-tests # https://nixos.mayflower.consulting/blog/2019/07/11/leveraging-nixos-tests-in-your-project/ @@ -177,134 +154,16 @@ in { }; # Test the machines with a Python program. - testScript = '' - # Boot the VMs. We used to do them all in parallel but the boot - # sequence got flaky at some point for some reason I don't - # understand. :/ It might be related to this: - # - # https://discourse.nixos.org/t/nixos-ppc64le-vm-does-not-have-dev-vda-device/11548/9 - # - # See <nixpkgs/nixos/modules/virtualisation/qemu-vm.nix> for the Nix - # that constructs the QEMU command that gets run. - # - # Boot them one at a time for now. - issuer.connect() - introducer.connect() - storage.connect() - client.connect() - api_stripe_com.connect() - - # The issuer and the storage server should accept SSH connections. This - # doesn't prove it is so but if it fails it's a pretty good indication - # it isn't so. - storage.wait_for_open_port(22) - ${runOnNode "issuer" (ssh "probeuser" "storage")} - ${runOnNode "issuer" (ssh "root" "storage")} - issuer.wait_for_open_port(22) - ${runOnNode "storage" (ssh "probeuser" "issuer")} - ${runOnNode "storage" (ssh "root" "issuer")} - - # Set up a Tahoe-LAFS introducer. - introducer.copy_from_host('${pemFile}', '/tmp/node.pem') - - try: - ${runOnNode "introducer" [ run-introducer "/tmp/node.pem" (toString introducerPort) introducerFURL ]} - except: - code, output = introducer.execute('cat /tmp/stdout /tmp/stderr') - introducer.log(output) - raise - - # - # Get a Tahoe-LAFS storage server up. - # - code, version = storage.execute('tahoe --version') - storage.log(version) - - # The systemd unit should reach the running state. - storage.wait_for_unit('tahoe.storage.service') - - # Some while after that the Tahoe-LAFS node should listen on the web API - # port. The port number here has to agree with the port number set in - # the private-storage.nix module. - storage.wait_for_open_port(3456) - - # Once the web API is listening it should be possible to scrape some - # status from the node if it is really working. - storage.succeed('tahoe -d /var/db/tahoe-lafs/storage status') - - # It should have Eliot logging turned on as well. - storage.succeed('[ -e /var/db/tahoe-lafs/storage/logs/eliot.json ]') - - # - # Storage appears to be working so try to get a client to speak with it. - # - ${runOnNode "client" [ run-client "/tmp/client" introducerFURL issuerURL ristrettoPublicKey ]} - client.wait_for_open_port(3456) - - # Make sure the fake Stripe API server is ready for requests. - try: - api_stripe_com.wait_for_unit("api.stripe.com") - except: - code, output = api_stripe_com.execute('journalctl -u api.stripe.com') - api_stripe_com.log(output) - raise - - # Get some ZKAPs from the issuer. - try: - ${runOnNode "client" [ - get-passes - "http://127.0.0.1:3456" - "/tmp/client/private/api_auth_token" - issuerURL - voucher - ]} - except: - code, output = client.execute('cat /tmp/stdout /tmp/stderr'); - client.log(output) - - # Dump the fake Stripe API server logs, too, since the error may arise - # from a PaymentServer/Stripe interaction. - code, output = api_stripe_com.execute('journalctl -u api.stripe.com') - api_stripe_com.log(output) - raise - - # The client should be prepped now. Make it try to use some storage. - try: - ${runOnNode "client" [ exercise-storage "/tmp/client" ]} - except: - code, output = client.execute('cat /tmp/stdout /tmp/stderr') - client.log(output) - raise - - # It should be possible to restart the storage service without the - # storage node fURL changing. - try: - furlfile = '/var/db/tahoe-lafs/storage/private/storage-plugin.privatestorageio-zkapauthz-v1.furl' - before = storage.execute('cat ' + furlfile) - ${runOnNode "storage" [ "systemctl" "restart" "tahoe.storage" ]} - after = storage.execute('cat ' + furlfile) - if (before != after): - raise Exception('fURL changes after storage node restart') - except: - code, output = storage.execute('cat /tmp/stdout /tmp/stderr') - storage.log(output) - raise - - # The client should actually still work, too. - try: - ${runOnNode "client" [ exercise-storage "/tmp/client" ]} - except: - code, output = client.execute('cat /tmp/stdout /tmp/stderr') - client.log(output) - raise - - # The issuer metrics should be accessible from the monitoring network. - issuer.execute('ifconfig lo:fauxvpn 172.23.23.2/24') - issuer.wait_until_succeeds("nc -z 172.23.23.2 80") - issuer.succeed('curl --silent --insecure --fail --output /dev/null http://172.23.23.2/metrics') - # The issuer metrics should NOT be accessible from any other network. - issuer.fail('curl --silent --insecure --fail --output /dev/null http://localhost/metrics') - client.fail('curl --silent --insecure --fail --output /dev/null http://issuer/metrics') - issuer.execute('ifconfig lo:fauxvpn down') - ''; + testScript = ourpkgs.lib.testing.makeTestScript { + testpath = ./test_privatestorage.py; + kwargs = { + inherit sshPrivateKeyFile pemFile introducerPort introducerFURL issuerURL ristrettoPublicKey voucher; + + # Supply some helper programs to help the tests stay a bit higher level. + run_introducer = ./run-introducer.py; + run_client = ./run-client.py; + get_passes = ./get-passes.py; + exercise_storage = ./exercise-storage.py; + }; + }; } diff --git a/nixos/tests/tahoe.nix b/nixos/tests/tahoe.nix index e39fd6d3fcb776e8e5215bb1264e08e2b7306c1f..a007e65efd2d6bee8ab4adba9df3cb2901f53526 100644 --- a/nixos/tests/tahoe.nix +++ b/nixos/tests/tahoe.nix @@ -1,5 +1,8 @@ -{ ... }: - { +{ pkgs }: +let + ourpkgs = pkgs.callPackage ../pkgs { }; +in +{ nodes = { storage = { config, pkgs, ourpkgs, ... }: { imports = [ @@ -23,50 +26,7 @@ }; }; }; - testScript = '' - start_all() - - # After the service starts, destroy the "created" marker to force it to - # re-create its internal state. - storage.wait_for_open_port(4001) - storage.succeed("systemctl stop tahoe.storage") - storage.succeed("rm /var/db/tahoe-lafs/storage.created") - storage.succeed("systemctl start tahoe.storage") - - # After it starts up again, verify it has consistent internal state and a - # backup of the prior state. - storage.wait_for_open_port(4001) - storage.succeed("[ -e /var/db/tahoe-lafs/storage ]") - storage.succeed("[ -e /var/db/tahoe-lafs/storage.created ]") - storage.succeed("[ -e /var/db/tahoe-lafs/storage.1 ]") - storage.succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.privkey ]") - storage.succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.pem ]") - storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.2 ]") - - # Stop it again, once again destroy the "created" marker, and this time also - # jam some partial state in the way that will need cleanup. - storage.succeed("systemctl stop tahoe.storage") - storage.succeed("rm /var/db/tahoe-lafs/storage.created") - storage.succeed("mkdir -p /var/db/tahoe-lafs/storage.atomic/partial") - try: - storage.succeed("systemctl start tahoe.storage") - except: - x, y = storage.execute("journalctl -u tahoe.storage") - storage.log(y) - raise - - # After it starts up again, verify it has consistent internal state and - # backups of the prior two states. It also has no copy of the inconsistent - # state because it could never have been used. - storage.wait_for_open_port(4001) - storage.succeed("[ -e /var/db/tahoe-lafs/storage ]") - storage.succeed("[ -e /var/db/tahoe-lafs/storage.created ]") - storage.succeed("[ -e /var/db/tahoe-lafs/storage.1 ]") - storage.succeed("[ -e /var/db/tahoe-lafs/storage.2 ]") - storage.succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.privkey ]") - storage.succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.pem ]") - storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.atomic ]") - storage.succeed("[ ! -e /var/db/tahoe-lafs/storage/partial ]") - storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.3 ]") - ''; + testScript = ourpkgs.lib.testing.makeTestScript { + testpath = ./test_tahoe.py; + }; } diff --git a/nixos/tests/test_privatestorage.py b/nixos/tests/test_privatestorage.py new file mode 100644 index 0000000000000000000000000000000000000000..e1f34fa4f1b0f603168fe825871d8cb81f52d8ce --- /dev/null +++ b/nixos/tests/test_privatestorage.py @@ -0,0 +1,148 @@ +def runOnNode(node, argv): + """ + Run a shell command on one of the nodes. The first argument is the name + of the node. The second is a list of the argv to run. + + The program's output is piped to systemd-cat and the python fragment + evaluates to success if the command exits with a success status. + """ + try: + node.succeed('set -eo pipefail; {} | systemd-cat'.format(" ".join(argv))) + except Exception as e: + code, output = node.execute('cat /tmp/stdout /tmp/stderr') + introducer.log(output) + raise + +def ssh(username, sshPrivateKeyFile, hostname): + """ + Generate a command which can be used with runOnNode to ssh to the given + host. + """ + return [ + "cp", sshPrivateKeyFile, "/tmp/ssh_key", ";", + "chmod", "0400", "/tmp/ssh_key", ";", + "ssh", "-oStrictHostKeyChecking=no", "-i", "/tmp/ssh_key", + "{username}@{hostname}".format(username=username, hostname=hostname), ":", + ] + +def test( + sshPrivateKeyFile, + pemFile, + run_introducer, + run_client, + get_passes, + exercise_storage, + introducerPort, + introducerFURL, + issuerURL, + ristrettoPublicKey, + voucher, +): + """ + """ + # Boot the VMs. We used to do them all in parallel but the boot + # sequence got flaky at some point for some reason I don't + # understand. :/ It might be related to this: + # + # https://discourse.nixos.org/t/nixos-ppc64le-vm-does-not-have-dev-vda-device/11548/9 + # + # See <nixpkgs/nixos/modules/virtualisation/qemu-vm.nix> for the Nix + # that constructs the QEMU command that gets run. + # + # Boot them one at a time for now. + issuer.connect() + introducer.connect() + storage.connect() + client.connect() + api_stripe_com.connect() + + # The issuer and the storage server should accept SSH connections. This + # doesn't prove it is so but if it fails it's a pretty good indication + # it isn't so. + storage.wait_for_open_port(22) + runOnNode(issuer, ssh("probeuser", sshPrivateKeyFile, "storage")) + runOnNode(issuer, ssh("root", sshPrivateKeyFile, "storage")) + issuer.wait_for_open_port(22) + runOnNode(storage, ssh("probeuser", sshPrivateKeyFile, "issuer")) + runOnNode(storage, ssh("root", sshPrivateKeyFile, "issuer")) + + # Set up a Tahoe-LAFS introducer. + introducer.copy_from_host(pemFile, '/tmp/node.pem') + + runOnNode(introducer, [run_introducer, "/tmp/node.pem", str(introducerPort), introducerFURL]) + + # + # Get a Tahoe-LAFS storage server up. + # + code, version = storage.execute('tahoe --version') + storage.log(version) + + # The systemd unit should reach the running state. + storage.wait_for_unit('tahoe.storage.service') + + # Some while after that the Tahoe-LAFS node should listen on the web API + # port. The port number here has to agree with the port number set in + # the private-storage.nix module. + storage.wait_for_open_port(3456) + + # Once the web API is listening it should be possible to scrape some + # status from the node if it is really working. + storage.succeed('tahoe -d /var/db/tahoe-lafs/storage status') + + # It should have Eliot logging turned on as well. + storage.succeed('[ -e /var/db/tahoe-lafs/storage/logs/eliot.json ]') + + # + # Storage appears to be working so try to get a client to speak with it. + # + runOnNode(client, [run_client, "/tmp/client", introducerFURL, issuerURL, ristrettoPublicKey]) + client.wait_for_open_port(3456) + + # Make sure the fake Stripe API server is ready for requests. + try: + api_stripe_com.wait_for_unit("api.stripe.com") + except: + code, output = api_stripe_com.execute('journalctl -u api.stripe.com') + api_stripe_com.log(output) + raise + + # Get some ZKAPs from the issuer. + try: + runOnNode(client, [ + get_passes, + "http://127.0.0.1:3456", + "/tmp/client/private/api_auth_token", + issuerURL, + voucher, + ]) + except: + # Dump the fake Stripe API server logs, too, since the error may arise + # from a PaymentServer/Stripe interaction. + code, output = api_stripe_com.execute('journalctl -u api.stripe.com') + api_stripe_com.log(output) + + raise + + # The client should be prepped now. Make it try to use some storage. + runOnNode(client, [exercise_storage, "/tmp/client"]) + + # It should be possible to restart the storage service without the + # storage node fURL changing. + furlfile = '/var/db/tahoe-lafs/storage/private/storage-plugin.privatestorageio-zkapauthz-v1.furl' + before = storage.execute('cat ' + furlfile) + runOnNode(storage, ["systemctl", "restart", "tahoe.storage"]) + after = storage.execute('cat ' + furlfile) + if (before != after): + raise Exception('fURL changes after storage node restart') + + # The client should actually still work, too. + runOnNode(client, [exercise_storage, "/tmp/client"]) + + # The issuer metrics should be accessible from the monitoring network. + issuer.execute('ifconfig lo:fauxvpn 172.23.23.2/24') + issuer.wait_until_succeeds("nc -z 172.23.23.2 80") + issuer.succeed('curl --silent --insecure --fail --output /dev/null http://172.23.23.2/metrics') + # The issuer metrics should NOT be accessible from any other network. + issuer.fail('curl --silent --insecure --fail --output /dev/null http://localhost/metrics') + client.fail('curl --silent --insecure --fail --output /dev/null http://issuer/metrics') + issuer.execute('ifconfig lo:fauxvpn down') diff --git a/nixos/tests/test_tahoe.py b/nixos/tests/test_tahoe.py new file mode 100644 index 0000000000000000000000000000000000000000..c5190c78b04cb2cc59b5275e45869dd53e0e81c3 --- /dev/null +++ b/nixos/tests/test_tahoe.py @@ -0,0 +1,45 @@ +def test(): + start_all() + + # After the service starts, destroy the "created" marker to force it to + # re-create its internal state. + storage.wait_for_open_port(4001) + storage.succeed("systemctl stop tahoe.storage") + storage.succeed("rm /var/db/tahoe-lafs/storage.created") + storage.succeed("systemctl start tahoe.storage") + + # After it starts up again, verify it has consistent internal state and a + # backup of the prior state. + storage.wait_for_open_port(4001) + storage.succeed("[ -e /var/db/tahoe-lafs/storage ]") + storage.succeed("[ -e /var/db/tahoe-lafs/storage.created ]") + storage.succeed("[ -e /var/db/tahoe-lafs/storage.1 ]") + storage.succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.privkey ]") + storage.succeed("[ -e /var/db/tahoe-lafs/storage.1/private/node.pem ]") + storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.2 ]") + + # Stop it again, once again destroy the "created" marker, and this time also + # jam some partial state in the way that will need cleanup. + storage.succeed("systemctl stop tahoe.storage") + storage.succeed("rm /var/db/tahoe-lafs/storage.created") + storage.succeed("mkdir -p /var/db/tahoe-lafs/storage.atomic/partial") + try: + storage.succeed("systemctl start tahoe.storage") + except: + x, y = storage.execute("journalctl -u tahoe.storage") + storage.log(y) + raise + + # After it starts up again, verify it has consistent internal state and + # backups of the prior two states. It also has no copy of the inconsistent + # state because it could never have been used. + storage.wait_for_open_port(4001) + storage.succeed("[ -e /var/db/tahoe-lafs/storage ]") + storage.succeed("[ -e /var/db/tahoe-lafs/storage.created ]") + storage.succeed("[ -e /var/db/tahoe-lafs/storage.1 ]") + storage.succeed("[ -e /var/db/tahoe-lafs/storage.2 ]") + storage.succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.privkey ]") + storage.succeed("[ -e /var/db/tahoe-lafs/storage.2/private/node.pem ]") + storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.atomic ]") + storage.succeed("[ ! -e /var/db/tahoe-lafs/storage/partial ]") + storage.succeed("[ ! -e /var/db/tahoe-lafs/storage.3 ]") diff --git a/nixpkgs-2105.json b/nixpkgs-2105.json deleted file mode 100644 index 523c1468f35019c6685f3a8486603d0936732dbe..0000000000000000000000000000000000000000 --- a/nixpkgs-2105.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "release2105", - "url": "https://releases.nixos.org/nixos/21.05/nixos-21.05.4547.2949ed36539/nixexprs.tar.xz", - "sha256": "0nm5znl7lh3qws29ppzpzsqscyw3hk7q0128xqmga2g86qcmy38x" -} \ No newline at end of file diff --git a/nixpkgs-2105.nix b/nixpkgs-2105.nix index 536d913b89ba6a57d8d683381ea1c8f40e026b4f..e33347a21c29186826256e60bea0122fc85322bd 100644 --- a/nixpkgs-2105.nix +++ b/nixpkgs-2105.nix @@ -1 +1,6 @@ -import (builtins.fetchTarball (builtins.fromJSON (builtins.readFile ./nixpkgs-2105.json))) +# This actually imports nixos-21.11 but we need to keep this file around so that +# upgrades work, as the on-node deployment script expects this file in the checkout. +# See https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/222#note_18600 +# This file can be removed once all nodes have been updated to point to the new file. + +import ./nixpkgs.nix diff --git a/nixpkgs.json b/nixpkgs.json new file mode 100644 index 0000000000000000000000000000000000000000..62afaf9ea5f9896250a677bb383e16b8d0bc081a --- /dev/null +++ b/nixpkgs.json @@ -0,0 +1,5 @@ +{ + "name": "source", + "url": "https://releases.nixos.org/nixos/21.11/nixos-21.11.335883.7adc9c14ec7/nixexprs.tar.xz", + "sha256": "0r0sgphydyv0xggzyfas5v7dznf42sdgib8s8zmf7dbi09yb4y1l" +} \ No newline at end of file diff --git a/nixpkgs.nix b/nixpkgs.nix new file mode 100644 index 0000000000000000000000000000000000000000..a49c447874e45bea0804185636468568f5bd5035 --- /dev/null +++ b/nixpkgs.nix @@ -0,0 +1 @@ +import (builtins.fetchTarball (builtins.fromJSON (builtins.readFile ./nixpkgs.json))) diff --git a/shell.nix b/shell.nix index a5741377eec5ebd4b8862a0ea47e15edfdac2731..b8be3a3a6088a987468329ad29919a6957313c6a 100644 --- a/shell.nix +++ b/shell.nix @@ -1,7 +1,7 @@ let - release2105 = import ./nixpkgs-2105.nix { }; + pinned-pkgs = import ./nixpkgs.nix { }; in -{ pkgs ? release2105, lib ? pkgs.lib, python ? pkgs.python3 }: +{ pkgs ? pinned-pkgs, lib ? pkgs.lib, python ? pkgs.python3 }: let tools = pkgs.callPackage ./tools {}; in @@ -10,7 +10,7 @@ pkgs.mkShell { # first adds that path to the store, and then interpolates the store path # into the string. We use `builtins.toString` to convert the path to a # string without copying it to the store before interpolating. Either the - # path is already in the store (e.g. when `pkgs` is `release2105`) so we + # path is already in the store (e.g. when `pkgs` is `pinned-pkgs`) so we # avoid making a second copy with a longer name, or the user passed in local # path (e.g. a checkout of nixpkgs) and we point at it directly, rather than # a snapshot of it. diff --git a/tools/update-nixpkgs b/tools/update-nixpkgs index 09c823b0a419b5937d4953337b94a26c4b502e32..3c6832c95cee09632318f7a4ef4efc7099e317e1 100755 --- a/tools/update-nixpkgs +++ b/tools/update-nixpkgs @@ -10,7 +10,7 @@ from ps_tools import get_url_hash # We pass this to builtins.fetchTarball which only supports sha256 HASH_TYPE = "sha256" -DEFAULT_CHANNEL = "nixos-21.05" +DEFAULT_CHANNEL = "nixos-21.11" CHANNEL_URL_TEMPLATE = "https://channels.nixos.org/{channel}/nixexprs.tar.xz" @@ -37,7 +37,7 @@ def main(): "repo_file", metavar="repo-file", nargs="?", - default=Path(__file__).parent.with_name("nixpkgs-2105.json"), + default=Path(__file__).parent.with_name("nixpkgs.json"), type=Path, help="JSON file with pinned configuration.", )