Add monitoring to production deployment (WIP)

morph/grid/production/config.json

 { "publicStoragePort": 8898
 , "ristrettoSigningKeyPath": "../../PrivateStorageSecrets/ristretto.signing-key"
 , "stripeSecretKeyPath": "../../PrivateStorageSecrets/stripe.secret"
+, "monitoringvpnSecretKeyDir": "../../PrivateStorageSecrets/monitoringvpn"
 , "passValue": 1000000
 , "issuerDomain": "payments.privatestorage.io"
 , "letsEncryptAdminEmail": "jean-paul@privatestorage.io"

morph/grid/production/grid.nix

@@ -7,6 +7,35 @@ import ../../lib/make-grid.nix {
   nodes = cfg:
     let
       sshUsers = import ../../../../PrivateStorageSecrets/production-users.nix;
+
+      # TBD: derive these automatically:
+      hostsMap = {
+        "172.23.23.1"  = [ "monitoring" "monitoring.monitoringvpn" ];
+        "172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
+        "172.23.23.21" = [ "storage001" "storage001.monitoringvpn" ];
+        "172.23.23.22" = [ "storage002" "storage002.monitoringvpn" ];
+        "172.23.23.23" = [ "storage003" "storage003.monitoringvpn" ];
+        "172.23.23.24" = [ "storage004" "storage004.monitoringvpn" ];
+        "172.23.23.25" = [ "storage005" "storage005.monitoringvpn" ];
+      };
+      vpnClientIPs = [
+        "172.23.23.11"
+        "172.23.23.21"
+        "172.23.23.22"
+        "172.23.23.23"
+        "172.23.23.24"
+        "172.23.23.25"
+      ];
+      nodeExporterTargets = [
+        "monitoring"
+        "payments"
+        "storage001"
+        "storage002"
+        "storage003"
+        "storage004"
+        "storage005"
+      ];
+
     in {
     # Here are the hosts that are in this morph network.  This is sort of like
     # a server manifest.  We try to keep as many of the specific details as
@@ -22,6 +51,7 @@ import ../../lib/make-grid.nix {
     # The names must be unique!
     "payments.privatestorage.io" = import ../../lib/make-issuer.nix ({
       publicIPv4 = "18.184.142.208";
+      monitoringvpnIPv4 = "172.23.23.11";
       inherit sshUsers;
       hardware = ../../lib/issuer-aws.nix;
       stateVersion = "19.03";
@@ -32,30 +62,47 @@ import ../../lib/make-grid.nix {
         inherit sshUsers;
         hardware = ./storage001-hardware.nix;
         stateVersion = "19.09";
+        monitoringvpnIPv4 = "172.23.23.21";
     } // cfg);
     "storage002" = import ../../lib/make-storage.nix ({
         cfg = import ./storage002-config.nix;
         inherit sshUsers;
         hardware = ./storage002-hardware.nix;
         stateVersion = "19.09";
+        monitoringvpnIPv4 = "172.23.23.22";
     } // cfg);
     "storage003" = import ../../lib/make-storage.nix ({
         cfg = import ./storage003-config.nix;
         inherit sshUsers;
         hardware = ./storage003-hardware.nix;
         stateVersion = "19.09";
+        monitoringvpnIPv4 = "172.23.23.23";
     } // cfg);
     "storage004" = import ../../lib/make-storage.nix ({
         cfg = import ./storage004-config.nix;
         inherit sshUsers;
         hardware = ./storage004-hardware.nix;
         stateVersion = "19.09";
+        monitoringvpnIPv4 = "172.23.23.24";
     } // cfg);
     "storage005" = import ../../lib/make-storage.nix ({
         cfg = import ./storage005-config.nix;
         inherit sshUsers;
         hardware = ./storage005-hardware.nix;
         stateVersion = "19.03";
+        monitoringvpnIPv4 = "172.23.23.25";
+    } // cfg);
+
+    "monitoring" = import ../../lib/make-monitoring.nix ({
+      publicIPv4 = "monitoring.private.storage"; # XXX TBD when the machine is online
+      monitoringvpnIPv4 = "172.23.23.1";
+      inherit vpnClientIPs;
+      inherit hostsMap;
+      inherit nodeExporterTargets;
+      nginxExporterTargets = [ ];
+      hardware = ../../lib/issuer-aws.nix;
+      stateVersion = "19.09";
+      inherit sshUsers;
     } // cfg);
   };
 }

morph/lib/make-storage.nix

@@ -3,6 +3,7 @@
 , hardware                   # The path to the hardware configuration for this node.
 , publicStoragePort          # The storage port number on which to accept connections.
 , ristrettoSigningKeyPath    # The *local* path to the Ristretto signing key file.
+, monitoringvpnSecretKeyDir  # The directory that holds the VPN keys.
 , passValue                  # Bytes component of size×time value of passes.
 , sshUsers                   # Users for which to configure SSH access to this node.
 , stateVersion               # The value for system.stateVersion on this node.
@@ -11,6 +12,7 @@
                              # to avoid breaking some software such as
                              # database servers. You should change this only
                              # after NixOS release notes say you should.
+, monitoringvpnIPv4          # This node's IP in the monitoring VPN.
 , ...
 }: rec {
   deployment = {
@@ -40,6 +42,10 @@
     # Bring in our module for configuring the Tahoe-LAFS service and other
     # Private Storage-specific things.
     ../../nixos/modules/private-storage.nix
+    # Connect to the monitoringvpn.
+    ../../nixos/modules/monitoring/vpn/client.nix
+    # Expose base system metrics over the monitoringvpn.
+    ../../nixos/modules/monitoring/exporters/node.nix
   ];
 
   # Pass the configuration specific to this host to the 100TB module to be
@@ -67,4 +73,10 @@
   };
 
   system.stateVersion = stateVersion;
+
+  services.private-storage.monitoring.vpn.client = {
+    enable = true;
+    ip = monitoringvpnIPv4;
+    endpoint = "monitoring.private.storage:51820"; # XXX TBD when the machine is online
+  };
 }