Merge branch 'collapse-customize' into 'develop'

Merge most of the code from the `customize-*.nix` nodes into the base modules.

See merge request !233

morph/grid/local/grid.nix

@@ -59,6 +59,7 @@ let
     grid = {
       publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
       privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
+      inherit (grid-config) monitoringvpnEndpoint;
     };
     # Configure deployment management authorization for all systems in the grid.
     services.private-storage.deployment = {
@@ -70,41 +71,44 @@ let
   payments = {
     imports = [
       gridlib.issuer
-      (gridlib.customize-issuer (grid-config // {
-          monitoringvpnIPv4 = "172.23.23.11";
-      }))
       grid-module
     ];
     config = {
+      grid.monitoringvpnIPv4 = "172.23.23.11";
       grid.publicIPv4 = "192.168.67.21";
+      grid.issuer = {
+        inherit (grid-config) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
+      };
     };
   };
 
   storage1 = {
     imports = [
       gridlib.storage
-      (gridlib.customize-storage (grid-config // {
-        monitoringvpnIPv4 = "172.23.23.12";
-        stateVersion = "19.09";
-      }))
       grid-module
     ];
     config = {
+      grid.monitoringvpnIPv4 = "172.23.23.12";
       grid.publicIPv4 = "192.168.67.22";
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = "19.09";
     };
   };
 
   storage2 = {
     imports = [
       gridlib.storage
-      (gridlib.customize-storage (grid-config // {
-        monitoringvpnIPv4 = "172.23.23.13";
-        stateVersion = "19.09";
-      }))
       grid-module
     ];
     config = {
+      grid.monitoringvpnIPv4 = "172.23.23.13";
       grid.publicIPv4 = "192.168.67.23";
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = "19.09";
     };
   };
 
@@ -119,12 +123,12 @@ let
         inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
         googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
         enableSlackAlert = false;
-        monitoringvpnIPv4 = "172.23.23.1";
         stateVersion = "19.09";
       })
       grid-module
     ];
     config = {
+      grid.monitoringvpnIPv4 = "172.23.23.1";
       grid.publicIPv4 = "192.168.67.24";
     };
   };

morph/grid/production/grid.nix

@@ -21,6 +21,7 @@ let
     grid = {
       publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
       privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
+      inherit (grid-config) monitoringvpnEndpoint;
     };
     # Configure deployment management authorization for all systems in the grid.
     services.private-storage.deployment = {
@@ -33,11 +34,14 @@ let
     imports = [
       gridlib.issuer
       gridlib.hardware-aws
-      (gridlib.customize-issuer (grid-config // {
-        monitoringvpnIPv4 = "172.23.23.11";
-      }))
       grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.11";
+      grid.issuer = {
+        inherit (grid-config) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
+      };
+    };
   };
 
   monitoring = {
@@ -52,11 +56,13 @@ let
         inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
         googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
         enableSlackAlert = true;
-        monitoringvpnIPv4 = "172.23.23.1";
         stateVersion = "19.09";
       })
       grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.1";
+    };
   };
 
   defineStorageNode = name: { vpnIP, stateVersion }:
@@ -79,17 +85,17 @@ let
 
       # Get all of the configuration that is common across all storage nodes.
       gridlib.storage
-
-      # Then customize the storage system a little bit based on this node's particulars.
-      (gridlib.customize-storage (grid-config // nodecfg // {
-        monitoringvpnIPv4 = vpnIP;
-        inherit stateVersion;
-      }))
-
       # Also configure deployment management authorization
       grid-module
     ];
 
+    config = {
+      grid.monitoringvpnIPv4 = vpnIP;
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = stateVersion;
+
       # And supply configuration for those hardware / network / bootloader
       # options.  See the 100tb module for handling of this value.  The module
       # name is quoted because `1` makes `100tb` look an awful lot like a
@@ -100,6 +106,7 @@ let
       # TODO would be nice to enable only on machines that have such a device.
       services.private-storage.monitoring.megacli2prom.enable = true;
     };
+  };
 
   # Define all of the storage nodes for this grid.
   storageNodes = builtins.mapAttrs defineStorageNode {

morph/grid/testing/grid.nix

@@ -21,6 +21,7 @@ let
     grid = {
       publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
       privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
+      inherit (grid-config) monitoringvpnEndpoint;
     };
     # Configure deployment management authorization for all systems in the grid.
     services.private-storage.deployment = {
@@ -33,11 +34,14 @@ let
     imports = [
       gridlib.issuer
       gridlib.hardware-aws
-      (gridlib.customize-issuer (grid-config // {
-        monitoringvpnIPv4 = "172.23.23.11";
-      }))
       grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.11";
+      grid.issuer = {
+        inherit (grid-config) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
+      };
+    };
   };
 
   storage001 = {
@@ -45,12 +49,15 @@ let
       gridlib.storage
       gridlib.hardware-aws
       ./testing001-hardware.nix
-      (gridlib.customize-storage (grid-config // {
-        monitoringvpnIPv4 = "172.23.23.12";
-        stateVersion = "19.03";
-      }))
       grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.12";
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = "19.03";
+    };
   };
 
   monitoring = {
@@ -65,11 +72,13 @@ let
         inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
         googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
         enableSlackAlert = true;
-        monitoringvpnIPv4 = "172.23.23.1";
         stateVersion = "19.09";
       })
       grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.1";
+    };
   };
 
   # TBD: derive these automatically:

morph/lib/base.nix

@@ -18,6 +18,18 @@
       corresponding private keys for the system.
       '';
     };
+    monitoringvpnIPv4 = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The IPv4 address of this node on the monitoring VPN.
+      '';
+    };
+    monitoringvpnEndpoint = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The domain name and port of the monitoring VPN endpoint.
+      '';
+    };
   };
 
   # Any extra NixOS modules to load on all our servers.  Note that just

morph/lib/customize-issuer.nix

-# Define a function which returns a value which fills in all the holes left by
-# ``issuer.nix``.
-{
-  # A string giving the IP address and port number (":"-separated) of the VPN
-  # server.
-  monitoringvpnEndpoint
-
-  # A string giving the VPN IPv4 address for this system.
-, monitoringvpnIPv4
-
-  # A string giving an email address to use for Let's Encrypt registration and
-  # certificate issuance.
-, letsEncryptAdminEmail
-
-  # A list of strings giving the domain names that point at this issuer
-  # system.  These will all be included in Let's Encrypt certificate.
-, issuerDomains
-
-  # A list of strings giving CORS Origins will the issuer will be configured
-  # to allow.
-, allowedChargeOrigins
-, ...
-}:
-{ config, ... }:
-let
-  inherit (config.grid) publicKeyPath privateKeyPath;
-in {
-  deployment.secrets = {
-    # ``.../monitoringvpn`` is a path on the deployment system of a directory
-    # containing a number of VPN-related secrets.  This is expected to contain
-    # a number of files named like ``<VPN IPv4 address>.key`` containing the
-    # VPN private key for the corresponding host.  It must also contain
-    # ``server.pub`` and ``preshared.key`` holding the VPN server's public key
-    # and the pre-shared key, respectively.  All of these things are used as
-    # the sources of various VPN-related morph secrets.
-    "monitoringvpn-secret-key".source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key";
-    "monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key";
-  };
-
-  services.private-storage.monitoring.vpn.client = {
-    enable = true;
-    ip = monitoringvpnIPv4;
-    endpoint = monitoringvpnEndpoint;
-    endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub";
-  };
-
-  services.private-storage-issuer = {
-    inherit letsEncryptAdminEmail allowedChargeOrigins;
-    domains = issuerDomains;
-  };
-
-  system.stateVersion = "19.03";
-}

morph/lib/customize-monitoring.nix

@@ -8,9 +8,12 @@
   # allows Grafana to show us hostnames instead of VPN IP addresses.
   hostsMap
 
-  # See ``customize-issuer.nix``.
-, monitoringvpnIPv4
+  # A string giving an email address to use for Let's Encrypt registration and
+  # certificate issuance.
 , letsEncryptAdminEmail
+
+  # A list of strings giving the domain names that point at this monitoring
+  # system.  These will all be included in Let's Encrypt certificate.
 , monitoringDomains
 
   # A list of VPN IP addresses as strings indicating which clients will be
@@ -47,7 +50,7 @@
 }:
 { config, ... }:
 let
-  inherit (config.grid) publicKeyPath privateKeyPath;
+  inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4;
 in {
   deployment.secrets = let
     # When Grafana SSO is disabled there is not necessarily any client secret
@@ -93,12 +96,8 @@ in {
           action = ["sudo" "systemctl" "restart" "grafana.service"];
         };
       };
-    monitoringvpn = {
-      "monitoringvpn-private-key".source = "${privateKeyPath}/monitoringvpn/server.key";
-      "monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key";
-    };
     in
-      grafanaSSO // grafanaSlackUrl // monitoringvpn;
+      grafanaSSO // grafanaSlackUrl;
 
   networking.hosts = hostsMap;
 

morph/lib/customize-storage.nix

-# Define a function which returns a value which fills in all the holes left by
-# ``storage.nix``.
-{
-  # See ``customize-issuer.nix``
-  monitoringvpnEndpoint
-, monitoringvpnIPv4
-
-  # An integer giving the value of a single pass in byte×months.
-, passValue
-
-  # An integer giving the port number to include in Tahoe storage service
-  # advertisements and on which to listen for storage connections.
-, publicStoragePort
-
-  # A string giving the NixOS state version for the system.
-, stateVersion
-, ...
-}:
-{ config, ... }:
-let
-  inherit (config.grid) publicKeyPath privateKeyPath;
-in {
-  deployment.secrets = {
-    "monitoringvpn-secret-key".source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key";
-    "monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key";
-  };
-
-  services.private-storage = {
-    inherit passValue publicStoragePort;
-  };
-
-  services.private-storage.monitoring.vpn.client = {
-    enable = true;
-    ip = monitoringvpnIPv4;
-    endpoint = monitoringvpnEndpoint;
-    endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub";
-  };
-
-  system.stateVersion = stateVersion;
-}

morph/lib/default.nix

@@ -8,10 +8,7 @@
   hardware-vagrant = import ./hardware-vagrant.nix;
 
   issuer = import ./issuer.nix;
-  customize-issuer = import ./customize-issuer.nix;
-
   storage = import ./storage.nix;
-  customize-storage = import ./customize-storage.nix;
 
   monitoring = import ./monitoring.nix;
   customize-monitoring = import ./customize-monitoring.nix;

morph/lib/issuer.nix

-# This, along with `customize-issuer.nix, contains all of the NixOS system
-# configuration necessary to specify an "issuer"-type system.  Originally, this
-# file has all the static configuration, and `customize-issuer.nix` was a function
-# that filled in the holes. We are in the process of merging the modules, using settings
-# instead of function arguments.
-# See https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/issues/80
-{ config, ...}:
+# This contains all of the NixOS system configuration necessary to specify an
+# "issuer"-type system.
+{ lib, config, ...}:
 let
-  inherit (config.grid) publicKeyPath privateKeyPath;
+  inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnEndpoint monitoringvpnIPv4;
+  inherit (config.grid.issuer) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
 in {
+  imports = [
+    ../../nixos/modules/monitoring/vpn/client.nix
+    ../../nixos/modules/monitoring/exporters/node.nix
+  ];
+
+  options.grid.issuer = {
+    letsEncryptAdminEmail = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        A string giving an email address to use for Let's Encrypt registration and
+        certificate issuance.
+      '';
+    };
+
+    issuerDomains = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        A list of strings giving the domain names that point at this issuer
+        system.  These will all be included in Let's Encrypt certificate.
+      '';
+    };
+
+    allowedChargeOrigins = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        A list of strings giving CORS Origins will the issuer will be configured
+        to allow.
+      '';
+    };
+  };
+
+  config = {
     deployment = {
       secrets = {
         "ristretto-signing-key" = {
@@ -29,6 +58,7 @@ in {
 
         "monitoringvpn-secret-key" = {
           destination = "/run/keys/monitoringvpn/client.key";
+          source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key";
           owner.user = "root";
           owner.group = "root";
           permissions = "0400";
@@ -36,6 +66,7 @@ in {
         };
         "monitoringvpn-preshared-key" = {
           destination = "/run/keys/monitoringvpn/preshared.key";
+          source = "${privateKeyPath}/monitoringvpn/preshared.key";
           owner.user = "root";
           owner.group = "root";
           permissions = "0400";
@@ -44,11 +75,6 @@ in {
       };
     };
 
-  imports = [
-    ../../nixos/modules/monitoring/vpn/client.nix
-    ../../nixos/modules/monitoring/exporters/node.nix
-  ];
-
     services.private-storage-issuer = {
       enable = true;
       tls = true;
@@ -56,5 +82,17 @@ in {
       stripeSecretKeyPath = config.deployment.secrets.stripe-secret-key.destination;
       database = "SQLite3";
       databasePath = "${config.fileSystems."zkapissuer-data".mountPoint}/vouchers.sqlite3";
+      inherit letsEncryptAdminEmail allowedChargeOrigins;
+      domains = issuerDomains;
+    };
+
+    services.private-storage.monitoring.vpn.client = {
+      enable = true;
+      ip = monitoringvpnIPv4;
+      endpoint = monitoringvpnEndpoint;
+      endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub";
+    };
+
+    system.stateVersion = "19.03";
   };
 }

morph/lib/monitoring.nix

 # Similar to ``issuer.nix`` but for a "monitoring"-type system.  Holes are
 # filled by ``customize-monitoring.nix``.
-{
+{ config, ...}:
+let
+  inherit (config.grid) privateKeyPath;
+in {
   deployment = {
     secrets = {
       "monitoringvpn-private-key" = {
         destination = "/run/keys/monitoringvpn/server.key";
+        source = "${privateKeyPath}/monitoringvpn/server.key";
         owner.user = "root";
         owner.group = "root";
         permissions = "0400";
@@ -12,6 +16,7 @@
       };
       "monitoringvpn-preshared-key" = {
         destination = "/run/keys/monitoringvpn/preshared.key";
+        source = "${privateKeyPath}/monitoringvpn/preshared.key";
         owner.user = "root";
         owner.group = "root";
         permissions = "0400";

morph/lib/storage.nix

-# Similar to ``issuer.nix`` but for a "storage"-type system.  Holes are filled
-# by ``customize-storage.nix``.
-{ config, ...} :
+# This contains all of the NixOS system configuration necessary to specify an
+# "storage"-type system.
+{ lib, config, ...} :
 let
-  inherit (config.grid) publicKeyPath privateKeyPath;
+  inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4 monitoringvpnEndpoint;
 in {
+  # Any extra NixOS modules to load on this server.
+  imports = [
+    # Bring in our module for configuring the Tahoe-LAFS service and other
+    # Private Storage-specific things.
+    ../../nixos/modules/private-storage.nix
+    # Connect to the monitoringvpn.
+    ../../nixos/modules/monitoring/vpn/client.nix
+    # Expose base system metrics over the monitoringvpn.
+    ../../nixos/modules/monitoring/exporters/node.nix
+    # Collect Tahoe OpenMetrics statistics.
+    ../../nixos/modules/monitoring/exporters/tahoe.nix
+  ];
+
+  options.grid.storage = {
+    passValue = lib.mkOption {
+      type = lib.types.int;
+      description = ''
+        An integer giving the value of a single pass in byte×months.
+      '';
+    };
+
+    publicStoragePort = lib.mkOption {
+      type = lib.types.port;
+      description = ''
+        An integer giving the port number to include in Tahoe storage service
+        advertisements and on which to listen for storage connections.
+      '';
+    };
+  };
+
+  config = {
     deployment = {
       secrets = {
         "ristretto-signing-key" = {
@@ -19,6 +50,7 @@ in {
         };
         "monitoringvpn-secret-key" = {
           destination = "/run/keys/monitoringvpn/client.key";
+          source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key";
           owner.user = "root";
           owner.group = "root";
           permissions = "0400";
@@ -26,6 +58,7 @@ in {
         };
         "monitoringvpn-preshared-key" = {
           destination = "/run/keys/monitoringvpn/preshared.key";
+          source = "${privateKeyPath}/monitoringvpn/preshared.key";
           owner.user = "root";
           owner.group = "root";
           permissions = "0400";
@@ -34,19 +67,6 @@ in {
       };
     };
 
-  # Any extra NixOS modules to load on this server.
-  imports = [
-    # Bring in our module for configuring the Tahoe-LAFS service and other
-    # Private Storage-specific things.
-    ../../nixos/modules/private-storage.nix
-    # Connect to the monitoringvpn.
-    ../../nixos/modules/monitoring/vpn/client.nix
-    # Expose base system metrics over the monitoringvpn.
-    ../../nixos/modules/monitoring/exporters/node.nix
-    # Collect Tahoe OpenMetrics statistics.
-    ../../nixos/modules/monitoring/exporters/tahoe.nix
-  ];
-
     services.private-storage.monitoring.tahoe.enable = true;
 
     # Turn on the Private Storage (Tahoe-LAFS) service.
@@ -55,5 +75,14 @@ in {
       enable = true;
       # Give it the Ristretto signing key to support authorization.
       ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination;
+      inherit (config.grid.storage) passValue publicStoragePort;
+    };
+
+    services.private-storage.monitoring.vpn.client = {
+      enable = true;
+      ip = monitoringvpnIPv4;
+      endpoint = monitoringvpnEndpoint;
+      endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub";
+    };
   };
 }