diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix
index ed3cda9e787ba567c5c5eed19557fc281a80ceb6..e672f3d14ef5f7389511a818921a9a75c6e948fe 100644
--- a/morph/grid/local/grid.nix
+++ b/morph/grid/local/grid.nix
@@ -59,7 +59,7 @@ let
grid = {
publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
- inherit (grid-config) monitoringvpnEndpoint;
+ inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
};
# Configure deployment management authorization for all systems in the grid.
services.private-storage.deployment = {
@@ -77,7 +77,7 @@ let
grid.monitoringvpnIPv4 = "172.23.23.11";
grid.publicIPv4 = "192.168.67.21";
grid.issuer = {
- inherit (grid-config) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
+ inherit (grid-config) issuerDomains allowedChargeOrigins;
};
};
};
@@ -115,33 +115,22 @@ let
monitoring = {
imports = [
gridlib.monitoring
- (gridlib.customize-monitoring {
- inherit hostsMap vpnClientIPs
- nodeExporterTargets
- paymentExporterTargets
- blackboxExporterHttpsTargets;
- inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
- googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
- enableSlackAlert = false;
- stateVersion = "19.09";
- })
grid-module
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.1";
grid.publicIPv4 = "192.168.67.24";
+ grid.monitoring = {
+ inherit paymentExporterTargets blackboxExporterHttpsTargets;
+ inherit (grid-config) monitoringDomains;
+ googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+ enableSlackAlert = false;
+ };
+ system.stateVersion = "19.09";
};
};
# TBD: derive these automatically:
- hostsMap = {
- "172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ];
- "172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
- "172.23.23.12" = [ "storage1" "storage1.monitoringvpn" ];
- "172.23.23.13" = [ "storage2" "storage2.monitoringvpn" ];
- };
- vpnClientIPs = [ "172.23.23.11" "172.23.23.12" "172.23.23.13" ];
- nodeExporterTargets = [ "monitoring" "payments" "storage1" "storage2" ];
paymentExporterTargets = [ "payments" ];
blackboxExporterHttpsTargets = [
# "https://private.storage/"
diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix
index 12873a4bc010887c54f522a01115061e9875726f..ab45d4ba7f67e71383d28120bd925ac3a05f04ef 100644
--- a/morph/grid/production/grid.nix
+++ b/morph/grid/production/grid.nix
@@ -21,7 +21,7 @@ let
grid = {
publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
- inherit (grid-config) monitoringvpnEndpoint;
+ inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
};
# Configure deployment management authorization for all systems in the grid.
services.private-storage.deployment = {
@@ -39,7 +39,7 @@ let
config = {
grid.monitoringvpnIPv4 = "172.23.23.11";
grid.issuer = {
- inherit (grid-config) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
+ inherit (grid-config) issuerDomains allowedChargeOrigins;
};
};
};
@@ -48,20 +48,17 @@ let
imports = [
gridlib.monitoring
gridlib.hardware-aws
- (gridlib.customize-monitoring {
- inherit hostsMap vpnClientIPs
- nodeExporterTargets
- paymentExporterTargets
- blackboxExporterHttpsTargets;
- inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
- googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
- enableSlackAlert = true;
- stateVersion = "19.09";
- })
grid-module
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.1";
+ grid.monitoring = {
+ inherit paymentExporterTargets blackboxExporterHttpsTargets;
+ inherit (grid-config) monitoringDomains;
+ googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+ enableSlackAlert = true;
+ };
+ system.stateVersion = "19.09";
};
};
@@ -117,33 +114,6 @@ let
storage005 = { vpnIP = "172.23.23.25"; stateVersion = "19.03"; };
};
- # TBD: derive these automatically:
- hostsMap = {
- "172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ];
- "172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
- "172.23.23.21" = [ "storage001" "storage001.monitoringvpn" ];
- "172.23.23.22" = [ "storage002" "storage002.monitoringvpn" ];
- "172.23.23.23" = [ "storage003" "storage003.monitoringvpn" ];
- "172.23.23.24" = [ "storage004" "storage004.monitoringvpn" ];
- "172.23.23.25" = [ "storage005" "storage005.monitoringvpn" ];
- };
- vpnClientIPs = [
- "172.23.23.11"
- "172.23.23.21"
- "172.23.23.22"
- "172.23.23.23"
- "172.23.23.24"
- "172.23.23.25"
- ];
- nodeExporterTargets = [
- "monitoring"
- "payments"
- "storage001"
- "storage002"
- "storage003"
- "storage004"
- "storage005"
- ];
paymentExporterTargets = [ "payments" ];
blackboxExporterHttpsTargets = [
"https://private.storage/"
diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix
index 7f7aeccbaba1f4e8d2c9576ece080791d03fd065..19839ae83fa16c31adf0fcd9e3727a8304f8dd6c 100644
--- a/morph/grid/testing/grid.nix
+++ b/morph/grid/testing/grid.nix
@@ -21,7 +21,7 @@ let
grid = {
publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
- inherit (grid-config) monitoringvpnEndpoint;
+ inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
};
# Configure deployment management authorization for all systems in the grid.
services.private-storage.deployment = {
@@ -39,7 +39,7 @@ let
config = {
grid.monitoringvpnIPv4 = "172.23.23.11";
grid.issuer = {
- inherit (grid-config) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
+ inherit (grid-config) issuerDomains allowedChargeOrigins;
};
};
};
@@ -64,31 +64,21 @@ let
imports = [
gridlib.monitoring
gridlib.hardware-aws
- (gridlib.customize-monitoring {
- inherit hostsMap vpnClientIPs
- nodeExporterTargets
- paymentExporterTargets
- blackboxExporterHttpsTargets;
- inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
- googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
- enableSlackAlert = true;
- stateVersion = "19.09";
- })
grid-module
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.1";
+ grid.monitoring = {
+ inherit paymentExporterTargets blackboxExporterHttpsTargets;
+ inherit (grid-config) monitoringDomains;
+ googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+ enableSlackAlert = true;
+ };
+ system.stateVersion = "19.09";
};
};
# TBD: derive these automatically:
- hostsMap = {
- "172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ];
- "172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
- "172.23.23.12" = [ "storage001" "storage001.monitoringvpn" ];
- };
- vpnClientIPs = [ "172.23.23.11" "172.23.23.12" ];
- nodeExporterTargets = [ "monitoring" "payments" "storage001" ];
paymentExporterTargets = [ "payments" ];
blackboxExporterHttpsTargets = [
"https://privatestorage-staging.com/"
diff --git a/morph/lib/base.nix b/morph/lib/base.nix
index 9b698b9629c12c7343fbd2e9eea5d2ee599b612e..c814631f7e04ee6c8a5206893863d6e6d6fe9e97 100644
--- a/morph/lib/base.nix
+++ b/morph/lib/base.nix
@@ -30,6 +30,14 @@
The domain name and port of the monitoring VPN endpoint.
'';
};
+
+ letsEncryptAdminEmail = lib.mkOption {
+ type = lib.types.str;
+ description = ''
+ A string giving an email address to use for Let's Encrypt registration and
+ certificate issuance.
+ '';
+ };
};
# Any extra NixOS modules to load on all our servers. Note that just
diff --git a/morph/lib/customize-monitoring.nix b/morph/lib/customize-monitoring.nix
deleted file mode 100644
index 67544de5bab466fab0927b71b493ff81935192e9..0000000000000000000000000000000000000000
--- a/morph/lib/customize-monitoring.nix
+++ /dev/null
@@ -1,126 +0,0 @@
-# Define a function which returns a value which fills in all the holes left by
-# ``monitoring.nix``.
-{
- # A set mapping VPN IP addresses as strings to lists of hostnames as
- # strings. The system's ``/etc/hosts`` will be populated with this
- # information. Apart from helping with normal forward resolution, this
- # *also* gives us reverse resolution from the VPN IPs to hostnames which
- # allows Grafana to show us hostnames instead of VPN IP addresses.
- hostsMap
-
- # A string giving an email address to use for Let's Encrypt registration and
- # certificate issuance.
-, letsEncryptAdminEmail
-
- # A list of strings giving the domain names that point at this monitoring
- # system. These will all be included in Let's Encrypt certificate.
-, monitoringDomains
-
- # A list of VPN IP addresses as strings indicating which clients will be
- # allowed onto the VPN.
-, vpnClientIPs
-
- # A list of VPN clients (IP addresses or hostnames) as strings indicating
- # which nodes to scrape "nodeExporter" metrics from.
-, nodeExporterTargets
-
- # A list of VPN clients (IP addresses or hostnames) as strings indicating
- # which nodes to scrape "nginxExporter" metrics from.
-, nginxExporterTargets ? []
-
- # A list of VPN clients (IP addresses or hostnames) as strings indicating
- # which nodes to scrape PaymentServer metrics from.
-, paymentExporterTargets ? []
-
- # A list of HTTPS servers (URLs, IP addresses or hostnames) as strings indicating
- # which nodes the BlackboxExporter should scrape HTTP and TLS metrics from.
-, blackboxExporterHttpsTargets ? []
-
- # A string containing the GSuite OAuth2 ClientID to use to authenticate
- # logins to Grafana.
-, googleOAuthClientID
-
- # Whether to enable alerting via Slack.
- # When true requires a grafana-slack-url file (see private-keys/README.rst).
-, enableSlackAlert ? false
-
- # A string giving the NixOS state version for the system.
-, stateVersion
-, ...
-}:
-{ config, ... }:
-let
- inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4;
-in {
- deployment.secrets = let
- # When Grafana SSO is disabled there is not necessarily any client secret
- # available. Avoid telling morph that there is one in this case (so it
- # avoids trying to read it and then failing). Even if the secret did
- # exist, if SSO is disabled there's no point sending the secret to the
- # server.
- #
- # Also, we have to define this whole secret here so that we can configure
- # it completely or not at all. morph gets angry if we half configure it
- # (say, by just omitting the "source" value).
- grafanaSSO =
- if googleOAuthClientID == ""
- then { }
- else {
- "grafana-google-sso-secret" = {
- source = "${privateKeyPath}/grafana-google-sso.secret";
- destination = "/run/keys/grafana-google-sso.secret";
- owner.user = config.systemd.services.grafana.serviceConfig.User;
- owner.group = config.users.users.grafana.group;
- permissions = "0400";
- action = ["sudo" "systemctl" "restart" "grafana.service"];
- };
- "grafana-admin-password" = {
- source = "${privateKeyPath}/grafana-admin.password";
- destination = "/run/keys/grafana-admin.password";
- owner.user = config.systemd.services.grafana.serviceConfig.User;
- owner.group = config.users.users.grafana.group;
- permissions = "0400";
- action = ["sudo" "systemctl" "restart" "grafana.service"];
- };
- };
- grafanaSlackUrl =
- if !enableSlackAlert
- then { }
- else {
- "grafana-slack-url" = {
- source = "${privateKeyPath}/grafana-slack-url";
- destination = "/run/keys/grafana-slack-url";
- owner.user = config.systemd.services.grafana.serviceConfig.User;
- owner.group = config.users.users.grafana.group;
- permissions = "0400";
- action = ["sudo" "systemctl" "restart" "grafana.service"];
- };
- };
- in
- grafanaSSO // grafanaSlackUrl;
-
- networking.hosts = hostsMap;
-
- services.private-storage.monitoring.vpn.server = {
- enable = true;
- ip = monitoringvpnIPv4;
- inherit vpnClientIPs;
- pubKeysPath = "${publicKeyPath}/monitoringvpn";
- };
-
- services.private-storage.monitoring.prometheus = {
- inherit nodeExporterTargets;
- inherit nginxExporterTargets;
- inherit paymentExporterTargets;
- inherit blackboxExporterHttpsTargets;
- };
-
- services.private-storage.monitoring.grafana = {
- inherit letsEncryptAdminEmail;
- inherit googleOAuthClientID;
- inherit enableSlackAlert;
- domains = monitoringDomains;
- };
-
- system.stateVersion = stateVersion;
-}
diff --git a/morph/lib/default.nix b/morph/lib/default.nix
index 766fda5102589fbdc32e26c17fbc94731ab71a73..88c83bc2211da2e00e69f95d7d2110d3ee636cc3 100644
--- a/morph/lib/default.nix
+++ b/morph/lib/default.nix
@@ -9,9 +9,7 @@
issuer = import ./issuer.nix;
storage = import ./storage.nix;
-
monitoring = import ./monitoring.nix;
- customize-monitoring = import ./customize-monitoring.nix;
modules = builtins.toString ../../nixos/modules;
diff --git a/morph/lib/issuer.nix b/morph/lib/issuer.nix
index b2d48c2cee8cb914d7c3c9acedfcaf9ad2997178..69b0527cd74e0752ded6ffbe7513db126f0613f5 100644
--- a/morph/lib/issuer.nix
+++ b/morph/lib/issuer.nix
@@ -3,7 +3,7 @@
{ lib, config, ...}:
let
inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnEndpoint monitoringvpnIPv4;
- inherit (config.grid.issuer) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
+ inherit (config.grid.issuer) issuerDomains allowedChargeOrigins;
in {
imports = [
../../nixos/modules/monitoring/vpn/client.nix
@@ -11,14 +11,6 @@ in {
];
options.grid.issuer = {
- letsEncryptAdminEmail = lib.mkOption {
- type = lib.types.str;
- description = ''
- A string giving an email address to use for Let's Encrypt registration and
- certificate issuance.
- '';
- };
-
issuerDomains = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
@@ -82,7 +74,8 @@ in {
stripeSecretKeyPath = config.deployment.secrets.stripe-secret-key.destination;
database = "SQLite3";
databasePath = "${config.fileSystems."zkapissuer-data".mountPoint}/vouchers.sqlite3";
- inherit letsEncryptAdminEmail allowedChargeOrigins;
+ inherit (config.grid) letsEncryptAdminEmail;
+ inherit allowedChargeOrigins;
domains = issuerDomains;
};
diff --git a/morph/lib/monitoring.nix b/morph/lib/monitoring.nix
index e10dd31c4dee7404d436285ad80c9babc350c246..0cedcf6498e485a69075c964ce3d4064baeadcae 100644
--- a/morph/lib/monitoring.nix
+++ b/morph/lib/monitoring.nix
@@ -1,37 +1,166 @@
-# Similar to ``issuer.nix`` but for a "monitoring"-type system. Holes are
-# filled by ``customize-monitoring.nix``.
-{ config, ...}:
+# This contains all of the NixOS system configuration necessary to specify an
+# "monitoring"-type system.
+{ lib, config, nodes, ...}:
let
- inherit (config.grid) privateKeyPath;
+ cfg = config.grid.monitoring;
+ inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4 letsEncryptAdminEmail;
+
+ # This collects information about monitored hosts from their configuration for use below.
+ monitoringHosts = lib.mapAttrsToList (name: node: rec {
+ inherit name;
+ vpnIPv4 = node.config.grid.monitoringvpnIPv4;
+ vpnHostName = "${name}.monitoringvpn";
+ hostNames = [name vpnHostName];
+ }) nodes;
+
+ # A set mapping VPN IP addresses as strings to lists of hostnames as
+ # strings. The system's ``/etc/hosts`` will be populated with this
+ # information. Apart from helping with normal forward resolution, this
+ # *also* gives us reverse resolution from the VPN IPs to hostnames which
+ # allows Grafana to show us hostnames instead of VPN IP addresses.
+ hostsMap = lib.listToAttrs (map (node: lib.nameValuePair node.vpnIPv4 node.hostNames) monitoringHosts);
+ # A list of VPN IP addresses as strings indicating which clients will be
+ # allowed onto the VPN.
+ vpnClientIPs = lib.remove monitoringvpnIPv4 (map (node: node.vpnIPv4) monitoringHosts);
+ # A list of VPN clients (IP addresses or hostnames) as strings indicating
+ # which nodes to scrape "nodeExporter" metrics from.
+ nodeExporterTargets = map (node: node.name) monitoringHosts;
in {
- deployment = {
- secrets = {
- "monitoringvpn-private-key" = {
+ imports = [
+ ../../nixos/modules/monitoring/vpn/server.nix
+ ../../nixos/modules/monitoring/server/grafana.nix
+ ../../nixos/modules/monitoring/server/prometheus.nix
+ ../../nixos/modules/monitoring/exporters/node.nix
+ ../../nixos/modules/monitoring/exporters/blackbox.nix
+ # Loki 0.3.0 from Nixpkgs 19.09 is too old and does not work:
+ # ../../nixos/modules/monitoring/server/loki.nix
+ ];
+
+ options.grid.monitoring = {
+ paymentExporterTargets = lib.mkOption {
+ type = lib.types.listOf lib.types.str;
+ description = ''
+ A list of VPN clients (IP addresses or hostnames) as strings indicating
+ which nodes to scrape PaymentServer metrics from.
+ '';
+ };
+
+ blackboxExporterHttpsTargets = lib.mkOption {
+ type = lib.types.listOf lib.types.str;
+ description = ''
+ A list of HTTPS servers (URLs, IP addresses or hostnames) as strings indicating
+ which nodes the BlackboxExporter should scrape HTTP and TLS metrics from.
+ '';
+ };
+
+ monitoringDomains = lib.mkOption {
+ type = lib.types.listOf lib.types.str;
+ description = ''
+ A list of strings giving the domain names that point at this monitoring
+ system. These will all be included in Let's Encrypt certificate.
+ '';
+ };
+
+
+ googleOAuthClientID = lib.mkOption {
+ type = lib.types.str;
+ default = "";
+ description = ''
+ A string containing the GSuite OAuth2 ClientID to use to authenticate
+ logins to Grafana.
+ '';
+ };
+
+ enableSlackAlert = lib.mkOption {
+ type = lib.types.bool;
+ default = false;
+ description = ''
+ Whether to enable alerting via Slack.
+ When true requires a grafana-slack-url file (see private-keys/README.rst).
+ '';
+ };
+ };
+
+ config = {
+ assertions = [
+ {
+ assertion = let
+ vpnIPs = (map (node: node.vpnIPv4) monitoringHosts);
+ in vpnIPs == lib.unique vpnIPs;
+ message = ''
+ Duplicate grid.monitoringvpnIPv4 values specified for different nodes.
+ '';
+ }
+ ];
+
+ deployment.secrets = lib.mkMerge [
+ {
+ "monitoringvpn-private-key" = {
destination = "/run/keys/monitoringvpn/server.key";
source = "${privateKeyPath}/monitoringvpn/server.key";
owner.user = "root";
owner.group = "root";
permissions = "0400";
action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
- };
- "monitoringvpn-preshared-key" = {
+ };
+ "monitoringvpn-preshared-key" = {
destination = "/run/keys/monitoringvpn/preshared.key";
source = "${privateKeyPath}/monitoringvpn/preshared.key";
owner.user = "root";
owner.group = "root";
permissions = "0400";
action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
- };
+ };
+ }
+ (lib.mkIf (cfg.googleOAuthClientID != "") {
+ "grafana-google-sso-secret" = {
+ source = "${privateKeyPath}/grafana-google-sso.secret";
+ destination = "/run/keys/grafana-google-sso.secret";
+ owner.user = config.systemd.services.grafana.serviceConfig.User;
+ owner.group = config.users.users.grafana.group;
+ permissions = "0400";
+ action = ["sudo" "systemctl" "restart" "grafana.service"];
+ };
+ "grafana-admin-password" = {
+ source = "${privateKeyPath}/grafana-admin.password";
+ destination = "/run/keys/grafana-admin.password";
+ owner.user = config.systemd.services.grafana.serviceConfig.User;
+ owner.group = config.users.users.grafana.group;
+ permissions = "0400";
+ action = ["sudo" "systemctl" "restart" "grafana.service"];
+ };
+ })
+ (lib.mkIf cfg.enableSlackAlert {
+ "grafana-slack-url" = {
+ source = "${privateKeyPath}/grafana-slack-url";
+ destination = "/run/keys/grafana-slack-url";
+ owner.user = config.systemd.services.grafana.serviceConfig.User;
+ owner.group = config.users.users.grafana.group;
+ permissions = "0400";
+ action = ["sudo" "systemctl" "restart" "grafana.service"];
+ };
+ })
+ ];
+
+ networking.hosts = hostsMap;
+
+ services.private-storage.monitoring.vpn.server = {
+ enable = true;
+ ip = monitoringvpnIPv4;
+ inherit vpnClientIPs;
+ pubKeysPath = "${publicKeyPath}/monitoringvpn";
};
- };
- imports = [
- ../../nixos/modules/monitoring/vpn/server.nix
- ../../nixos/modules/monitoring/server/grafana.nix
- ../../nixos/modules/monitoring/server/prometheus.nix
- ../../nixos/modules/monitoring/exporters/node.nix
- ../../nixos/modules/monitoring/exporters/blackbox.nix
- # Loki 0.3.0 from Nixpkgs 19.09 is too old and does not work:
- # ../../nixos/modules/monitoring/server/loki.nix
- ];
+ services.private-storage.monitoring.prometheus = {
+ inherit nodeExporterTargets;
+ inherit (cfg) paymentExporterTargets blackboxExporterHttpsTargets;
+ nginxExporterTargets = [];
+ };
+
+ services.private-storage.monitoring.grafana = {
+ inherit (cfg) googleOAuthClientID enableSlackAlert ;
+ inherit letsEncryptAdminEmail;
+ domains = cfg.monitoringDomains;
+ };
+ };
}