From c3bf2c32510ef7ba295c630d3df49ddc9c739347 Mon Sep 17 00:00:00 2001 From: Tom Prince <tom.prince@private.storage> Date: Wed, 19 Jan 2022 15:57:09 -0700 Subject: [PATCH] monitor per-node --- morph/grid/local/grid.nix | 3 +-- morph/grid/production/grid.nix | 3 +-- morph/grid/testing/grid.nix | 3 +-- morph/lib/issuer.nix | 1 + morph/lib/monitored-node.nix | 6 +++++ morph/lib/monitoring.nix | 15 ++++--------- .../modules/monitoring/server/prometheus.nix | 22 +++++-------------- 7 files changed, 19 insertions(+), 34 deletions(-) diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix index da8a8381..15fbb010 100644 --- a/morph/grid/local/grid.nix +++ b/morph/grid/local/grid.nix @@ -121,7 +121,7 @@ let grid.monitoringvpnIPv4 = "172.23.23.1"; grid.publicIPv4 = "192.168.56.24"; grid.monitoring = { - inherit paymentExporterTargets blackboxExporterHttpsTargets; + inherit blackboxExporterHttpsTargets; inherit (grid-config) monitoringDomains; googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; enableSlackAlert = false; @@ -131,7 +131,6 @@ let }; # TBD: derive these automatically: - paymentExporterTargets = [ "payments" ]; blackboxExporterHttpsTargets = [ # "https://private.storage/" # "https://payments.private.storage/" diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix index ab45d4ba..3a309dc8 100644 --- a/morph/grid/production/grid.nix +++ b/morph/grid/production/grid.nix @@ -53,7 +53,7 @@ let config = { grid.monitoringvpnIPv4 = "172.23.23.1"; grid.monitoring = { - inherit paymentExporterTargets blackboxExporterHttpsTargets; + inherit blackboxExporterHttpsTargets; inherit (grid-config) monitoringDomains; googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; enableSlackAlert = true; @@ -114,7 +114,6 @@ let storage005 = { vpnIP = "172.23.23.25"; stateVersion = "19.03"; }; }; - paymentExporterTargets = [ "payments" ]; blackboxExporterHttpsTargets = [ "https://private.storage/" "https://www.private.storage/" diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix index 19839ae8..01ba9f02 100644 --- a/morph/grid/testing/grid.nix +++ b/morph/grid/testing/grid.nix @@ -69,7 +69,7 @@ let config = { grid.monitoringvpnIPv4 = "172.23.23.1"; grid.monitoring = { - inherit paymentExporterTargets blackboxExporterHttpsTargets; + inherit blackboxExporterHttpsTargets; inherit (grid-config) monitoringDomains; googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; enableSlackAlert = true; @@ -79,7 +79,6 @@ let }; # TBD: derive these automatically: - paymentExporterTargets = [ "payments" ]; blackboxExporterHttpsTargets = [ "https://privatestorage-staging.com/" "https://www.privatestorage-staging.com/" diff --git a/morph/lib/issuer.nix b/morph/lib/issuer.nix index 1f715171..428a23bf 100644 --- a/morph/lib/issuer.nix +++ b/morph/lib/issuer.nix @@ -48,6 +48,7 @@ in { }; }; }; + grid.monitoring.httpsScrapePorts = [ 443 ]; services.private-storage-issuer = { enable = true; tls = true; diff --git a/morph/lib/monitored-node.nix b/morph/lib/monitored-node.nix index b4e1f47e..8739dced 100644 --- a/morph/lib/monitored-node.nix +++ b/morph/lib/monitored-node.nix @@ -2,6 +2,12 @@ let inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnEndpoint monitoringvpnIPv4; in { + options.grid.monitoring = { + httpsScrapePorts = lib.mkOption { + default = []; + # TODO + }; + }; config = { deployment = { secrets = { diff --git a/morph/lib/monitoring.nix b/morph/lib/monitoring.nix index 7115efba..3c98a80a 100644 --- a/morph/lib/monitoring.nix +++ b/morph/lib/monitoring.nix @@ -10,6 +10,7 @@ let inherit name; vpnIPv4 = node.config.grid.monitoringvpnIPv4; vpnHostName = "${name}.monitoringvpn"; + httpsScrapeTargets = map (port: "${vpnHostName}:${toString port}") node.config.grid.monitoring.httpsScrapePorts or []; hostNames = [name vpnHostName]; }) nodes; @@ -25,6 +26,7 @@ let # A list of VPN clients (IP addresses or hostnames) as strings indicating # which nodes to scrape "nodeExporter" metrics from. nodeExporterTargets = map (node: node.name) monitoringHosts; + httpsExporterTargets = lib.concatMap (node: node.httpsScrapeTargets) monitoringHosts; in { imports = [ ../../nixos/modules/monitoring/vpn/server.nix @@ -36,14 +38,6 @@ in { ]; options.grid.monitoring = { - paymentExporterTargets = lib.mkOption { - type = lib.types.listOf lib.types.str; - description = '' - A list of VPN clients (IP addresses or hostnames) as strings indicating - which nodes to scrape PaymentServer metrics from. - ''; - }; - blackboxExporterHttpsTargets = lib.mkOption { type = lib.types.listOf lib.types.str; description = '' @@ -151,9 +145,8 @@ in { }; services.private-storage.monitoring.prometheus = { - inherit nodeExporterTargets; - inherit (cfg) paymentExporterTargets blackboxExporterHttpsTargets; - nginxExporterTargets = []; + inherit nodeExporterTargets httpsExporterTargets; + inherit (cfg) blackboxExporterHttpsTargets; }; services.private-storage.monitoring.grafana = { diff --git a/nixos/modules/monitoring/server/prometheus.nix b/nixos/modules/monitoring/server/prometheus.nix index 3bb00a5b..799a12cc 100644 --- a/nixos/modules/monitoring/server/prometheus.nix +++ b/nixos/modules/monitoring/server/prometheus.nix @@ -21,15 +21,10 @@ in { example = [ "node1" "node2" ]; description = "List of nodes (hostnames or IPs) to scrape."; }; - nginxExporterTargets = lib.mkOption { + httpsExporterTargets = lib.mkOption { type = with lib.types; listOf str; - example = [ "node1" "node2" ]; - description = "List of nodes (hostnames or IPs) to scrape."; - }; - paymentExporterTargets = lib.mkOption { - type = with lib.types; listOf str; - example = [ "node1" "node2" ]; - description = "List of nodes (hostnames or IPs) to scrape."; + example = [ "node1:443" "node2:9090" ]; + description = "List of nodes (hostnames or IPs) and ports to scrape."; }; blackboxExporterHttpsTargets = lib.mkOption { type = with lib.types; listOf str; @@ -53,20 +48,13 @@ in { relabel_configs = [ dropPortNumber ]; } { - job_name = "nginx-exporters"; - static_configs = [{ - targets = map (x: x + ":" + (toString exportersCfg.nginx.port)) cfg.nginxExporterTargets; - }]; - relabel_configs = [ dropPortNumber ]; - } - { - job_name = "payment-exporters"; + job_name = "https-exporters"; scheme = "https"; tls_config.insecure_skip_verify = true; static_configs = [{ # Explicitly setting the port number so the relabel_config can filter it out again. # Leaving it out makes the port number show in Grafana. - targets = map (x: x + ":443") cfg.paymentExporterTargets; + targets = cfg.httpsExporterTargets; }]; relabel_configs = [ dropPortNumber ]; } -- GitLab