From c3bf2c32510ef7ba295c630d3df49ddc9c739347 Mon Sep 17 00:00:00 2001
From: Tom Prince <tom.prince@private.storage>
Date: Wed, 19 Jan 2022 15:57:09 -0700
Subject: [PATCH] monitor per-node
---
morph/grid/local/grid.nix | 3 +--
morph/grid/production/grid.nix | 3 +--
morph/grid/testing/grid.nix | 3 +--
morph/lib/issuer.nix | 1 +
morph/lib/monitored-node.nix | 6 +++++
morph/lib/monitoring.nix | 15 ++++---------
.../modules/monitoring/server/prometheus.nix | 22 +++++--------------
7 files changed, 19 insertions(+), 34 deletions(-)
diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix
index da8a8381..15fbb010 100644
--- a/morph/grid/local/grid.nix
+++ b/morph/grid/local/grid.nix
@@ -121,7 +121,7 @@ let
grid.monitoringvpnIPv4 = "172.23.23.1";
grid.publicIPv4 = "192.168.56.24";
grid.monitoring = {
- inherit paymentExporterTargets blackboxExporterHttpsTargets;
+ inherit blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
enableSlackAlert = false;
@@ -131,7 +131,6 @@ let
};
# TBD: derive these automatically:
- paymentExporterTargets = [ "payments" ];
blackboxExporterHttpsTargets = [
# "https://private.storage/"
# "https://payments.private.storage/"
diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix
index ab45d4ba..3a309dc8 100644
--- a/morph/grid/production/grid.nix
+++ b/morph/grid/production/grid.nix
@@ -53,7 +53,7 @@ let
config = {
grid.monitoringvpnIPv4 = "172.23.23.1";
grid.monitoring = {
- inherit paymentExporterTargets blackboxExporterHttpsTargets;
+ inherit blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
enableSlackAlert = true;
@@ -114,7 +114,6 @@ let
storage005 = { vpnIP = "172.23.23.25"; stateVersion = "19.03"; };
};
- paymentExporterTargets = [ "payments" ];
blackboxExporterHttpsTargets = [
"https://private.storage/"
"https://www.private.storage/"
diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix
index 19839ae8..01ba9f02 100644
--- a/morph/grid/testing/grid.nix
+++ b/morph/grid/testing/grid.nix
@@ -69,7 +69,7 @@ let
config = {
grid.monitoringvpnIPv4 = "172.23.23.1";
grid.monitoring = {
- inherit paymentExporterTargets blackboxExporterHttpsTargets;
+ inherit blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
enableSlackAlert = true;
@@ -79,7 +79,6 @@ let
};
# TBD: derive these automatically:
- paymentExporterTargets = [ "payments" ];
blackboxExporterHttpsTargets = [
"https://privatestorage-staging.com/"
"https://www.privatestorage-staging.com/"
diff --git a/morph/lib/issuer.nix b/morph/lib/issuer.nix
index 1f715171..428a23bf 100644
--- a/morph/lib/issuer.nix
+++ b/morph/lib/issuer.nix
@@ -48,6 +48,7 @@ in {
};
};
};
+ grid.monitoring.httpsScrapePorts = [ 443 ];
services.private-storage-issuer = {
enable = true;
tls = true;
diff --git a/morph/lib/monitored-node.nix b/morph/lib/monitored-node.nix
index b4e1f47e..8739dced 100644
--- a/morph/lib/monitored-node.nix
+++ b/morph/lib/monitored-node.nix
@@ -2,6 +2,12 @@
let
inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnEndpoint monitoringvpnIPv4;
in {
+ options.grid.monitoring = {
+ httpsScrapePorts = lib.mkOption {
+ default = [];
+ # TODO
+ };
+ };
config = {
deployment = {
secrets = {
diff --git a/morph/lib/monitoring.nix b/morph/lib/monitoring.nix
index 7115efba..3c98a80a 100644
--- a/morph/lib/monitoring.nix
+++ b/morph/lib/monitoring.nix
@@ -10,6 +10,7 @@ let
inherit name;
vpnIPv4 = node.config.grid.monitoringvpnIPv4;
vpnHostName = "${name}.monitoringvpn";
+ httpsScrapeTargets = map (port: "${vpnHostName}:${toString port}") node.config.grid.monitoring.httpsScrapePorts or [];
hostNames = [name vpnHostName];
}) nodes;
@@ -25,6 +26,7 @@ let
# A list of VPN clients (IP addresses or hostnames) as strings indicating
# which nodes to scrape "nodeExporter" metrics from.
nodeExporterTargets = map (node: node.name) monitoringHosts;
+ httpsExporterTargets = lib.concatMap (node: node.httpsScrapeTargets) monitoringHosts;
in {
imports = [
../../nixos/modules/monitoring/vpn/server.nix
@@ -36,14 +38,6 @@ in {
];
options.grid.monitoring = {
- paymentExporterTargets = lib.mkOption {
- type = lib.types.listOf lib.types.str;
- description = ''
- A list of VPN clients (IP addresses or hostnames) as strings indicating
- which nodes to scrape PaymentServer metrics from.
- '';
- };
-
blackboxExporterHttpsTargets = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
@@ -151,9 +145,8 @@ in {
};
services.private-storage.monitoring.prometheus = {
- inherit nodeExporterTargets;
- inherit (cfg) paymentExporterTargets blackboxExporterHttpsTargets;
- nginxExporterTargets = [];
+ inherit nodeExporterTargets httpsExporterTargets;
+ inherit (cfg) blackboxExporterHttpsTargets;
};
services.private-storage.monitoring.grafana = {
diff --git a/nixos/modules/monitoring/server/prometheus.nix b/nixos/modules/monitoring/server/prometheus.nix
index 3bb00a5b..799a12cc 100644
--- a/nixos/modules/monitoring/server/prometheus.nix
+++ b/nixos/modules/monitoring/server/prometheus.nix
@@ -21,15 +21,10 @@ in {
example = [ "node1" "node2" ];
description = "List of nodes (hostnames or IPs) to scrape.";
};
- nginxExporterTargets = lib.mkOption {
+ httpsExporterTargets = lib.mkOption {
type = with lib.types; listOf str;
- example = [ "node1" "node2" ];
- description = "List of nodes (hostnames or IPs) to scrape.";
- };
- paymentExporterTargets = lib.mkOption {
- type = with lib.types; listOf str;
- example = [ "node1" "node2" ];
- description = "List of nodes (hostnames or IPs) to scrape.";
+ example = [ "node1:443" "node2:9090" ];
+ description = "List of nodes (hostnames or IPs) and ports to scrape.";
};
blackboxExporterHttpsTargets = lib.mkOption {
type = with lib.types; listOf str;
@@ -53,20 +48,13 @@ in {
relabel_configs = [ dropPortNumber ];
}
{
- job_name = "nginx-exporters";
- static_configs = [{
- targets = map (x: x + ":" + (toString exportersCfg.nginx.port)) cfg.nginxExporterTargets;
- }];
- relabel_configs = [ dropPortNumber ];
- }
- {
- job_name = "payment-exporters";
+ job_name = "https-exporters";
scheme = "https";
tls_config.insecure_skip_verify = true;
static_configs = [{
# Explicitly setting the port number so the relabel_config can filter it out again.
# Leaving it out makes the port number show in Grafana.
- targets = map (x: x + ":443") cfg.paymentExporterTargets;
+ targets = cfg.httpsExporterTargets;
}];
relabel_configs = [ dropPortNumber ];
}
--
GitLab