From 3082fc5e21a0abf1d03688b88b3fff18e64bf587 Mon Sep 17 00:00:00 2001
From: Florian Sesser <florian@privatestorage.io>
Date: Sun, 30 May 2021 14:47:08 +0000
Subject: [PATCH] Add monitoring to production deployment (WIP)
---
morph/grid/production/config.json | 1 +
morph/grid/production/grid.nix | 47 +++++++++++++++++++++++++++++++
morph/lib/make-storage.nix | 12 ++++++++
3 files changed, 60 insertions(+)
diff --git a/morph/grid/production/config.json b/morph/grid/production/config.json
index ec60acc7..80cbfa23 100644
--- a/morph/grid/production/config.json
+++ b/morph/grid/production/config.json
@@ -1,6 +1,7 @@
{ "publicStoragePort": 8898
, "ristrettoSigningKeyPath": "../../PrivateStorageSecrets/ristretto.signing-key"
, "stripeSecretKeyPath": "../../PrivateStorageSecrets/stripe.secret"
+, "monitoringvpnSecretKeyDir": "../../PrivateStorageSecrets/monitoringvpn"
, "passValue": 1000000
, "issuerDomain": "payments.privatestorage.io"
, "letsEncryptAdminEmail": "jean-paul@privatestorage.io"
diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix
index 7c9abe14..13d26eff 100644
--- a/morph/grid/production/grid.nix
+++ b/morph/grid/production/grid.nix
@@ -7,6 +7,35 @@ import ../../lib/make-grid.nix {
nodes = cfg:
let
sshUsers = import ../../../../PrivateStorageSecrets/production-users.nix;
+
+ # TBD: derive these automatically:
+ hostsMap = {
+ "172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ];
+ "172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
+ "172.23.23.21" = [ "storage001" "storage001.monitoringvpn" ];
+ "172.23.23.22" = [ "storage002" "storage002.monitoringvpn" ];
+ "172.23.23.23" = [ "storage003" "storage003.monitoringvpn" ];
+ "172.23.23.24" = [ "storage004" "storage004.monitoringvpn" ];
+ "172.23.23.25" = [ "storage005" "storage005.monitoringvpn" ];
+ };
+ vpnClientIPs = [
+ "172.23.23.11"
+ "172.23.23.21"
+ "172.23.23.22"
+ "172.23.23.23"
+ "172.23.23.24"
+ "172.23.23.25"
+ ];
+ nodeExporterTargets = [
+ "monitoring"
+ "payments"
+ "storage001"
+ "storage002"
+ "storage003"
+ "storage004"
+ "storage005"
+ ];
+
in {
# Here are the hosts that are in this morph network. This is sort of like
# a server manifest. We try to keep as many of the specific details as
@@ -22,6 +51,7 @@ import ../../lib/make-grid.nix {
# The names must be unique!
"payments.privatestorage.io" = import ../../lib/make-issuer.nix ({
publicIPv4 = "18.184.142.208";
+ monitoringvpnIPv4 = "172.23.23.11";
inherit sshUsers;
hardware = ../../lib/issuer-aws.nix;
stateVersion = "19.03";
@@ -32,30 +62,47 @@ import ../../lib/make-grid.nix {
inherit sshUsers;
hardware = ./storage001-hardware.nix;
stateVersion = "19.09";
+ monitoringvpnIPv4 = "172.23.23.21";
} // cfg);
"storage002" = import ../../lib/make-storage.nix ({
cfg = import ./storage002-config.nix;
inherit sshUsers;
hardware = ./storage002-hardware.nix;
stateVersion = "19.09";
+ monitoringvpnIPv4 = "172.23.23.22";
} // cfg);
"storage003" = import ../../lib/make-storage.nix ({
cfg = import ./storage003-config.nix;
inherit sshUsers;
hardware = ./storage003-hardware.nix;
stateVersion = "19.09";
+ monitoringvpnIPv4 = "172.23.23.23";
} // cfg);
"storage004" = import ../../lib/make-storage.nix ({
cfg = import ./storage004-config.nix;
inherit sshUsers;
hardware = ./storage004-hardware.nix;
stateVersion = "19.09";
+ monitoringvpnIPv4 = "172.23.23.24";
} // cfg);
"storage005" = import ../../lib/make-storage.nix ({
cfg = import ./storage005-config.nix;
inherit sshUsers;
hardware = ./storage005-hardware.nix;
stateVersion = "19.03";
+ monitoringvpnIPv4 = "172.23.23.25";
+ } // cfg);
+
+ "monitoring" = import ../../lib/make-monitoring.nix ({
+ publicIPv4 = "monitoring.private.storage"; # XXX TBD when the machine is online
+ monitoringvpnIPv4 = "172.23.23.1";
+ inherit vpnClientIPs;
+ inherit hostsMap;
+ inherit nodeExporterTargets;
+ nginxExporterTargets = [ ];
+ hardware = ../../lib/issuer-aws.nix;
+ stateVersion = "19.09";
+ inherit sshUsers;
} // cfg);
};
}
diff --git a/morph/lib/make-storage.nix b/morph/lib/make-storage.nix
index af0867c8..1c000e11 100644
--- a/morph/lib/make-storage.nix
+++ b/morph/lib/make-storage.nix
@@ -3,6 +3,7 @@
, hardware # The path to the hardware configuration for this node.
, publicStoragePort # The storage port number on which to accept connections.
, ristrettoSigningKeyPath # The *local* path to the Ristretto signing key file.
+, monitoringvpnSecretKeyDir # The directory that holds the VPN keys.
, passValue # Bytes component of size×time value of passes.
, sshUsers # Users for which to configure SSH access to this node.
, stateVersion # The value for system.stateVersion on this node.
@@ -11,6 +12,7 @@
# to avoid breaking some software such as
# database servers. You should change this only
# after NixOS release notes say you should.
+, monitoringvpnIPv4 # This node's IP in the monitoring VPN.
, ...
}: rec {
deployment = {
@@ -40,6 +42,10 @@
# Bring in our module for configuring the Tahoe-LAFS service and other
# Private Storage-specific things.
../../nixos/modules/private-storage.nix
+ # Connect to the monitoringvpn.
+ ../../nixos/modules/monitoring/vpn/client.nix
+ # Expose base system metrics over the monitoringvpn.
+ ../../nixos/modules/monitoring/exporters/node.nix
];
# Pass the configuration specific to this host to the 100TB module to be
@@ -67,4 +73,10 @@
};
system.stateVersion = stateVersion;
+
+ services.private-storage.monitoring.vpn.client = {
+ enable = true;
+ ip = monitoringvpnIPv4;
+ endpoint = "monitoring.private.storage:51820"; # XXX TBD when the machine is online
+ };
}
--
GitLab