Skip to content
Snippets Groups Projects
Commit 572ccb7a authored by Tom Prince's avatar Tom Prince
Browse files

Merge branch 'generate-vpn-config' into 'develop' 

Collect some monitoring host details automatically.

See merge request privatestorage/PrivateStorageio!234
parents 6bb58ac8 172f6c54
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
morph/grid/local/grid.nix
+ 9
− 20
View file @ 572ccb7a
......@@ -59,7 +59,7 @@ let
grid = {
publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
inherit (grid-config) monitoringvpnEndpoint;
inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
};
# Configure deployment management authorization for all systems in the grid.
services.private-storage.deployment = {
......@@ -77,7 +77,7 @@ let
grid.monitoringvpnIPv4 = "172.23.23.11";
grid.publicIPv4 = "192.168.67.21";
grid.issuer = {
inherit (grid-config) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
inherit (grid-config) issuerDomains allowedChargeOrigins;
};
};
};
......@@ -115,33 +115,22 @@ let
monitoring = {
imports = [
gridlib.monitoring
(gridlib.customize-monitoring {
inherit hostsMap vpnClientIPs
nodeExporterTargets
paymentExporterTargets
blackboxExporterHttpsTargets;
inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
enableSlackAlert = false;
stateVersion = "19.09";
})
grid-module
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.1";
grid.publicIPv4 = "192.168.67.24";
grid.monitoring = {
inherit paymentExporterTargets blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
enableSlackAlert = false;
};
system.stateVersion = "19.09";
};
};
# TBD: derive these automatically:
hostsMap = {
"172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ];
"172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
"172.23.23.12" = [ "storage1" "storage1.monitoringvpn" ];
"172.23.23.13" = [ "storage2" "storage2.monitoringvpn" ];
};
vpnClientIPs = [ "172.23.23.11" "172.23.23.12" "172.23.23.13" ];
nodeExporterTargets = [ "monitoring" "payments" "storage1" "storage2" ];
paymentExporterTargets = [ "payments" ];
blackboxExporterHttpsTargets = [
# "https://private.storage/"
......
morph/grid/production/grid.nix
+ 9
− 39
View file @ 572ccb7a
......@@ -21,7 +21,7 @@ let
grid = {
publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
inherit (grid-config) monitoringvpnEndpoint;
inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
};
# Configure deployment management authorization for all systems in the grid.
services.private-storage.deployment = {
......@@ -39,7 +39,7 @@ let
config = {
grid.monitoringvpnIPv4 = "172.23.23.11";
grid.issuer = {
inherit (grid-config) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
inherit (grid-config) issuerDomains allowedChargeOrigins;
};
};
};
......@@ -48,20 +48,17 @@ let
imports = [
gridlib.monitoring
gridlib.hardware-aws
(gridlib.customize-monitoring {
inherit hostsMap vpnClientIPs
nodeExporterTargets
paymentExporterTargets
blackboxExporterHttpsTargets;
inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
enableSlackAlert = true;
stateVersion = "19.09";
})
grid-module
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.1";
grid.monitoring = {
inherit paymentExporterTargets blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
enableSlackAlert = true;
};
system.stateVersion = "19.09";
};
};
......@@ -117,33 +114,6 @@ let
storage005 = { vpnIP = "172.23.23.25"; stateVersion = "19.03"; };
};
# TBD: derive these automatically:
hostsMap = {
"172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ];
"172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
"172.23.23.21" = [ "storage001" "storage001.monitoringvpn" ];
"172.23.23.22" = [ "storage002" "storage002.monitoringvpn" ];
"172.23.23.23" = [ "storage003" "storage003.monitoringvpn" ];
"172.23.23.24" = [ "storage004" "storage004.monitoringvpn" ];
"172.23.23.25" = [ "storage005" "storage005.monitoringvpn" ];
};
vpnClientIPs = [
"172.23.23.11"
"172.23.23.21"
"172.23.23.22"
"172.23.23.23"
"172.23.23.24"
"172.23.23.25"
];
nodeExporterTargets = [
"monitoring"
"payments"
"storage001"
"storage002"
"storage003"
"storage004"
"storage005"
];
paymentExporterTargets = [ "payments" ];
blackboxExporterHttpsTargets = [
"https://private.storage/"
......
morph/grid/testing/grid.nix
+ 9
− 19
View file @ 572ccb7a
......@@ -21,7 +21,7 @@ let
grid = {
publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
inherit (grid-config) monitoringvpnEndpoint;
inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
};
# Configure deployment management authorization for all systems in the grid.
services.private-storage.deployment = {
......@@ -39,7 +39,7 @@ let
config = {
grid.monitoringvpnIPv4 = "172.23.23.11";
grid.issuer = {
inherit (grid-config) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
inherit (grid-config) issuerDomains allowedChargeOrigins;
};
};
};
......@@ -64,31 +64,21 @@ let
imports = [
gridlib.monitoring
gridlib.hardware-aws
(gridlib.customize-monitoring {
inherit hostsMap vpnClientIPs
nodeExporterTargets
paymentExporterTargets
blackboxExporterHttpsTargets;
inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
enableSlackAlert = true;
stateVersion = "19.09";
})
grid-module
];
config = {
grid.monitoringvpnIPv4 = "172.23.23.1";
grid.monitoring = {
inherit paymentExporterTargets blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
enableSlackAlert = true;
};
system.stateVersion = "19.09";
};
};
# TBD: derive these automatically:
hostsMap = {
"172.23.23.1" = [ "monitoring" "monitoring.monitoringvpn" ];
"172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
"172.23.23.12" = [ "storage001" "storage001.monitoringvpn" ];
};
vpnClientIPs = [ "172.23.23.11" "172.23.23.12" ];
nodeExporterTargets = [ "monitoring" "payments" "storage001" ];
paymentExporterTargets = [ "payments" ];
blackboxExporterHttpsTargets = [
"https://privatestorage-staging.com/"
......
morph/lib/base.nix
+ 8
− 0
View file @ 572ccb7a
......@@ -30,6 +30,14 @@
The domain name and port of the monitoring VPN endpoint.
'';
};
letsEncryptAdminEmail = lib.mkOption {
type = lib.types.str;
description = ''
A string giving an email address to use for Let's Encrypt registration and
certificate issuance.
'';
};
};
# Any extra NixOS modules to load on all our servers. Note that just
......
morph/lib/customize-monitoring.nix deleted 100644 → 0
+ 0
− 126
View file @ 6bb58ac8
# Define a function which returns a value which fills in all the holes left by
# ``monitoring.nix``.
{
# A set mapping VPN IP addresses as strings to lists of hostnames as
# strings. The system's ``/etc/hosts`` will be populated with this
# information. Apart from helping with normal forward resolution, this
# *also* gives us reverse resolution from the VPN IPs to hostnames which
# allows Grafana to show us hostnames instead of VPN IP addresses.
hostsMap
# A string giving an email address to use for Let's Encrypt registration and
# certificate issuance.
, letsEncryptAdminEmail
# A list of strings giving the domain names that point at this monitoring
# system. These will all be included in Let's Encrypt certificate.
, monitoringDomains
# A list of VPN IP addresses as strings indicating which clients will be
# allowed onto the VPN.
, vpnClientIPs
# A list of VPN clients (IP addresses or hostnames) as strings indicating
# which nodes to scrape "nodeExporter" metrics from.
, nodeExporterTargets
# A list of VPN clients (IP addresses or hostnames) as strings indicating
# which nodes to scrape "nginxExporter" metrics from.
, nginxExporterTargets ? []
# A list of VPN clients (IP addresses or hostnames) as strings indicating
# which nodes to scrape PaymentServer metrics from.
, paymentExporterTargets ? []
# A list of HTTPS servers (URLs, IP addresses or hostnames) as strings indicating
# which nodes the BlackboxExporter should scrape HTTP and TLS metrics from.
, blackboxExporterHttpsTargets ? []
# A string containing the GSuite OAuth2 ClientID to use to authenticate
# logins to Grafana.
, googleOAuthClientID
# Whether to enable alerting via Slack.
# When true requires a grafana-slack-url file (see private-keys/README.rst).
, enableSlackAlert ? false
# A string giving the NixOS state version for the system.
, stateVersion
, ...
}:
{ config, ... }:
let
inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4;
in {
deployment.secrets = let
# When Grafana SSO is disabled there is not necessarily any client secret
# available. Avoid telling morph that there is one in this case (so it
# avoids trying to read it and then failing). Even if the secret did
# exist, if SSO is disabled there's no point sending the secret to the
# server.
#
# Also, we have to define this whole secret here so that we can configure
# it completely or not at all. morph gets angry if we half configure it
# (say, by just omitting the "source" value).
grafanaSSO =
if googleOAuthClientID == ""
then { }
else {
"grafana-google-sso-secret" = {
source = "${privateKeyPath}/grafana-google-sso.secret";
destination = "/run/keys/grafana-google-sso.secret";
owner.user = config.systemd.services.grafana.serviceConfig.User;
owner.group = config.users.users.grafana.group;
permissions = "0400";
action = ["sudo" "systemctl" "restart" "grafana.service"];
};
"grafana-admin-password" = {
source = "${privateKeyPath}/grafana-admin.password";
destination = "/run/keys/grafana-admin.password";
owner.user = config.systemd.services.grafana.serviceConfig.User;
owner.group = config.users.users.grafana.group;
permissions = "0400";
action = ["sudo" "systemctl" "restart" "grafana.service"];
};
};
grafanaSlackUrl =
if !enableSlackAlert
then { }
else {
"grafana-slack-url" = {
source = "${privateKeyPath}/grafana-slack-url";
destination = "/run/keys/grafana-slack-url";
owner.user = config.systemd.services.grafana.serviceConfig.User;
owner.group = config.users.users.grafana.group;
permissions = "0400";
action = ["sudo" "systemctl" "restart" "grafana.service"];
};
};
in
grafanaSSO // grafanaSlackUrl;
networking.hosts = hostsMap;
services.private-storage.monitoring.vpn.server = {
enable = true;
ip = monitoringvpnIPv4;
inherit vpnClientIPs;
pubKeysPath = "${publicKeyPath}/monitoringvpn";
};
services.private-storage.monitoring.prometheus = {
inherit nodeExporterTargets;
inherit nginxExporterTargets;
inherit paymentExporterTargets;
inherit blackboxExporterHttpsTargets;
};
services.private-storage.monitoring.grafana = {
inherit letsEncryptAdminEmail;
inherit googleOAuthClientID;
inherit enableSlackAlert;
domains = monitoringDomains;
};
system.stateVersion = stateVersion;
}
morph/lib/default.nix
+ 0
− 2
View file @ 572ccb7a
......@@ -9,9 +9,7 @@
issuer = import ./issuer.nix;
storage = import ./storage.nix;
monitoring = import ./monitoring.nix;
customize-monitoring = import ./customize-monitoring.nix;
modules = builtins.toString ../../nixos/modules;
......
morph/lib/issuer.nix
+ 3
− 10
View file @ 572ccb7a
......@@ -3,7 +3,7 @@
{ lib, config, ...}:
let
inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnEndpoint monitoringvpnIPv4;
inherit (config.grid.issuer) letsEncryptAdminEmail issuerDomains allowedChargeOrigins;
inherit (config.grid.issuer) issuerDomains allowedChargeOrigins;
in {
imports = [
../../nixos/modules/monitoring/vpn/client.nix
......@@ -11,14 +11,6 @@ in {
];
options.grid.issuer = {
letsEncryptAdminEmail = lib.mkOption {
type = lib.types.str;
description = ''
A string giving an email address to use for Let's Encrypt registration and
certificate issuance.
'';
};
issuerDomains = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
......@@ -82,7 +74,8 @@ in {
stripeSecretKeyPath = config.deployment.secrets.stripe-secret-key.destination;
database = "SQLite3";
databasePath = "${config.fileSystems."zkapissuer-data".mountPoint}/vouchers.sqlite3";
inherit letsEncryptAdminEmail allowedChargeOrigins;
inherit (config.grid) letsEncryptAdminEmail;
inherit allowedChargeOrigins;
domains = issuerDomains;
};
......
morph/lib/monitoring.nix
+ 149
− 20
View file @ 572ccb7a
# Similar to ``issuer.nix`` but for a "monitoring"-type system. Holes are
# filled by ``customize-monitoring.nix``.
{ config, ...}:
# This contains all of the NixOS system configuration necessary to specify an
# "monitoring"-type system.
{ lib, config, nodes, ...}:
let
inherit (config.grid) privateKeyPath;
cfg = config.grid.monitoring;
inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4 letsEncryptAdminEmail;
# This collects information about monitored hosts from their configuration for use below.
monitoringHosts = lib.mapAttrsToList (name: node: rec {
inherit name;
vpnIPv4 = node.config.grid.monitoringvpnIPv4;
vpnHostName = "${name}.monitoringvpn";
hostNames = [name vpnHostName];
}) nodes;
# A set mapping VPN IP addresses as strings to lists of hostnames as
# strings. The system's ``/etc/hosts`` will be populated with this
# information. Apart from helping with normal forward resolution, this
# *also* gives us reverse resolution from the VPN IPs to hostnames which
# allows Grafana to show us hostnames instead of VPN IP addresses.
hostsMap = lib.listToAttrs (map (node: lib.nameValuePair node.vpnIPv4 node.hostNames) monitoringHosts);
# A list of VPN IP addresses as strings indicating which clients will be
# allowed onto the VPN.
vpnClientIPs = lib.remove monitoringvpnIPv4 (map (node: node.vpnIPv4) monitoringHosts);
# A list of VPN clients (IP addresses or hostnames) as strings indicating
# which nodes to scrape "nodeExporter" metrics from.
nodeExporterTargets = map (node: node.name) monitoringHosts;
in {
deployment = {
secrets = {
"monitoringvpn-private-key" = {
imports = [
../../nixos/modules/monitoring/vpn/server.nix
../../nixos/modules/monitoring/server/grafana.nix
../../nixos/modules/monitoring/server/prometheus.nix
../../nixos/modules/monitoring/exporters/node.nix
../../nixos/modules/monitoring/exporters/blackbox.nix
# Loki 0.3.0 from Nixpkgs 19.09 is too old and does not work:
# ../../nixos/modules/monitoring/server/loki.nix
];
options.grid.monitoring = {
paymentExporterTargets = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
A list of VPN clients (IP addresses or hostnames) as strings indicating
which nodes to scrape PaymentServer metrics from.
'';
};
blackboxExporterHttpsTargets = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
A list of HTTPS servers (URLs, IP addresses or hostnames) as strings indicating
which nodes the BlackboxExporter should scrape HTTP and TLS metrics from.
'';
};
monitoringDomains = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
A list of strings giving the domain names that point at this monitoring
system. These will all be included in Let's Encrypt certificate.
'';
};
googleOAuthClientID = lib.mkOption {
type = lib.types.str;
default = "";
description = ''
A string containing the GSuite OAuth2 ClientID to use to authenticate
logins to Grafana.
'';
};
enableSlackAlert = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Whether to enable alerting via Slack.
When true requires a grafana-slack-url file (see private-keys/README.rst).
'';
};
};
config = {
assertions = [
{
assertion = let
vpnIPs = (map (node: node.vpnIPv4) monitoringHosts);
in vpnIPs == lib.unique vpnIPs;
message = ''
Duplicate grid.monitoringvpnIPv4 values specified for different nodes.
'';
}
];
deployment.secrets = lib.mkMerge [
{
"monitoringvpn-private-key" = {
destination = "/run/keys/monitoringvpn/server.key";
source = "${privateKeyPath}/monitoringvpn/server.key";
owner.user = "root";
owner.group = "root";
permissions = "0400";
action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
};
"monitoringvpn-preshared-key" = {
};
"monitoringvpn-preshared-key" = {
destination = "/run/keys/monitoringvpn/preshared.key";
source = "${privateKeyPath}/monitoringvpn/preshared.key";
owner.user = "root";
owner.group = "root";
permissions = "0400";
action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
};
};
}
(lib.mkIf (cfg.googleOAuthClientID != "") {
"grafana-google-sso-secret" = {
source = "${privateKeyPath}/grafana-google-sso.secret";
destination = "/run/keys/grafana-google-sso.secret";
owner.user = config.systemd.services.grafana.serviceConfig.User;
owner.group = config.users.users.grafana.group;
permissions = "0400";
action = ["sudo" "systemctl" "restart" "grafana.service"];
};
"grafana-admin-password" = {
source = "${privateKeyPath}/grafana-admin.password";
destination = "/run/keys/grafana-admin.password";
owner.user = config.systemd.services.grafana.serviceConfig.User;
owner.group = config.users.users.grafana.group;
permissions = "0400";
action = ["sudo" "systemctl" "restart" "grafana.service"];
};
})
(lib.mkIf cfg.enableSlackAlert {
"grafana-slack-url" = {
source = "${privateKeyPath}/grafana-slack-url";
destination = "/run/keys/grafana-slack-url";
owner.user = config.systemd.services.grafana.serviceConfig.User;
owner.group = config.users.users.grafana.group;
permissions = "0400";
action = ["sudo" "systemctl" "restart" "grafana.service"];
};
})
];
networking.hosts = hostsMap;
services.private-storage.monitoring.vpn.server = {
enable = true;
ip = monitoringvpnIPv4;
inherit vpnClientIPs;
pubKeysPath = "${publicKeyPath}/monitoringvpn";
};
};
imports = [
../../nixos/modules/monitoring/vpn/server.nix
../../nixos/modules/monitoring/server/grafana.nix
../../nixos/modules/monitoring/server/prometheus.nix
../../nixos/modules/monitoring/exporters/node.nix
../../nixos/modules/monitoring/exporters/blackbox.nix
# Loki 0.3.0 from Nixpkgs 19.09 is too old and does not work:
# ../../nixos/modules/monitoring/server/loki.nix
];
services.private-storage.monitoring.prometheus = {
inherit nodeExporterTargets;
inherit (cfg) paymentExporterTargets blackboxExporterHttpsTargets;
nginxExporterTargets = [];
};
services.private-storage.monitoring.grafana = {
inherit (cfg) googleOAuthClientID enableSlackAlert ;
inherit letsEncryptAdminEmail;
domains = cfg.monitoringDomains;
};
};
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment