# This contains all of the NixOS system configuration necessary to specify an # "monitoring"-type system. { lib, config, nodes, ...}: let cfg = config.grid.monitoring; inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4 letsEncryptAdminEmail; # This collects information about monitored hosts from their configuration for use below. monitoringHosts = lib.mapAttrsToList (name: node: rec { inherit name; vpnIPv4 = node.config.grid.monitoringvpnIPv4; vpnHostName = "${name}.monitoringvpn"; hostNames = [name vpnHostName]; }) nodes; # A set mapping VPN IP addresses as strings to lists of hostnames as # strings. The system's ``/etc/hosts`` will be populated with this # information. Apart from helping with normal forward resolution, this # *also* gives us reverse resolution from the VPN IPs to hostnames which # allows Grafana to show us hostnames instead of VPN IP addresses. hostsMap = lib.listToAttrs (map (node: lib.nameValuePair node.vpnIPv4 node.hostNames) monitoringHosts); # A list of VPN IP addresses as strings indicating which clients will be # allowed onto the VPN. vpnClientIPs = lib.remove monitoringvpnIPv4 (map (node: node.vpnIPv4) monitoringHosts); # A list of VPN clients (IP addresses or hostnames) as strings indicating # which nodes to scrape "nodeExporter" metrics from. nodeExporterTargets = map (node: node.vpnHostName) monitoringHosts; in { imports = [ ../../nixos/modules/monitoring/vpn/server.nix ../../nixos/modules/monitoring/server/grafana.nix ../../nixos/modules/monitoring/server/prometheus.nix ../../nixos/modules/monitoring/exporters/node.nix ../../nixos/modules/monitoring/exporters/blackbox.nix # Loki 0.3.0 from Nixpkgs 19.09 is too old and does not work: # ../../nixos/modules/monitoring/server/loki.nix ]; options.grid.monitoring = { paymentExporterTargets = lib.mkOption { type = lib.types.listOf lib.types.str; description = '' A list of VPN clients (IP addresses or hostnames) as strings indicating which nodes to scrape PaymentServer metrics from. ''; }; blackboxExporterHttpsTargets = lib.mkOption { type = lib.types.listOf lib.types.str; description = '' A list of HTTPS servers (URLs, IP addresses or hostnames) as strings indicating which nodes the BlackboxExporter should scrape HTTP and TLS metrics from. ''; }; monitoringDomains = lib.mkOption { type = lib.types.listOf lib.types.str; description = '' A list of strings giving the domain names that point at this monitoring system. These will all be included in Let's Encrypt certificate. ''; }; googleOAuthClientID = lib.mkOption { type = lib.types.str; default = ""; description = '' A string containing the GSuite OAuth2 ClientID to use to authenticate logins to Grafana. ''; }; enableSlackAlert = lib.mkOption { type = lib.types.bool; default = false; description = '' Whether to enable alerting via Slack. When true requires a grafana-slack-url file (see private-keys/README.rst). ''; }; }; config = { assertions = [ { assertion = let vpnIPs = (map (node: node.vpnIPv4) monitoringHosts); in vpnIPs == lib.unique vpnIPs; message = '' Duplicate grid.monitoringvpnIPv4 values specified for different nodes. ''; } ]; deployment.secrets = lib.mkMerge [ { "monitoringvpn-private-key" = { destination = "/run/keys/monitoringvpn/server.key"; source = "${privateKeyPath}/monitoringvpn/server.key"; owner.user = "root"; owner.group = "root"; permissions = "0400"; action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; }; "monitoringvpn-preshared-key" = { destination = "/run/keys/monitoringvpn/preshared.key"; source = "${privateKeyPath}/monitoringvpn/preshared.key"; owner.user = "root"; owner.group = "root"; permissions = "0400"; action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"]; }; "grafana-admin-password" = { source = "${privateKeyPath}/grafana-admin.password"; destination = "/run/keys/grafana-admin.password"; owner.user = config.systemd.services.grafana.serviceConfig.User; owner.group = config.users.users.grafana.group; permissions = "0400"; action = ["sudo" "systemctl" "restart" "grafana.service"]; }; } (lib.mkIf (cfg.googleOAuthClientID != "") { "grafana-google-sso-secret" = { source = "${privateKeyPath}/grafana-google-sso.secret"; destination = "/run/keys/grafana-google-sso.secret"; owner.user = config.systemd.services.grafana.serviceConfig.User; owner.group = config.users.users.grafana.group; permissions = "0400"; action = ["sudo" "systemctl" "restart" "grafana.service"]; }; }) (lib.mkIf cfg.enableSlackAlert { "grafana-slack-url" = { source = "${privateKeyPath}/grafana-slack-url"; destination = "/run/keys/grafana-slack-url"; owner.user = config.systemd.services.grafana.serviceConfig.User; owner.group = config.users.users.grafana.group; permissions = "0400"; action = ["sudo" "systemctl" "restart" "grafana.service"]; }; }) ]; networking.hosts = hostsMap; services.private-storage.monitoring.vpn.server = { enable = true; ip = monitoringvpnIPv4; inherit vpnClientIPs; pubKeysPath = "${publicKeyPath}/monitoringvpn"; }; services.private-storage.monitoring.prometheus = { inherit nodeExporterTargets; inherit (cfg) paymentExporterTargets blackboxExporterHttpsTargets; nginxExporterTargets = []; }; services.private-storage.monitoring.grafana = { inherit (cfg) googleOAuthClientID enableSlackAlert ; inherit letsEncryptAdminEmail; domains = cfg.monitoringDomains; }; }; }