monitoring.nix

# This contains all of the NixOS system configuration necessary to specify an
# "monitoring"-type system.
{ lib, config, nodes, ...}:
let
  cfg = config.grid.monitoring;
  inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4 letsEncryptAdminEmail;

  # This collects information about monitored hosts from their configuration for use below.
  monitoringHosts = lib.mapAttrsToList (name: node: rec {
    inherit name;
    vpnIPv4 = node.config.grid.monitoringvpnIPv4;
    vpnHostName = "${name}.monitoringvpn";
    hostNames = [name vpnHostName];
  }) nodes;

  # A set mapping VPN IP addresses as strings to lists of hostnames as
  # strings.  The system's ``/etc/hosts`` will be populated with this
  # information.  Apart from helping with normal forward resolution, this
  # *also* gives us reverse resolution from the VPN IPs to hostnames which
  # allows Grafana to show us hostnames instead of VPN IP addresses.
  hostsMap = lib.listToAttrs (map (node: lib.nameValuePair node.vpnIPv4 node.hostNames) monitoringHosts);
  # A list of VPN IP addresses as strings indicating which clients will be
  # allowed onto the VPN.
  vpnClientIPs = lib.remove monitoringvpnIPv4 (map (node: node.vpnIPv4) monitoringHosts);
  # A list of VPN clients (IP addresses or hostnames) as strings indicating
  # which nodes to scrape "nodeExporter" metrics from.
  nodeExporterTargets = map (node: node.vpnHostName) monitoringHosts;
in {
  imports = [
    ../../nixos/modules/monitoring/vpn/server.nix
    ../../nixos/modules/monitoring/server/grafana.nix
    ../../nixos/modules/monitoring/server/prometheus.nix
    ../../nixos/modules/monitoring/server/loki.nix
    ../../nixos/modules/monitoring/exporters/blackbox.nix
  ];

  options.grid.monitoring = {
    paymentExporterTargets = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      description = ''
        A list of VPN clients (IP addresses or hostnames) as strings indicating
        which nodes to scrape PaymentServer metrics from.
      '';
    };

    blackboxExporterHttpsTargets = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      description = ''
        A list of HTTPS servers (URLs, IP addresses or hostnames) as strings indicating
        which nodes the BlackboxExporter should scrape HTTP and TLS metrics from.
      '';
    };

    monitoringDomains = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      description = ''
        A list of strings giving the domain names that point at this monitoring
        system.  These will all be included in Let's Encrypt certificate.
      '';
    };

    googleOAuthClientID = lib.mkOption {
      type = lib.types.str;
      default = "";
      description = ''
        A string containing the GSuite OAuth2 ClientID to use to authenticate
        logins to Grafana.
      '';
    };

    enableSlackAlert = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Whether to enable alerting via Slack.
        When true requires a grafana-slack-url file (see private-keys/README.rst).
      '';
    };
  };

  config = {
    assertions = [
      {
        assertion = let
          vpnIPs = (map (node: node.vpnIPv4) monitoringHosts);
        in vpnIPs == lib.unique vpnIPs;
        message = ''
          Duplicate grid.monitoringvpnIPv4 values specified for different nodes.
        '';
      }
    ];

    deployment.secrets = lib.mkMerge [
      {
        "monitoringvpn-private-key" = {
        destination = "/run/keys/monitoringvpn/server.key";
        source = "${privateKeyPath}/monitoringvpn/server.key";
        owner.user = "root";
        owner.group = "root";
        permissions = "0400";
        action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
        };
        "monitoringvpn-preshared-key" = {
        destination = "/run/keys/monitoringvpn/preshared.key";
        source = "${privateKeyPath}/monitoringvpn/preshared.key";
        owner.user = "root";
        owner.group = "root";
        permissions = "0400";
        action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
        };
        "grafana-admin-password" = {
          source = "${privateKeyPath}/grafana-admin.password";
          destination = "/run/keys/grafana-admin.password";
          owner.user = config.systemd.services.grafana.serviceConfig.User;
          owner.group = config.users.users.grafana.group;
          permissions = "0400";
          action = ["sudo" "systemctl" "restart" "grafana.service"];
        };
      }
      (lib.mkIf (cfg.googleOAuthClientID != "") {
        "grafana-google-sso-secret" = {
          source = "${privateKeyPath}/grafana-google-sso.secret";
          destination = "/run/keys/grafana-google-sso.secret";
          owner.user = config.systemd.services.grafana.serviceConfig.User;
          owner.group = config.users.users.grafana.group;
          permissions = "0400";
          action = ["sudo" "systemctl" "restart" "grafana.service"];
        };
      })
      (lib.mkIf cfg.enableSlackAlert {
        "grafana-slack-url" = {
          source = "${privateKeyPath}/grafana-slack-url";
          destination = "/run/keys/grafana-slack-url";
          owner.user = config.systemd.services.grafana.serviceConfig.User;
          owner.group = config.users.users.grafana.group;
          permissions = "0400";
          action = ["sudo" "systemctl" "restart" "grafana.service"];
        };
      })
    ];

    networking.hosts = hostsMap;

    services.private-storage.monitoring.vpn.server = {
      enable = true;
      ip = monitoringvpnIPv4;
      inherit vpnClientIPs;
      pubKeysPath = "${publicKeyPath}/monitoringvpn";
    };

    services.private-storage.monitoring.prometheus = {
      inherit nodeExporterTargets;
      inherit (cfg) paymentExporterTargets blackboxExporterHttpsTargets;
      nginxExporterTargets = [];
    };

    services.private-storage.monitoring.grafana = {
      inherit (cfg) googleOAuthClientID enableSlackAlert;
      inherit letsEncryptAdminEmail;
      domains = cfg.monitoringDomains;
    };

    services.private-storage.monitoring.exporters.node.enable = true;
  };
}