prometheus.nix

# Prometheus server
#
# Scope: Pull data from our cluster machines into TSDB
# See https://christine.website/blog/prometheus-grafana-loki-nixos-2020-11-20

{ config, lib, ... }:
let

  exportersCfg = config.services.prometheus.exporters;
  cfg = config.services.private-storage.monitoring.prometheus;
  dropPortNumber = {
    source_labels = [ "__address__" ];
    regex = "^(.*)(?:\\.monitoringvpn):\\d+$";
    target_label = "instance";
  };
  logRetention = toString(config.services.private-storage.monitoring.policy.logRetentionSeconds) + "s";

in {
  options.services.private-storage.monitoring.prometheus = {
    nodeExporterTargets = lib.mkOption {
      type = with lib.types; listOf str;
      example = [ "node1" "node2" ];
      description = "List of nodes (hostnames or IPs) to scrape.";
    };
    nginxExporterTargets = lib.mkOption {
      type = with lib.types; listOf str;
      example = [ "node1" "node2" ];
      description = "List of nodes (hostnames or IPs) to scrape.";
    };
    paymentExporterTargets = lib.mkOption {
      type = with lib.types; listOf str;
      example = [ "node1" "node2" ];
      description = "List of nodes (hostnames or IPs) to scrape.";
    };
    blackboxExporterHttpsTargets = lib.mkOption {
      type = with lib.types; listOf str;
      example = [ "https://node1.com/" "https://node2.org/" ];
      description = "List of https URLs to scrape.";
    };
  };

  config = rec {
    # networking.firewall.allowedTCPPorts = [ services.prometheus.port ];

    services.prometheus = {
      enable = true;
      # port = 9090; # Option only in recent (20.09?) nixpkgs, 9090 default
      retentionTime = logRetention;
      scrapeConfigs = [
        {
          job_name = "node-exporters";
          static_configs = [{
            targets = map (x: x + ":" + (toString exportersCfg.node.port)) cfg.nodeExporterTargets;
          }];
          relabel_configs = [ dropPortNumber ];
        }
        {
          job_name = "nginx-exporters";
          static_configs = [{
            targets = map (x: x + ":" + (toString exportersCfg.nginx.port)) cfg.nginxExporterTargets;
          }];
          relabel_configs = [ dropPortNumber ];
        }
        {
          job_name = "payment-exporters";
          scheme = "https";
          tls_config.insecure_skip_verify = true;
          static_configs = [{
            # Explicitly setting the port number so the relabel_config can filter it out again.
            # Leaving it out makes the port number show in Grafana.
            targets = map (x: x + ":443") cfg.paymentExporterTargets;
          }];
          relabel_configs = [ dropPortNumber ];
        }
        {
          # The Blackbox exporter is using Prometheus' "Multi-Target Exporter Pattern",
          # see https://prometheus.io/docs/guides/multi-target-exporter/
          job_name = "blackboxExporterHttps";
          static_configs = [{
            targets = cfg.blackboxExporterHttpsTargets;
          }];
          metrics_path = "/probe";
          params.module = [ "https_2xx" ];
          relabel_configs = [
            {
              source_labels = [ "__address__" ];
              target_label = "__param_target";
            }
            {
              source_labels = [ "__param_target" ];
              target_label = "instance";
            }
            {
              source_labels = [];
              target_label = "__address__";
              # The blackbox exporter’s real hostname:port
              replacement = "monitoring:9115";
            }
          ];
        }
      ];
    };
  };
}