promtail.nix

# Promtail log forwarder configuration
#
# Scope: Tail logs on the local system and send them to Loki
#
# Description: This is not strictly an "exporter" like the Prometheus
#              exporters, but it is very similar in what it is doing -
#              preparing local data and sending it off to a TSDB.

{ config, options, lib, ... }:

let
  cfg = config.services.private-storage.monitoring.exporters.promtail;
  hostName = config.networking.hostName;

in {
  options.services.private-storage.monitoring.exporters.promtail = {
    enable = lib.mkEnableOption "Promtail log exporter service";
    lokiUrl = lib.mkOption {
      type = lib.types.str;
      description = ''
        The server URL that logs should be pushed to.
      '';
      # Resolving names is hard, let's have breakfast
      # If you are curious why there's a plain IP address in here, read all of
      # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/251
      # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/257
      # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/258
      default = "http://172.23.23.1:3100/loki/api/v1/push";
    };
  };

  config = lib.mkIf cfg.enable {
    services.promtail.enable = true;
    networking.firewall.interfaces.monitoringvpn.allowedTCPPorts = [ 9080 ];
    services.promtail.configuration = {
      server = {
        http_listen_port = 9080; # Using /metrics for health check
        grpc_listen_address = "127.0.0.1"; # unused, but no option to turn it off.
        grpc_listen_port = 9094; # unused, but no option to turn it off.
      };

      clients = [{
          url = cfg.lokiUrl;
      }];

      scrape_configs = [{
        job_name = "systemd-journal";
        journal = {
          labels = {
            job = "systemd-journal";
            host = hostName;
          };
        };
        # The journal has many internal labels, that by default will
        # be dropped because of their "__" prefix.  To keep them, rename them.
        # https://grafana.com/docs/loki/latest/clients/promtail/scraping/#journal-scraping-linux-only
        # https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
        relabel_configs = [{
          source_labels = [ "__journal__systemd_unit" ];
          target_label = "unit";
        }];
      }];
    };
  };
}