promtail.nix

# Promtail log forwarder configuration
#
# Scope: Tail logs on the local system and send them to Loki
#
# Description: This is not strictly an "exporter" like the Prometheus
#              exporters, but it is very similar in what it is doing -
#              preparing local data and sending it off to a TSDB.

{ config, options, lib, ... }:

let
  cfg = config.services.private-storage.monitoring.exporters.promtail;
  hostName = config.networking.hostName;
  logRetention = toString(config.services.private-storage.monitoring.policy.logRetentionSeconds) + "s";

in {
  options.services.private-storage.monitoring.exporters.promtail = {
    enable = lib.mkEnableOption "Promtail log exporter service";
    lokiUrl = lib.mkOption {
      type = lib.types.str;
      description = ''
        The server URL that logs should be pushed to.
      '';
      # Resolving names is hard, let's have breakfast
      # If you are curious why there's a plain IP address in here, read all of
      # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/251
      # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/257
      # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/258
      default = "http://172.23.23.1:3100/loki/api/v1/push";
    };
  };

  config = lib.mkIf cfg.enable {
    services.promtail.enable = true;
    networking.firewall.interfaces.monitoringvpn.allowedTCPPorts = [ 9080 ];

    # Since we'll send our journald logs elsewhere, we don't need to keep them
    # here for very long.  Keep them for a *little* while just to provide some
    # context in case someone ends up looking at the logs on the system itself
    # but generally suppose that people will look at Loki instead.
    services.journald.extraConfig = ''
      # This tells journald it can discard log files that contain only log
      # entries that are older than 29 days.
      MaxRetentionSec=${logRetention}

      # This tells journald to start a new log file once a day.  Together with
      # the MaxRetentionSec setting, this means that entries are kept for
      # between 29 and 30 days (plus whatever scheduling slop journald has in
      # enforcing these limits).
      #
      # https://www.freedesktop.org/software/systemd/man/journald.conf.html
      # for further details about these options.
      #
      # A maximum retention of 30 days conforms to the published log retention
      # policy.
      MaxFileSec=1day
    '';

    services.promtail.configuration = {
      server = {
        http_listen_port = 9080; # Using /metrics for health check
        grpc_listen_address = "127.0.0.1"; # unused, but no option to turn it off.
        grpc_listen_port = 9094; # unused, but no option to turn it off.
      };

      clients = [{
          url = cfg.lokiUrl;
      }];

      scrape_configs = [{
        job_name = "systemd-journal";
        journal = {
          labels = {
            job = "systemd-journal";
            host = hostName;
          };
        };
        # The journal has many internal labels, that by default will
        # be dropped because of their "__" prefix.  To keep them, rename them.
        # https://grafana.com/docs/loki/latest/clients/promtail/scraping/#journal-scraping-linux-only
        # https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
        relabel_configs = [{
          source_labels = [ "__journal__systemd_unit" ];
          target_label = "unit";
        }];
      }];
    };
  };
}