# Grafana Server # # Scope: Beautiful plots of time series data retrieved from Prometheus # See https://christine.website/blog/prometheus-grafana-loki-nixos-2020-11-20 { config, lib, ... }: let cfg = config.services.private-storage.monitoring.grafana; in { options.services.private-storage.monitoring.grafana = { domains = lib.mkOption { type = lib.types.listOf lib.types.str; example = [ "grafana.grid.private.storage" ]; description = "The domain names at which the server is reachable."; }; prometheusUrl = lib.mkOption { type = lib.types.str; example = "http://prometheus:9090/"; default = "http://localhost:9090/"; description = "The URL of the Prometheus host to access"; }; lokiUrl = lib.mkOption { type = lib.types.str; example = "http://loki:3100/"; default = "http://localhost:3100/"; description = "The URL of the Loki host to access"; }; letsEncryptAdminEmail = lib.mkOption { type = lib.types.str; description = '' An email address to give to Let's Encrypt as an operational contact for the service's TLS certificate. ''; }; googleOAuthClientID = lib.mkOption { type = lib.types.str; example = "grafana-staging-345678"; default = "replace-by-your-client-id-or-set-empty-string-for-anonymous-access"; description = "The GSuite OAuth2 SSO Client ID. Empty string turns SSO auth off and anonymous (free for all) access on."; }; googleOAuthClientSecretFile = lib.mkOption { type = lib.types.path; example = /var/secret/monitoring-gsuite-client-secret; default = /run/keys/grafana-google-sso.secret; description = "The path to the GSuite SSO secret file."; }; adminPasswordFile = lib.mkOption { type = lib.types.path; example = "/var/secret/monitoring-admin-password"; default = /run/keys/grafana-admin.password; description = "A file containing the password for the Grafana Admin account."; }; enableSlackAlert = lib.mkOption { type = lib.types.bool; default = false; description = '' Enables the slack alerter. Expects a file that contains the secret Slack Web Hook URL in grafanaSlackUrlFile (see below). ''; }; grafanaSlackUrlFile = lib.mkOption { type = lib.types.path; default = /run/keys/grafana-slack-url; description = '' Where to find the file that containts the slack URL. ''; }; enableZulipAlert = lib.mkOption { type = lib.types.bool; default = false; description = '' Enables the Zulip alerter. Expects a file that contains the secret Zulip Web Hook URL in grafanaZulipUrlFile (see below). ''; }; grafanaZulipUrlFile = lib.mkOption { type = lib.types.path; default = /run/keys/grafana-zulip-url; description = '' Where to find the file that containts the Zulip URL. ''; }; }; config = let # We'll refer to this collection of domains by the first domain in the list. domain = builtins.head cfg.domains; in { # Port 80 for ACME ssl retrieval only. 443 for nginx -> grafana. networking.firewall.allowedTCPPorts = [ 80 443 ]; services.grafana = { enable = true; settings = { server = { domain = "${toString domain}"; http_port = 2342; http_addr = "127.0.0.1"; # Defend against DNS rebinding attacks. enforce_domain = true; # Force Grafana to believe it is reachable via https on the default port # number because that's where the nginx that forwards traffic to it is # listening. Grafana's own server listens on an internal address that # doesn't matter to anyone except our nginx instance. root_url = "https://%(domain)s/"; }; # No phoning home analytics.reporting_enabled = false; # Same time zone for all users by default date_formats.default_timezone = "UTC"; # The auth sections since NixOS 22.11 are named a bit funky with a dot in the name # # https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/grafana/#anonymous-authentication # https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/google/ "auth.anonymous" = lib.mkIf (cfg.googleOAuthClientID == "") { enabled = true; org_role = "Admin"; org_name = "Main Org."; }; "auth.google" = lib.mkIf (cfg.googleOAuthClientID != "") { enabled = true; # Grafana considers it "sign up" to let in a user it has # never seen before. allow_sign_up = true; client_secret = "$__file{${toString cfg.googleOAuthClientSecretFile}}"; client_id = cfg.googleOAuthClientID; }; # Give users that come through GSuite SSO the highest possible privileges: users.auto_assign_org_role = "Editor"; # Read the admin password from a file in our secrets folder: security.admin_password = "$__file{${toString cfg.adminPasswordFile}}"; }; provision = { enable = true; # See https://grafana.com/docs/grafana/latest/administration/provisioning/#datasources datasources.settings.datasources = [{ name = "Prometheus"; type = "prometheus"; uid = "LocalPrometheus"; access = "proxy"; url = cfg.prometheusUrl; isDefault = true; } { name = "Loki"; type = "loki"; uid = "LocalLoki"; access = "proxy"; url = cfg.lokiUrl; }]; # See https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards dashboards.settings.providers = [{ name = "provisioned"; options.path = ./grafana-dashboards; }]; # See https://grafana.com/docs/grafana/latest/alerting/set-up/provision-alerting-resources/file-provisioning/#provision-contact-points alerting.contactPoints.settings.contactPoints = [ ] ++ (lib.optionals (cfg.enableSlackAlert) [{ uid = "slack-notifier-1"; name = "Slack"; type = "slack"; is_default = true; send_reminder = false; settings = { username = "${domain}"; uploadImage = true; }; secure_settings = { # `$__file{}` reads the value from the named file. # See https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider url = "$__file{${toString cfg.grafanaSlackUrlFile}}"; }; }]) ++ (lib.optionals (cfg.enableZulipAlert) [{ # See https://zulip.com/integrations/doc/grafana uid = "zulip-notifier-1"; name = "Zulip"; type = "webhook"; is_default = true; send_reminder = false; settings = { url = "$__file{${toString cfg.grafanaZulipUrlFile}}"; }; }]); }; }; # nginx reverse proxy security.acme.defaults.email = cfg.letsEncryptAdminEmail; security.acme.acceptTerms = true; services.nginx = { enable = true; recommendedGzipSettings = true; recommendedOptimisation = true; recommendedProxySettings = true; recommendedTlsSettings = true; # Only allow PFS-enabled ciphers with AES256: sslCiphers = "AES256+EECDH:AES256+EDH:!aNULL"; virtualHosts."${domain}" = { serverAliases = builtins.tail cfg.domains; enableACME = true; forceSSL = true; locations."/" = { proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}"; proxyWebsockets = true; }; locations."/metrics" = { # Only allow our monitoringvpn subnet # And localhost since we're the monitoring server currently extraConfig = '' allow ${config.grid.monitoringvpnIPv4}/24; allow 127.0.0.1; allow ::1; deny all; ''; proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}"; }; }; }; # Let Grafana read from keys, if necessary. users.users.grafana.extraGroups = [ "keys" ]; }; }