nixos/modules/monitoring/server/grafana.nix

@@ -7,18 +7,9 @@
 
 let
   cfg = config.services.private-storage.monitoring.grafana;
-  grafanaAuth = if (cfg.googleOAuthClientID == "") then {
-                  anonymous.enable = true;
-                } else {
-                  google.enable = true;
-                  # Grafana considers it "sign up" to let in a user it has
-                  # never seen before.
-                  google.allowSignUp = true;
-                  google.clientSecretFile = cfg.googleOAuthClientSecretFile;
-                  google.clientId = cfg.googleOAuthClientID;
-                };
 
 in {
+
   options.services.private-storage.monitoring.grafana = {
     domains = lib.mkOption
     { type = lib.types.listOf lib.types.str;
@@ -77,6 +68,21 @@ in {
         Where to find the file that containts the slack URL.
       '';
     };
+    enableZulipAlert = lib.mkOption
+    { type = lib.types.bool;
+      default = false;
+      description = ''
+        Enables the Zulip alerter. Expects a file that contains
+        the secret Zulip Web Hook URL in grafanaZulipUrlFile (see below).
+      '';
+    };
+    grafanaZulipUrlFile = lib.mkOption
+    { type = lib.types.path;
+      default = /run/keys/grafana-zulip-url;
+      description = ''
+        Where to find the file that containts the Zulip URL.
+      '';
+    };
   };
 
   config =
@@ -90,59 +96,80 @@ in {
 
     services.grafana = {
       enable = true;
-      inherit domain; 
-      port = 2342;
-      addr = "127.0.0.1";
 
-      # No phoning home
-      analytics.reporting.enable = false;
+      settings = {
+
+        server = {
+          domain = "${toString domain}";
+          http_port = 2342;
+          http_addr = "127.0.0.1";
+
+          # Defend against DNS rebinding attacks.
+          enforce_domain = true;
 
           # Force Grafana to believe it is reachable via https on the default port
           # number because that's where the nginx that forwards traffic to it is
           # listening.  Grafana's own server listens on an internal address that
           # doesn't matter to anyone except our nginx instance.
-      rootUrl = "https://%(domain)s/";
+          root_url = "https://%(domain)s/";
+        };
+
+        # No phoning home
+        analytics.reporting_enabled = false;
 
-      extraOptions = {
-        # Defend against DNS rebinding attacks.
-        SERVER_ENFORCE_DOMAIN = "true";
         # Same time zone for all users by default
-        DATE_FORMATS_DEFAULT_TIMEZONE = "UTC";
-      };
+        date_formats.default_timezone = "UTC";
 
-      auth = {
-        anonymous.org_role = "Admin";
-        anonymous.org_name = "Main Org.";
-      } // grafanaAuth;
+        # The auth sections since NixOS 22.11 are named a bit funky with a dot in the name
+        #
+        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/grafana/#anonymous-authentication
+        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/google/
+        "auth.anonymous" = lib.mkIf (cfg.googleOAuthClientID == "") {
+          enabled = true;
+          org_role = "Admin";
+          org_name = "Main Org.";
+        };
+        "auth.google" = lib.mkIf (cfg.googleOAuthClientID != "") {
+          enabled = true;
+          # Grafana considers it "sign up" to let in a user it has
+          # never seen before.
+          allow_sign_up = true;
+          client_secret = "$__file{${toString cfg.googleOAuthClientSecretFile}}";
+          client_id = cfg.googleOAuthClientID;
+        };
 
         # Give users that come through GSuite SSO the highest possible privileges:
-      users.autoAssignOrgRole = "Editor";
+        users.auto_assign_org_role = "Editor";
 
         # Read the admin password from a file in our secrets folder:
-      security.adminPasswordFile = cfg.adminPasswordFile;
+        security.admin_password = "$__file{${toString cfg.adminPasswordFile}}";
+      };
 
       provision = {
         enable = true;
         # See https://grafana.com/docs/grafana/latest/administration/provisioning/#datasources
-        datasources = [{
+        datasources.settings.datasources = [{
           name = "Prometheus";
           type = "prometheus";
+          uid = "LocalPrometheus";
           access = "proxy";
           url = cfg.prometheusUrl;
           isDefault = true;
         } {
           name = "Loki";
           type = "loki";
+          uid = "LocalLoki";
           access = "proxy";
           url = cfg.lokiUrl;
         }];
         # See https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards
-        dashboards = [{
+        dashboards.settings.providers = [{
           name = "provisioned";
           options.path = ./grafana-dashboards;
         }];
-        # See https://grafana.com/docs/grafana/latest/administration/provisioning/#example-alert-notification-channels-config-file
-        notifiers = [ ] ++ (lib.optionals (cfg.enableSlackAlert) [{
+        # See https://grafana.com/docs/grafana/latest/alerting/set-up/provision-alerting-resources/file-provisioning/#provision-contact-points
+        alerting.contactPoints.settings.contactPoints =
+        [ ] ++ (lib.optionals (cfg.enableSlackAlert) [{
           uid = "slack-notifier-1";
           name = "Slack";
           type = "slack";
@@ -157,12 +184,22 @@ in {
             # See https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider
             url = "$__file{${toString cfg.grafanaSlackUrlFile}}";
           };
+        }]) ++ (lib.optionals (cfg.enableZulipAlert) [{
+          # See https://zulip.com/integrations/doc/grafana
+          uid = "zulip-notifier-1";
+          name = "Zulip";
+          type = "webhook";
+          is_default = true;
+          send_reminder = false;
+          settings = {
+            url = "$__file{${toString cfg.grafanaZulipUrlFile}}";
+          };
         }]);
       };
     };
 
     # nginx reverse proxy
-    security.acme.email = cfg.letsEncryptAdminEmail;
+    security.acme.defaults.email = cfg.letsEncryptAdminEmail;
     security.acme.acceptTerms = true;
     services.nginx = {
       enable = true;
@@ -180,9 +217,20 @@ in {
         enableACME = true;
         forceSSL = true;
         locations."/" = {
-          proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+          proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
           proxyWebsockets = true;
         };
+        locations."/metrics" = {
+          # Only allow our monitoringvpn subnet
+          # And localhost since we're the monitoring server currently
+          extraConfig = ''
+            allow ${config.grid.monitoringvpnIPv4}/24;
+            allow 127.0.0.1;
+            allow ::1;
+            deny all;
+          '';
+          proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
+        };
       };
     };
 

nixos/modules/monitoring/server/loki.nix

 # Loki Server
 #
-# Scope: Log aggregator
+# Scope: Log ingester and aggregator to be run on the monitoring node
+#
+# See also:
+#   - The configuration is adapted from
+#     https://grafana.com/docs/loki/latest/configuration/examples/#complete-local-configyaml
+#
 
-{
-  config.networking.firewall.allowedTCPPorts = [ 3100 ];
+{ config, ...}:
+let
+  logRetention = toString(config.services.private-storage.monitoring.policy.logRetentionSeconds) + "s";
+
+in {
+  config.networking.firewall.interfaces.monitoringvpn.allowedTCPPorts = [ 3100 ];
 
   config.services.loki = {
     enable = true;
@@ -12,31 +21,39 @@
       {
         auth_enabled = false;
 
-        server = {
-          http_listen_port = 3100;
-        };
-
-        ingester = {
-          lifecycler = {
-            address = "0.0.0.0";
+        common = {
           ring = {
             kvstore = {
               store = "inmemory";
             };
+          };
+          instance_addr = "127.0.0.1";
           replication_factor = 1;
+          path_prefix = "/var/lib/loki";
+          storage = {
+            filesystem = {
+              chunks_directory = "/var/lib/loki/chunks";
+              rules_directory = "/var/lib/loki/rules";
+            };
+          };
+        };
+
+        server = {
+          http_listen_port = 3100;
+          grpc_listen_port = 9095; # unused, but no option to turn it off.
+          grpc_listen_address = "127.0.0.1"; # unused, but no option to turn it off.
         };
+
+        ingester = {
+          lifecycler = {
             final_sleep = "0s";
           };
-          chunk_idle_period = "1h"; # Any chunk not receiving new logs in this time will be flushed
-          max_chunk_age = "1h"; # All chunks will be flushed when they hit this age, default is 1h
-          chunk_target_size = 1048576; # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
-          chunk_retain_period = "30s"; # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
-          max_transfer_retries = 0; # Chunk transfers disabled
+          chunk_target_size = 1536000; # As per https://grafana.com/docs/loki/v2.2.1/best-practices/
         };
 
         schema_config = {
           configs = [{
-            from = "2020-10-24"; # TODO: Should this be "today"?
+            from = "2020-12-26";
             store = "boltdb-shipper";
             object_store = "filesystem";
             schema = "v11";
@@ -47,30 +64,19 @@
           }];
         };
 
-        storage_config = {
-          boltdb_shipper = {
-            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
-            cache_location = "/var/lib/loki/boltdb-shipper-cache";
-            cache_ttl = "24h";         # Can be increased for faster performance over longer query periods, uses more disk space
-            shared_store = "filesystem";
-          };
-          filesystem = {
-            directory = "/var/lib/loki/chunks";
-          };
-        };
-
         limits_config = {
-          reject_old_samples = true;
-          reject_old_samples_max_age = "168h";
-        };
-
-        chunk_store_config = {
-          max_look_back_period = "336h";
+          allow_structured_metadata = false;
         };
 
         table_manager = {
           retention_deletes_enabled = true;
-          retention_period = "336h";
+          retention_period = logRetention;
+        };
+
+        compactor = {
+          retention_enabled = true;
+          delete_request_store = "filesystem";
+          working_directory = "/var/lib/loki/compactor";
         };
       };
   };