nixos/modules/monitoring/server/loki.nix

 # Loki Server
 #
-# Scope: Log aggregator
+# Scope: Log ingester and aggregator to be run on the monitoring node
+#
+# See also:
+#   - The configuration is adapted from
+#     https://grafana.com/docs/loki/latest/configuration/examples/#complete-local-configyaml
+#
 
-{
-  config.networking.firewall.allowedTCPPorts = [ 3100 ];
+{ config, ...}:
+let
+  logRetention = toString(config.services.private-storage.monitoring.policy.logRetentionSeconds) + "s";
+
+in {
+  config.networking.firewall.interfaces.monitoringvpn.allowedTCPPorts = [ 3100 ];
 
   config.services.loki = {
     enable = true;
@@ -12,31 +21,39 @@
       {
         auth_enabled = false;
 
-        server = {
-          http_listen_port = 3100;
-        };
-
-        ingester = {
-          lifecycler = {
-            address = "0.0.0.0";
+        common = {
           ring = {
             kvstore = {
               store = "inmemory";
             };
+          };
+          instance_addr = "127.0.0.1";
           replication_factor = 1;
+          path_prefix = "/var/lib/loki";
+          storage = {
+            filesystem = {
+              chunks_directory = "/var/lib/loki/chunks";
+              rules_directory = "/var/lib/loki/rules";
+            };
+          };
+        };
+
+        server = {
+          http_listen_port = 3100;
+          grpc_listen_port = 9095; # unused, but no option to turn it off.
+          grpc_listen_address = "127.0.0.1"; # unused, but no option to turn it off.
         };
+
+        ingester = {
+          lifecycler = {
             final_sleep = "0s";
           };
-          chunk_idle_period = "1h"; # Any chunk not receiving new logs in this time will be flushed
-          max_chunk_age = "1h"; # All chunks will be flushed when they hit this age, default is 1h
-          chunk_target_size = 1048576; # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
-          chunk_retain_period = "30s"; # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
-          max_transfer_retries = 0; # Chunk transfers disabled
+          chunk_target_size = 1536000; # As per https://grafana.com/docs/loki/v2.2.1/best-practices/
         };
 
         schema_config = {
           configs = [{
-            from = "2020-10-24"; # TODO: Should this be "today"?
+            from = "2020-12-26";
             store = "boltdb-shipper";
             object_store = "filesystem";
             schema = "v11";
@@ -47,30 +64,19 @@
           }];
         };
 
-        storage_config = {
-          boltdb_shipper = {
-            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
-            cache_location = "/var/lib/loki/boltdb-shipper-cache";
-            cache_ttl = "24h";         # Can be increased for faster performance over longer query periods, uses more disk space
-            shared_store = "filesystem";
-          };
-          filesystem = {
-            directory = "/var/lib/loki/chunks";
-          };
-        };
-
         limits_config = {
-          reject_old_samples = true;
-          reject_old_samples_max_age = "168h";
-        };
-
-        chunk_store_config = {
-          max_look_back_period = "336h";
+          allow_structured_metadata = false;
         };
 
         table_manager = {
           retention_deletes_enabled = true;
-          retention_period = "336h";
+          retention_period = logRetention;
+        };
+
+        compactor = {
+          retention_enabled = true;
+          delete_request_store = "filesystem";
+          working_directory = "/var/lib/loki/compactor";
         };
       };
   };

nixos/modules/monitoring/server/prometheus.nix

@@ -10,9 +10,10 @@ let
   cfg = config.services.private-storage.monitoring.prometheus;
   dropPortNumber = {
     source_labels = [ "__address__" ];
-    regex = "^(.*):\\d+$";
+    regex = "^(.*)(?:\\.monitoringvpn):\\d+$";
     target_label = "instance";
   };
+  logRetention = toString(config.services.private-storage.monitoring.policy.logRetentionSeconds) + "s";
 
 in {
   options.services.private-storage.monitoring.prometheus = {
@@ -44,6 +45,7 @@ in {
     services.prometheus = {
       enable = true;
       # port = 9090; # Option only in recent (20.09?) nixpkgs, 9090 default
+      retentionTime = logRetention;
       scrapeConfigs = [
         {
           job_name = "node-exporters";