diff --git a/morph/lib/base.nix b/morph/lib/base.nix
index 0ab7733324d991917c787f2404f4c24df3927e94..71d4dfb57c8723a384a3dfb9d08a6069914de3d2 100644
--- a/morph/lib/base.nix
+++ b/morph/lib/base.nix
@@ -57,6 +57,8 @@
# qualified domain name.
deployment.targetHost = config.networking.fqdn;
+ services.private-storage.monitoring.exporters.promtail.enable = true;
+
assertions = [
# This is a check to save somebody in the future trying to debug why
# setting `nixpkgs.config` is not having an effect.
diff --git a/morph/lib/monitoring.nix b/morph/lib/monitoring.nix
index 3cf1680e1e1d0d1399b3e5741cc8b3850f68c2e5..d299d62ae7997511897517f9574e33c6de94b7a5 100644
--- a/morph/lib/monitoring.nix
+++ b/morph/lib/monitoring.nix
@@ -30,10 +30,9 @@ in {
../../nixos/modules/monitoring/vpn/server.nix
../../nixos/modules/monitoring/server/grafana.nix
../../nixos/modules/monitoring/server/prometheus.nix
+ ../../nixos/modules/monitoring/server/loki.nix
../../nixos/modules/monitoring/exporters/node.nix
../../nixos/modules/monitoring/exporters/blackbox.nix
- # Loki 0.3.0 from Nixpkgs 19.09 is too old and does not work:
- # ../../nixos/modules/monitoring/server/loki.nix
];
options.grid.monitoring = {
diff --git a/nixos/modules/default.nix b/nixos/modules/default.nix
index 1772d399639aa8b4ec2c9ac6a218c4dd8a6169da..f7e247f99406ad982c3b1e59d8248e2c80a3a658 100644
--- a/nixos/modules/default.nix
+++ b/nixos/modules/default.nix
@@ -12,5 +12,6 @@
imports = [
./packages.nix
./issuer.nix
+ ./monitoring/exporters/promtail.nix
];
}
diff --git a/nixos/modules/monitoring/exporters/promtail.nix b/nixos/modules/monitoring/exporters/promtail.nix
new file mode 100644
index 0000000000000000000000000000000000000000..83de3250af5f02635ec5c790eedf445b1e38a92e
--- /dev/null
+++ b/nixos/modules/monitoring/exporters/promtail.nix
@@ -0,0 +1,65 @@
+# Promtail log forwarder configuration
+#
+# Scope: Tail logs on the local system and send them to Loki
+#
+# Description: This is not strictly an "exporter" like the Prometheus
+# exporters, but it is very similar in what it is doing -
+# preparing local data and sending it off to a TSDB.
+
+{ config, options, lib, ... }:
+
+let
+ cfg = config.services.private-storage.monitoring.exporters.promtail;
+ hostName = config.networking.hostName;
+
+in {
+ options.services.private-storage.monitoring.exporters.promtail = {
+ enable = lib.mkEnableOption "Promtail log exporter service";
+ lokiUrl = lib.mkOption {
+ type = lib.types.str;
+ description = ''
+ The server URL that logs should be pushed to.
+ '';
+ # Resolving names is hard, let's have breakfast
+ # If you are curious why there's a plain IP address in here, read all of
+ # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/251
+ # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/257
+ # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/258
+ default = "http://172.23.23.1:3100/loki/api/v1/push";
+ };
+ };
+
+ config = lib.mkIf cfg.enable {
+ services.promtail.enable = true;
+ networking.firewall.interfaces.monitoringvpn.allowedTCPPorts = [ 9080 ];
+ services.promtail.configuration = {
+ server = {
+ http_listen_port = 9080; # Using /metrics for health check
+ grpc_listen_address = "127.0.0.1"; # unused, but no option to turn it off.
+ grpc_listen_port = 9094; # unused, but no option to turn it off.
+ };
+
+ clients = [{
+ url = cfg.lokiUrl;
+ }];
+
+ scrape_configs = [{
+ job_name = "systemd-journal";
+ journal = {
+ labels = {
+ job = "systemd-journal";
+ host = hostName;
+ };
+ };
+ # The journal has many internal labels, that by default will
+ # be dropped because of their "__" prefix. To keep them, rename them.
+ # https://grafana.com/docs/loki/latest/clients/promtail/scraping/#journal-scraping-linux-only
+ # https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
+ relabel_configs = [{
+ source_labels = [ "__journal__systemd_unit" ];
+ target_label = "unit";
+ }];
+ }];
+ };
+ };
+}
diff --git a/nixos/modules/monitoring/server/grafana-dashboards/payments.json b/nixos/modules/monitoring/server/grafana-dashboards/payments.json
index 7d6f6bb12bae5c1401b1199a8b2831b39a4ba955..ffad8e9c42606da20be04207cd807256c5476f3c 100644
--- a/nixos/modules/monitoring/server/grafana-dashboards/payments.json
+++ b/nixos/modules/monitoring/server/grafana-dashboards/payments.json
@@ -8,19 +8,25 @@
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
"type": "dashboard"
}
]
},
"description": "PaymentServer and related metrics",
"editable": true,
- "gnetId": null,
+ "fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
+ "liveNow": false,
"panels": [
{
"collapsed": false,
- "datasource": null,
"gridPos": {
"h": 1,
"w": 24,
@@ -33,53 +39,107 @@
"type": "row"
},
{
- "aliasColors": {
- "Attempts": "yellow",
- "Successes": "green"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Our calls to the Stripe API: Attempted and successful credit card charges.",
"fieldConfig": {
- "defaults": {},
- "overrides": []
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Attempts"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "yellow",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Successes"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "green",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 1
},
- "hiddenSeries": false,
"id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.5.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.3.4",
"targets": [
{
"exemplar": true,
@@ -97,107 +157,97 @@
"refId": "C"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Stripe",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:350",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "$$hashKey": "object:351",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {
- "Redeemed vouchers": "yellow"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "",
"fieldConfig": {
- "defaults": {},
- "overrides": []
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Redeemed vouchers"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "yellow",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 1
},
- "hiddenSeries": false,
"id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.5.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "$$hashKey": "object:223",
- "alias": "Redeemed vouchers",
- "yaxis": 1
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
},
- {
- "$$hashKey": "object:230",
- "alias": "Issued signatures",
- "yaxis": 2
+ "tooltip": {
+ "mode": "single"
}
- ],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ },
+ "pluginVersion": "8.3.4",
"targets": [
{
"exemplar": true,
@@ -216,53 +266,11 @@
"refId": "B"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Redemption",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:285",
- "format": "short",
- "label": "",
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:286",
- "decimals": null,
- "format": "short",
- "label": "",
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"collapsed": false,
- "datasource": null,
"gridPos": {
"h": 1,
"w": 24,
@@ -275,52 +283,78 @@
"type": "row"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "HTTPS responses per second",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 9
},
- "hiddenSeries": false,
"id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.5.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.3.4",
"targets": [
{
"exemplar": true,
@@ -331,96 +365,83 @@
"refId": "A"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Requests per second",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:452",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:453",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 9
},
- "hiddenSeries": false,
"id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.5.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.3.4",
"targets": [
{
"exemplar": true,
@@ -437,103 +458,144 @@
"refId": "B"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Error rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:576",
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:577",
- "format": "percent",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {
- "=< 0.1s": "blue",
- "=< 1s": "green",
- "=< 5s": "yellow",
- "> 5s": "orange"
- },
- "bars": false,
- "cacheTimeout": null,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Request durations, stacked",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 20,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "=< 0.1s"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "blue",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "=< 1s"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "green",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "=< 5s"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "yellow",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "> 5s"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "orange",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
},
- "fill": 2,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 9
},
- "hiddenSeries": false,
"id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.5.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.3.4",
"targets": [
{
"exemplar": true,
@@ -580,52 +642,11 @@
"refId": "C"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Durations",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:625",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:626",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"collapsed": false,
- "datasource": null,
"gridPos": {
"h": 1,
"w": 24,
@@ -638,52 +659,78 @@
"type": "row"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "HTTPS responses per second",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 17
},
- "hiddenSeries": false,
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.5.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.3.4",
"targets": [
{
"expr": "rate(http_responses_total{path=\"v1/redeem\"}[5m])",
@@ -693,95 +740,82 @@
"refId": "A"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Requests per second",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:751",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:752",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 17
},
- "hiddenSeries": false,
"id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.5.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.3.4",
"targets": [
{
"expr": "sum(http_responses_total{path=\"v1/redeem\", status=\"4XX\"}) / sum(http_responses_total{path=\"v1/redeem\"})",
@@ -794,103 +828,144 @@
"refId": "B"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Error rate",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:804",
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:805",
- "format": "percent",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {
- "=< 0.1s": "blue",
- "=< 1s": "green",
- "=< 5s": "yellow",
- "> 5s": "orange"
- },
- "bars": false,
- "cacheTimeout": null,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Request durations, stacked.",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 20,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "=< 0.1s"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "blue",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "=< 1s"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "green",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "=< 5s"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "yellow",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "> 5s"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "orange",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
},
- "fill": 2,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 17
},
- "hiddenSeries": false,
"id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
- },
- "percentage": false,
- "pluginVersion": "7.5.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.3.4",
"targets": [
{
"exemplar": true,
@@ -937,52 +1012,151 @@
"refId": "C"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Durations",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:853",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 24
+ },
+ "id": 26,
+ "panels": [],
+ "title": "Logs",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "description": "Exercise in counting maybe interesting lines. This can be alerted on.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 25
+ },
+ "id": 28,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
},
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "expr": "count_over_time({host=\"payments\",unit=\"zkapissuer.service\"} !~ \"200|GET|Accept\" [5m])",
+ "maxLines": 0,
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "title": "Number of maybe interesting log lines",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "description": "Exercise in filtering the payment server logs.",
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 25
+ },
+ "id": 30,
+ "options": {
+ "dedupStrategy": "none",
+ "enableLogDetails": true,
+ "prettifyLogMessage": false,
+ "showCommonLabels": false,
+ "showLabels": false,
+ "showTime": false,
+ "sortOrder": "Descending",
+ "wrapLogMessage": false
+ },
+ "targets": [
{
- "$$hashKey": "object:854",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
+ "datasource": {
+ "type": "loki",
+ "uid": "P8E80F9AEF21F6940"
+ },
+ "expr": "{host=\"payments\",unit=\"zkapissuer.service\"} !~ \"200|GET|Accept\"",
+ "refId": "A"
}
],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "title": "Maybe interesting lines",
+ "type": "logs"
}
],
"refresh": "5m",
- "schemaVersion": 27,
+ "schemaVersion": 34,
"style": "dark",
"tags": [],
"templating": {
@@ -1009,5 +1183,6 @@
"timezone": "",
"title": "Payments",
"uid": "Payments",
- "version": 1
+ "version": 1,
+ "weekStart": ""
}
diff --git a/nixos/modules/monitoring/server/loki.nix b/nixos/modules/monitoring/server/loki.nix
index 96554523f06d0d86c620db445b2443575a1c3fd3..491d1a4c5edd1100ea17c26bbe8e8799b9424582 100644
--- a/nixos/modules/monitoring/server/loki.nix
+++ b/nixos/modules/monitoring/server/loki.nix
@@ -1,9 +1,14 @@
# Loki Server
#
-# Scope: Log aggregator
+# Scope: Log ingester and aggregator to be run on the monitoring node
+#
+# See also:
+# - The configuration is adapted from
+# https://grafana.com/docs/loki/latest/configuration/examples/#complete-local-configyaml
+#
{
- config.networking.firewall.allowedTCPPorts = [ 3100 ];
+ config.networking.firewall.interfaces.monitoringvpn.allowedTCPPorts = [ 3100 ];
config.services.loki = {
enable = true;
@@ -14,11 +19,12 @@
server = {
http_listen_port = 3100;
+ grpc_listen_port = 9095; # unused, but no option to turn it off.
+ grpc_listen_address = "127.0.0.1"; # unused, but no option to turn it off.
};
ingester = {
lifecycler = {
- address = "0.0.0.0";
ring = {
kvstore = {
store = "inmemory";
@@ -27,50 +33,35 @@
};
final_sleep = "0s";
};
- chunk_idle_period = "1h"; # Any chunk not receiving new logs in this time will be flushed
- max_chunk_age = "1h"; # All chunks will be flushed when they hit this age, default is 1h
- chunk_target_size = 1048576; # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
- chunk_retain_period = "30s"; # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+ chunk_target_size = 1536000; # As per https://grafana.com/docs/loki/v2.2.1/best-practices/
max_transfer_retries = 0; # Chunk transfers disabled
};
schema_config = {
configs = [{
- from = "2020-10-24"; # TODO: Should this be "today"?
- store = "boltdb-shipper";
+ from = "2020-12-26";
+ store = "boltdb";
object_store = "filesystem";
schema = "v11";
index = {
prefix = "index_";
- period = "24h";
};
}];
};
storage_config = {
- boltdb_shipper = {
- active_index_directory = "/var/lib/loki/boltdb-shipper-active";
- cache_location = "/var/lib/loki/boltdb-shipper-cache";
- cache_ttl = "24h"; # Can be increased for faster performance over longer query periods, uses more disk space
- shared_store = "filesystem";
+ boltdb = {
+ directory = "/var/lib/loki/index";
};
+
filesystem = {
directory = "/var/lib/loki/chunks";
};
};
- limits_config = {
- reject_old_samples = true;
- reject_old_samples_max_age = "168h";
- };
-
- chunk_store_config = {
- max_look_back_period = "336h";
- };
-
table_manager = {
retention_deletes_enabled = true;
- retention_period = "336h";
+ retention_period = "336h"; # two weeks
};
};
};