From 4fbcfb9ed8f87eeb7e866cd09467d5910915deff Mon Sep 17 00:00:00 2001
From: Florian Sesser <florian@leastauthority.com>
Date: Tue, 20 Aug 2024 10:38:27 +0000
Subject: [PATCH] First try - latencies not yet great, need better summing
---
.../grafana-dashboards/services-overview.json | 490 +++++++++++++++++-
1 file changed, 471 insertions(+), 19 deletions(-)
diff --git a/nixos/modules/monitoring/server/grafana-dashboards/services-overview.json b/nixos/modules/monitoring/server/grafana-dashboards/services-overview.json
index 421074b3..fb812227 100644
--- a/nixos/modules/monitoring/server/grafana-dashboards/services-overview.json
+++ b/nixos/modules/monitoring/server/grafana-dashboards/services-overview.json
@@ -3,7 +3,10 @@
"list": [
{
"builtIn": 1,
- "datasource": "-- Grafana --",
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
@@ -21,7 +24,8 @@
"description": "How are our user-facing services doing?",
"editable": true,
"fiscalYearStartMonth": 0,
- "graphTooltip": 0,
+ "graphTooltip": 1,
+ "id": 116,
"links": [],
"liveNow": false,
"panels": [
@@ -33,8 +37,415 @@
"x": 0,
"y": 0
},
+ "id": 39,
+ "panels": [],
+ "title": "Tahoe-LAFS Overview (WIP)",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "description": "Positive values are read & get queries (== server sending over the network)",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": true,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": 3600000,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 1
+ },
+ "id": 41,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "-sum(irate(tahoe_counters_storage_server_allocate[$__range]))",
+ "instant": false,
+ "legendFormat": "allocate",
+ "range": true,
+ "refId": "allocate"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "-sum(irate(tahoe_counters_storage_server_writev[$__range]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "writev",
+ "range": true,
+ "refId": "writev"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "-sum(irate(tahoe_counters_storage_server_write[$__range]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "write",
+ "range": true,
+ "refId": "write"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "-sum(irate(tahoe_counters_storage_server_close[$__range]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "close",
+ "range": true,
+ "refId": "close"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(tahoe_counters_storage_server_get[$__range]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "get",
+ "range": true,
+ "refId": "get"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(tahoe_counters_storage_server_read[$__range]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "read",
+ "range": true,
+ "refId": "read"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum(irate(tahoe_counters_storage_server_readv[$__range]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "readv",
+ "range": true,
+ "refId": "readv"
+ }
+ ],
+ "title": "Request rates",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": -1,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 1
+ },
+ "id": 42,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance)(irate(tahoe_counters_storage_server_bytes_added[$__range]))",
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "Writing"
+ }
+ ],
+ "title": "Byes added",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 1
+ },
+ "id": 43,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "tahoe_stats_storage_server_latencies_read_mean + tahoe_stats_storage_server_latencies_readv_mean",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "editorMode": "code",
+ "expr": "tahoe_stats_storage_server_latencies_readv{quantile=\"0.990\"}",
+ "hide": true,
+ "instant": false,
+ "legendFormat": "{{instance}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Latencies (WIP)",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
"id": 32,
"panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
+ "refId": "A"
+ }
+ ],
"title": "HTTPS endpoints",
"type": "row"
},
@@ -74,6 +485,10 @@
"noDataState": "no_data",
"notifications": []
},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
"description": "",
"fieldConfig": {
"defaults": {
@@ -81,6 +496,9 @@
"mode": "palette-classic"
},
"custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
@@ -92,6 +510,7 @@
"tooltip": false,
"viz": false
},
+ "insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
@@ -130,22 +549,28 @@
"h": 8,
"w": 8,
"x": 0,
- "y": 1
+ "y": 9
},
"id": 36,
"options": {
"legend": {
"calcs": [],
- "displayMode": "hidden",
- "placement": "bottom"
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
},
"tooltip": {
- "mode": "single"
+ "mode": "single",
+ "sort": "none"
}
},
"pluginVersion": "8.3.5",
"targets": [
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
"exemplar": true,
"expr": "probe_duration_seconds",
"interval": "",
@@ -192,6 +617,10 @@
"noDataState": "ok",
"notifications": []
},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
"description": "Shows all HTTP endpoints where probe_success == 0. This could have different reasons, likely ones being the service is down or the TLS certificate is not trusted.",
"fieldConfig": {
"defaults": {
@@ -199,6 +628,9 @@
"mode": "palette-classic"
},
"custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
@@ -210,6 +642,7 @@
"tooltip": false,
"viz": false
},
+ "insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
@@ -248,22 +681,28 @@
"h": 8,
"w": 8,
"x": 8,
- "y": 1
+ "y": 9
},
"id": 38,
"options": {
"legend": {
"calcs": [],
- "displayMode": "hidden",
- "placement": "bottom"
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
},
"tooltip": {
- "mode": "single"
+ "mode": "single",
+ "sort": "none"
}
},
"pluginVersion": "8.3.5",
"targets": [
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
"exemplar": true,
"expr": "count by (instance) (probe_http_status_code!=200 and probe_http_status_code!=401 and probe_http_status_code!=404)",
"interval": "",
@@ -311,6 +750,10 @@
"noDataState": "no_data",
"notifications": []
},
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
"description": "",
"fieldConfig": {
"defaults": {
@@ -318,6 +761,9 @@
"mode": "palette-classic"
},
"custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
@@ -329,6 +775,7 @@
"tooltip": false,
"viz": false
},
+ "insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
@@ -368,22 +815,28 @@
"h": 8,
"w": 8,
"x": 16,
- "y": 1
+ "y": 9
},
"id": 34,
"options": {
"legend": {
"calcs": [],
- "displayMode": "hidden",
- "placement": "bottom"
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
},
"tooltip": {
- "mode": "single"
+ "mode": "single",
+ "sort": "none"
}
},
"pluginVersion": "8.3.5",
"targets": [
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "LocalPrometheus"
+ },
"exemplar": true,
"expr": "probe_ssl_earliest_cert_expiry - time()",
"interval": "",
@@ -396,21 +849,20 @@
"type": "timeseries"
}
],
- "refresh": "30s",
- "schemaVersion": 34,
- "style": "dark",
+ "refresh": "auto",
+ "schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
- "from": "now-1h",
+ "from": "now-12h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Services overview",
"uid": "ServicesOverview",
- "version": 1,
+ "version": 3,
"weekStart": ""
}
--
GitLab