diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix
index da8a83812ceba910280bfc61210487b2f217113f..088d9e8c79422b82d638a42aeab5da1fcf14f536 100644
--- a/morph/grid/local/grid.nix
+++ b/morph/grid/local/grid.nix
@@ -1,8 +1,6 @@
let
- pkgs = import <nixpkgs> { };
-
gridlib = import ../../lib;
- grid-config = pkgs.lib.trivial.importJSON ./config.json;
+ grid-config = builtins.fromJSON (builtins.readFile ./config.json);
ssh-users = let
ssh-users-file = ./public-keys/users.nix;
@@ -131,7 +129,7 @@ let
};
# TBD: derive these automatically:
- paymentExporterTargets = [ "payments" ];
+ paymentExporterTargets = [ "payments.monitoringvpn" ];
blackboxExporterHttpsTargets = [
# "https://private.storage/"
# "https://payments.private.storage/"
diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix
index ab45d4ba7f67e71383d28120bd925ac3a05f04ef..06fe07f8277bf81e26e2f9f735783614c117a7b3 100644
--- a/morph/grid/production/grid.nix
+++ b/morph/grid/production/grid.nix
@@ -1,9 +1,7 @@
# See morph/grid/local/grid.nix for additional commentary.
let
- pkgs = import <nixpkgs> { };
-
gridlib = import ../../lib;
- grid-config = pkgs.lib.trivial.importJSON ./config.json;
+ grid-config = builtins.fromJSON (builtins.readFile ./config.json);
# Module with per-grid configuration
grid-module = {config, ...}: {
@@ -101,7 +99,7 @@ let
# Enable statistics gathering for MegaRAID cards.
# TODO would be nice to enable only on machines that have such a device.
- services.private-storage.monitoring.megacli2prom.enable = true;
+ services.private-storage.monitoring.exporters.megacli2prom.enable = true;
};
};
@@ -114,7 +112,7 @@ let
storage005 = { vpnIP = "172.23.23.25"; stateVersion = "19.03"; };
};
- paymentExporterTargets = [ "payments" ];
+ paymentExporterTargets = [ "payments.monitoringvpn" ];
blackboxExporterHttpsTargets = [
"https://private.storage/"
"https://www.private.storage/"
diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix
index 19839ae83fa16c31adf0fcd9e3727a8304f8dd6c..c033da1279fa44800e994dc07df3f5febc97d60d 100644
--- a/morph/grid/testing/grid.nix
+++ b/morph/grid/testing/grid.nix
@@ -1,9 +1,7 @@
# See morph/grid/local/grid.nix for additional commentary.
let
- pkgs = import <nixpkgs> { };
-
gridlib = import ../../lib;
- grid-config = pkgs.lib.trivial.importJSON ./config.json;
+ grid-config = builtins.fromJSON (builtins.readFile ./config.json);
# Module with per-grid configuration
grid-module = {config, ...}: {
@@ -79,7 +77,7 @@ let
};
# TBD: derive these automatically:
- paymentExporterTargets = [ "payments" ];
+ paymentExporterTargets = [ "payments.monitoringvpn" ];
blackboxExporterHttpsTargets = [
"https://privatestorage-staging.com/"
"https://www.privatestorage-staging.com/"
diff --git a/morph/lib/base.nix b/morph/lib/base.nix
index 71d4dfb57c8723a384a3dfb9d08a6069914de3d2..4173497720901bacd13555c22abbec04c08d76be 100644
--- a/morph/lib/base.nix
+++ b/morph/lib/base.nix
@@ -57,6 +57,9 @@
# qualified domain name.
deployment.targetHost = config.networking.fqdn;
+ # This is for desktop stuff
+ services.udisks2.enable = false;
+
services.private-storage.monitoring.exporters.promtail.enable = true;
assertions = [
diff --git a/morph/lib/storage.nix b/morph/lib/storage.nix
index 71e3c22371ad042c4ddbc5d8cd87db5cb05923af..9424784dcd33b8e12b9da9cc309a657be07f66b3 100644
--- a/morph/lib/storage.nix
+++ b/morph/lib/storage.nix
@@ -67,7 +67,7 @@ in {
};
};
- services.private-storage.monitoring.tahoe.enable = true;
+ services.private-storage.monitoring.exporters.tahoe.enable = true;
# Turn on the Private Storage (Tahoe-LAFS) service.
services.private-storage = {
diff --git a/nixos/modules/monitoring/exporters/megacli2prom.nix b/nixos/modules/monitoring/exporters/megacli2prom.nix
index c1f10aef7821343e2f0a572d006782858dcf7ef4..c27219ef814d3ea4a811959caec036976672256d 100644
--- a/nixos/modules/monitoring/exporters/megacli2prom.nix
+++ b/nixos/modules/monitoring/exporters/megacli2prom.nix
@@ -12,10 +12,10 @@
{ config, options, lib, ourpkgs, pkgs, ... }:
let
- cfg = config.services.private-storage.monitoring.megacli2prom;
+ cfg = config.services.private-storage.monitoring.exporters.megacli2prom;
in {
- options.services.private-storage.monitoring.megacli2prom = {
+ options.services.private-storage.monitoring.exporters.megacli2prom = {
enable = lib.mkEnableOption "MegaCli2Prom metrics gathering service";
outFile = lib.mkOption {
type = lib.types.str;
diff --git a/nixos/modules/monitoring/exporters/node.nix b/nixos/modules/monitoring/exporters/node.nix
index d854ff7398cd19ac4d4d5b3f8739073feb84834a..6a84a31ce58859969e2fb08cff79abb957c6099e 100644
--- a/nixos/modules/monitoring/exporters/node.nix
+++ b/nixos/modules/monitoring/exporters/node.nix
@@ -6,14 +6,22 @@
# monitoring system
# See https://nixos.org/manual/nixos/stable/#module-services-prometheus-exporters
-{ config, lib, pkgs, ... }:
+{ config, lib, pkgs, options, ... }:
with lib;
let
+ cfg = config.services.private-storage.monitoring.exporters.node;
mountsFileSystemType = fsType: {} != filterAttrs (n: v: v.fsType == fsType) config.fileSystems;
in {
+ options.services.private-storage.monitoring.exporters.node = {
+ textfiles-directory = lib.mkOption {
+ type = lib.types.str;
+ description = "Directory used by the textfiles collector.";
+ default = "/run/prometheus-node-exporter";
+ };
+ };
config.services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
@@ -36,7 +44,7 @@ in {
"stat"
"systemd"
"textfile"
- "textfile.directory /run/prometheus-node-exporter"
+ "textfile.directory ${cfg.textfiles-directory}"
#"thermal_zone" # not in nixpkgs 19.09
"time"
#"udp_queues" # not in nixpkgs 19.09
diff --git a/nixos/modules/monitoring/exporters/tahoe.nix b/nixos/modules/monitoring/exporters/tahoe.nix
index 9bffabe74d45341b981ce3eb1cf05323e71fb5c4..087e6532e372b11bb2db826456957fa14dfa3c3d 100644
--- a/nixos/modules/monitoring/exporters/tahoe.nix
+++ b/nixos/modules/monitoring/exporters/tahoe.nix
@@ -10,10 +10,11 @@
{ config, options, lib, pkgs, ... }:
let
- cfg = config.services.private-storage.monitoring.tahoe;
+ cfg = config.services.private-storage.monitoring.exporters.tahoe;
+ inherit (config.services.private-storage.monitoring.exporters.node) textfiles-directory;
in {
- options.services.private-storage.monitoring.tahoe = {
+ options.services.private-storage.monitoring.exporters.tahoe = {
enable = lib.mkEnableOption "Tahoe OpenMetrics collecting service";
scrapeEndpoint = lib.mkOption {
type = lib.types.str;
@@ -23,7 +24,7 @@ in {
outFile = lib.mkOption {
type = lib.types.str;
description = "Where to store the temporary file for node exporter to scrape?";
- default = "/run/prometheus-node-exporter/tahoe.prom";
+ default = "${textfiles-directory}/tahoe.prom";
};
interval = lib.mkOption {
type = lib.types.str;
diff --git a/nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json b/nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json
index 745f7b7ff03d9a0a4e93fc50946f8066fb08e7b9..8827f23e6b63f8ed8e4b18b45b8e36cbb532dc21 100644
--- a/nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json
+++ b/nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json
@@ -8,19 +8,25 @@
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
"type": "dashboard"
}
]
},
"description": "USE: Usage, Saturation and Error rate for our resources",
"editable": true,
- "gnetId": null,
- "graphTooltip": 0,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 1,
"links": [],
+ "liveNow": false,
"panels": [
{
"collapsed": false,
- "datasource": null,
"gridPos": {
"h": 1,
"w": 24,
@@ -33,52 +39,79 @@
"type": "row"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Some of our software runs in a single thread, so this shows max CPU per core (instead of averaged over all cores)",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
},
"overrides": []
},
- "fill": 0,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 1
},
- "hiddenSeries": false,
"id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "multi"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"expr": "1 - (max by (instance) (irate(node_cpu_seconds_total{mode=\"idle\"}[5m])))",
@@ -88,46 +121,8 @@
"refId": "A"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Max CPU % per core per node",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"alert": {
@@ -165,15 +160,57 @@
"noDataState": "no_data",
"notifications": []
},
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"fieldConfig": {
"defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line+area"
+ }
+ },
"displayName": "${__field.labels.instance}",
- "links": []
+ "links": [],
+ "mappings": [],
+ "max": 1,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "short"
},
"overrides": [
{
@@ -189,40 +226,24 @@
}
]
},
- "fill": 0,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 6,
"y": 1
},
- "hiddenSeries": false,
"id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "multi"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"expr": "node_load15",
@@ -232,55 +253,8 @@
"refId": "A"
}
],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 1,
- "visible": true
- }
- ],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "15 min load average",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": "1",
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"alert": {
@@ -318,53 +292,79 @@
"noDataState": "no_data",
"notifications": []
},
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "How much RAM is in use? Relative to available system memory.",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line+area"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 0.8
+ }
+ ]
+ },
+ "unit": "percentunit"
},
"overrides": []
},
- "fill": 0,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 12,
"y": 1
},
- "hiddenSeries": false,
"id": 2,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "multi"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"expr": "1 - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / node_memory_MemTotal_bytes\r\n",
@@ -374,56 +374,8 @@
"refId": "A"
}
],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 0.8,
- "visible": true
- }
- ],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "RAM used %",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "decimals": null,
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"alert": {
@@ -461,54 +413,78 @@
"noDataState": "no_data",
"notifications": []
},
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "How much Swap is in use? Relative to available swap.",
"fieldConfig": {
- "defaults": {},
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line+area"
+ }
+ },
+ "mappings": [],
+ "max": 1,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 0.1
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 18,
"y": 1
},
- "hiddenSeries": false,
"id": 30,
- "legend": {
- "alignAsTable": false,
- "avg": false,
- "current": false,
- "hideEmpty": false,
- "hideZero": false,
- "max": false,
- "min": false,
- "rightSide": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "multi"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"exemplar": true,
@@ -519,61 +495,11 @@
"refId": "A"
}
],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 0.1,
- "visible": true
- }
- ],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Swap used %",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:98",
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": "1",
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:99",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"collapsed": false,
- "datasource": null,
"gridPos": {
"h": 1,
"w": 24,
@@ -586,52 +512,77 @@
"type": "row"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Shows most saturated network link for every node. Bit/s.",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bps"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 9
},
- "hiddenSeries": false,
"id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"expr": "max by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|monitoringvpn\"}[5m]) * 8)",
@@ -648,98 +599,82 @@
"refId": "B"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Throughput",
- "tooltip": {
- "shared": false,
- "sort": 2,
- "value_type": "individual"
- },
"transformations": [],
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:226",
- "decimals": null,
- "format": "bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "$$hashKey": "object:227",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Packet and error count. Positive values mean transmit, negative receive.",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "pps"
},
"overrides": []
},
- "fill": 0,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 9
},
- "hiddenSeries": false,
"id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null as zero",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"expr": "- rate(node_network_receive_packets_total{device!~\"lo|monitoringvpn\"}[5m])",
@@ -770,46 +705,8 @@
"refId": "D"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Network pkt/s",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "pps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"alert": {
@@ -818,9 +715,10 @@
{
"evaluator": {
"params": [
- 10
+ -1,
+ 1
],
- "type": "gt"
+ "type": "outside_range"
},
"operator": {
"type": "and"
@@ -841,9 +739,10 @@
{
"evaluator": {
"params": [
- 10
+ -1,
+ 1
],
- "type": "gt"
+ "type": "outside_range"
},
"operator": {
"type": "or"
@@ -864,9 +763,10 @@
{
"evaluator": {
"params": [
- 10
+ -1,
+ 1
],
- "type": "gt"
+ "type": "outside_range"
},
"operator": {
"type": "or"
@@ -887,9 +787,10 @@
{
"evaluator": {
"params": [
- 10
+ -1,
+ 1
],
- "type": "gt"
+ "type": "outside_range"
},
"operator": {
"type": "or"
@@ -912,77 +813,123 @@
"for": "5m",
"frequency": "1m",
"handler": 1,
+ "message": "",
"name": "Network errors alert",
"noDataState": "no_data",
"notifications": []
},
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Network errors, drops etc. Should all be 0.",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 10
+ }
+ ]
+ },
+ "unit": "short"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 9
},
- "hiddenSeries": false,
"id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
- "expr": "node_network_transmit_errs_total\n",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "exemplar": true,
+ "expr": "rate(node_network_transmit_errs_total{device!=\"lo\"}[5m])\n",
"interval": "",
"legendFormat": "{{instance}} {{device}}",
"refId": "A"
},
{
- "expr": "node_network_transmit_drop_total",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "exemplar": true,
+ "expr": "rate(node_network_transmit_drop_total{device!=\"lo\"}[5m])",
"interval": "",
"legendFormat": "{{instance}} {{device}}",
"refId": "B"
},
{
- "expr": "- node_network_receive_drop_total",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "exemplar": true,
+ "expr": "- rate(node_network_receive_drop_total{device!=\"lo\"}[5m])",
"interval": "",
"legendFormat": "{{instance}} {{device}}",
"refId": "C"
},
{
- "expr": "- node_network_receive_errs_total",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "exemplar": true,
+ "expr": "- rate(node_network_receive_errs_total{device!=\"lo\"}[5m])",
"interval": "",
"legendFormat": "{{instance}} {{device}}",
"refId": "D"
@@ -991,61 +938,27 @@
"thresholds": [
{
"colorMode": "critical",
- "fill": true,
- "line": true,
+ "op": "lt",
+ "value": -1,
+ "visible": true
+ },
+ {
+ "colorMode": "critical",
"op": "gt",
- "value": 10,
+ "value": 1,
"visible": true
}
],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Network errors",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "collapsed": false,
- "datasource": null,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 16
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 16
},
"id": 18,
"panels": [],
@@ -1092,7 +1005,6 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": null,
"description": "Watch filesystems filling up. Shows only mounts over 10 % of available bytes used.",
"fieldConfig": {
"defaults": {
@@ -1114,6 +1026,8 @@
"legend": {
"avg": false,
"current": false,
+ "hideEmpty": false,
+ "hideZero": false,
"max": false,
"min": false,
"show": false,
@@ -1127,7 +1041,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.5.10",
+ "pluginVersion": "8.3.6",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -1157,21 +1071,17 @@
"visible": true
}
],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Storage usage %",
"tooltip": {
"shared": false,
- "sort": 0,
+ "sort": 2,
"value_type": "individual"
},
"transformations": [],
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -1179,7 +1089,6 @@
{
"$$hashKey": "object:131",
"format": "percentunit",
- "label": null,
"logBase": 1,
"max": "1",
"min": "0",
@@ -1188,65 +1097,86 @@
{
"$$hashKey": "object:132",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Input Output Operations per second. Positive values mean read, negative write.",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
},
"overrides": []
},
- "fill": 0,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 6,
"y": 17
},
- "hiddenSeries": false,
"id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null as zero",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"expr": "rate(node_disk_reads_completed_total[5m]) > 0",
@@ -1263,94 +1193,81 @@
"refId": "B"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "IOPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Max average storage latency per node. Positive values mean read, negative write.",
"fieldConfig": {
"defaults": {
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
},
"overrides": []
},
- "fill": 0,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 12,
"y": 17
},
- "hiddenSeries": false,
"id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null as zero",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"expr": "max by (instance, device) (rate(node_disk_read_time_seconds_total[5m]) / rate(node_disk_reads_completed_total[5m]))",
@@ -1367,46 +1284,8 @@
"refId": "B"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Storage latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"alert": {
@@ -1444,51 +1323,76 @@
"noDataState": "ok",
"notifications": []
},
- "aliasColors": {},
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"fieldConfig": {
- "defaults": {},
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 18,
"y": 17
},
- "hiddenSeries": false,
"id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "hideEmpty": false,
- "hideZero": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": false
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.10",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"exemplar": true,
@@ -1498,73 +1402,290 @@
"refId": "A"
}
],
- "thresholds": [
+ "title": "Degraded RAID arrays",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 24
+ },
+ "id": 34,
+ "panels": [],
+ "title": "Pressure Stall Information",
+ "type": "row"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 25
+ },
+ "id": 36,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
{
- "colorMode": "critical",
- "fill": false,
- "line": false,
- "op": "gt",
- "value": 0,
- "visible": true
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "exemplar": true,
+ "expr": "rate(node_pressure_cpu_waiting_seconds_total[5m])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
}
],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
- "title": "Degraded RAID arrays",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
+ "title": "CPU waiting",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 25
},
- "yaxes": [
+ "id": 38,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
{
- "$$hashKey": "object:151",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "exemplar": true,
+ "expr": "rate(node_pressure_memory_waiting_seconds_total[5m])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Memory waiting",
+ "type": "timeseries"
+ },
+ {
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 25
+ },
+ "id": 37,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom"
},
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "targets": [
{
- "$$hashKey": "object:152",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "exemplar": true,
+ "expr": "rate(node_pressure_io_waiting_seconds_total[5m])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
}
],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "title": "I/O waiting",
+ "type": "timeseries"
}
],
- "refresh": "30s",
- "schemaVersion": 27,
+ "refresh": "5s",
+ "schemaVersion": 34,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
- "from": "now-3h",
+ "from": "now-12h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Resources overview",
"uid": "ResourcesOverview",
- "version": 1
+ "version": 1,
+ "weekStart": ""
}
diff --git a/nixos/modules/monitoring/server/grafana-dashboards/services-overview.json b/nixos/modules/monitoring/server/grafana-dashboards/services-overview.json
index 09bfa3acde10c14fb41d3082779bc14d064b6554..b52b35dc297efa30eac1d472f81ef47ebcd45b30 100644
--- a/nixos/modules/monitoring/server/grafana-dashboards/services-overview.json
+++ b/nixos/modules/monitoring/server/grafana-dashboards/services-overview.json
@@ -8,19 +8,25 @@
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
"type": "dashboard"
}
]
},
"description": "How are our user-facing services doing?",
"editable": true,
- "gnetId": null,
+ "fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
+ "liveNow": false,
"panels": [
{
"collapsed": false,
- "datasource": null,
"gridPos": {
"h": 1,
"w": 24,
@@ -68,50 +74,76 @@
"noDataState": "no_data",
"notifications": []
},
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "",
"fieldConfig": {
- "defaults": {},
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line+area"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 3.142
+ }
+ ]
+ },
+ "unit": "s"
+ },
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 1
},
- "hiddenSeries": false,
"id": 36,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.11",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"exemplar": true,
@@ -121,57 +153,8 @@
"refId": "A"
}
],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 3.142,
- "visible": true
- }
- ],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Response times",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:425",
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "$$hashKey": "object:426",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"alert": {
@@ -209,50 +192,76 @@
"noDataState": "ok",
"notifications": []
},
- "aliasColors": {},
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "Shows all HTTP endpoints where probe_success == 0. This could have different reasons, likely ones being the service is down or the TLS certificate is not trusted.",
"fieldConfig": {
- "defaults": {},
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "line+area"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "transparent",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 1
},
- "hiddenSeries": false,
"id": 38,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.11",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"exemplar": true,
@@ -262,57 +271,8 @@
"refId": "A"
}
],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 0,
- "visible": true
- }
- ],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Probe fails",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:903",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "$$hashKey": "object:904",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"alert": {
@@ -351,50 +311,77 @@
"noDataState": "no_data",
"notifications": []
},
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
"description": "",
"fieldConfig": {
- "defaults": {},
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line+area"
+ }
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ },
+ {
+ "color": "transparent",
+ "value": 2419200
+ }
+ ]
+ },
+ "unit": "s"
+ },
"overrides": []
},
- "fill": 0,
- "fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 1
},
- "hiddenSeries": false,
"id": 34,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": false,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "hidden",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "single"
+ }
},
- "percentage": false,
- "pluginVersion": "7.5.11",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.5",
"targets": [
{
"exemplar": true,
@@ -405,73 +392,25 @@
"refId": "A"
}
],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "lt",
- "value": 2419200,
- "visible": true
- }
- ],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "TLS certificate expiry",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "$$hashKey": "object:179",
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": "0",
- "show": true
- },
- {
- "$$hashKey": "object:180",
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
}
],
"refresh": "30s",
- "schemaVersion": 27,
+ "schemaVersion": 34,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
- "from": "now-15m",
+ "from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Services overview",
"uid": "ServicesOverview",
- "version": 1
+ "version": 1,
+ "weekStart": ""
}
diff --git a/nixos/modules/restricted-service.nix b/nixos/modules/restricted-service.nix
new file mode 100644
index 0000000000000000000000000000000000000000..93401799559ec35082ffe1018bfbb1b32d103ccf
--- /dev/null
+++ b/nixos/modules/restricted-service.nix
@@ -0,0 +1,52 @@
+# Provide secure defaults for systemd services
+#
+# Good reads:
+# https://gist.github.com/ageis/f5595e59b1cddb1513d1b425a323db04
+# https://docs.arbitrary.ch/security/systemd.html
+# https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+
+{
+ DynamicUser = true;
+
+ # This set of restrictions is mostly dervied from
+ # - running `systemd-analyze security zkap-spending-service.service`
+ # - Looking at the restrictions from the nixos nginx config.
+ AmbientCapabilities = "";
+ CapabilityBoundingSet = "";
+ LockPersonality = true;
+ MemoryDenyWriteExecute = true;
+ NoNewPrivileges = true;
+ PrivateDevices = true;
+ PrivateMounts = true;
+ PrivateNetwork = true;
+ PrivateTmp = true;
+ PrivateUsers = true;
+ ProcSubset = "pid";
+ ProtectClock = true;
+ ProtectControlGroups = true;
+ ProtectHome = true;
+ ProtectHostname = true;
+ ProtectKernelLogs = true;
+ ProtectKernelModules = true;
+ ProtectKernelTunables = true;
+ ProtectProc = "invisible";
+ ProtectSystem = "strict";
+ RemoveIPC = true;
+ RestrictAddressFamilies = "AF_UNIX";
+ RestrictNamespaces = true;
+ RestrictRealtime = true;
+ RestrictSUIDSGID = true;
+ SystemCallArchitectures = "native";
+ # Lines starting with "~" are deny-list the others are allow-list
+ # Since the first line is allow, that bounds the set of allowed syscalls
+ # and the further lines restrict it.
+ SystemCallFilter = [
+ # From systemd.exec(5), @system-service is "A reasonable set of
+ # system calls used by common system [...]"
+ "@system-service"
+ # This is from the nginx config, except that `@ipc` is not removed,
+ # since twisted uses a self-pipe.
+ "~@cpu-emulation @debug @keyring @mount @obsolete @privileged @setuid"
+ ];
+ Umask = "0077";
+}
diff --git a/nixos/modules/spending.nix b/nixos/modules/spending.nix
index ccbce1d3b4fbec4cc4bdeb0efd67653f5160581a..0b9a63be1021d9833de350b6cee9870ad2fb137c 100644
--- a/nixos/modules/spending.nix
+++ b/nixos/modules/spending.nix
@@ -1,4 +1,4 @@
-# A NixOS module which can run a Ristretto-based issuer for PrivateStorage
+# A NixOS module which can run a service tracking spending of ZKAPs.
# ZKAPs.
{ lib, pkgs, config, ourpkgs, ... }@args: let
cfg = config.services.private-storage-spending;
@@ -40,70 +40,26 @@ in
wantedBy = [ "sockets.target" ];
listenStreams = [ cfg.unixSocket ];
};
+
# Add a systemd service to run zkap-spending-service.
systemd.services.zkap-spending-service = {
enable = true;
description = "ZKAP Spending Service";
wantedBy = [ "multi-user.target" ];
- serviceConfig.NonBlocking = true;
-
- # It really shouldn't ever exit on its own! If it does, it's a bug
- # we'll have to fix. Restart it and hope it doesn't happen too much
- # before we can fix whatever the issue is.
- serviceConfig.Restart = "always";
- serviceConfig.Type = "simple";
+ serviceConfig = (import ./restricted-service.nix) // {
+ NonBlocking = true;
- # Use a unnamed user.
- serviceConfig.DynamicUser = true;
+ # It really shouldn't ever exit on its own! If it does, it's a bug
+ # we'll have to fix. Restart it and hope it doesn't happen too much
+ # before we can fix whatever the issue is.
+ Restart = "always";
+ Type = "simple";
- serviceConfig = {
# Work around https://twistedmatrix.com/trac/ticket/10261
# Create a runtime directory so that the service has permission
# to change the mode on the socket.
RuntimeDirectory = "zkap-spending-service";
-
- # This set of restrictions is mostly dervied from
- # - running `systemd-analyze security zkap-spending-service.service
- # - Looking at the restrictions from the nixos nginx config.
- AmbientCapabilities = "";
- CapabilityBoundingSet = "";
- LockPersonality = true;
- MemoryDenyWriteExecute = true;
- NoNewPrivileges = true;
- PrivateDevices = true;
- PrivateMounts = true;
- PrivateNetwork = true;
- PrivateTmp = true;
- PrivateUsers = true;
- ProcSubset = "pid";
- ProtectClock = true;
- ProtectControlGroups = true;
- ProtectHome = true;
- ProtectHostname = true;
- ProtectKernelLogs = true;
- ProtectKernelModules = true;
- ProtectKernelTunables = true;
- ProtectProc = "invisible";
- ProtectSystem = "strict";
- RemoveIPC = true;
- RestrictAddressFamilies = "AF_UNIX";
- RestrictNamespaces = true;
- RestrictRealtime = true;
- RestrictSUIDSGID = true;
- SystemCallArchitectures = "native";
- # Lines starting with "~" are deny-list the others are allow-list
- # Since the first line is allow, that bounds the set of allowed syscalls
- # and the further lines restrict it.
- SystemCallFilter = [
- # From systemd.exec(5), @system-service is "A reasonable set of
- # system calls used by common system [...]"
- "@system-service"
- # This is from the nginx config, except that `@ipc` is not removed,
- # since twisted uses a self-pipe.
- "~@cpu-emulation @debug @keyring @mount @obsolete @privileged @setuid"
- ];
- Umask = "0077";
};
script = let
diff --git a/nixos/modules/tahoe.nix b/nixos/modules/tahoe.nix
index 44c381e6b6dfa6039d1dd6a49d44f1afaf51ab10..e4a83006e129e80a59ca9f8262acf74760b4fc98 100644
--- a/nixos/modules/tahoe.nix
+++ b/nixos/modules/tahoe.nix
@@ -293,7 +293,7 @@ in
isSystemUser = true;
group = "tahoe.${node}";
});
- users.groups = flip mapAttrs' cfg.introducers (node: _:
+ users.groups = flip mapAttrs' cfg.nodes (node: _:
nameValuePair "tahoe.${node}" {
});
})
diff --git a/nixpkgs.json b/nixpkgs.json
index 62afaf9ea5f9896250a677bb383e16b8d0bc081a..1ef93ddcc57942d20ca7fdb599c7261ad09c02c3 100644
--- a/nixpkgs.json
+++ b/nixpkgs.json
@@ -1,5 +1,5 @@
{
"name": "source",
- "url": "https://releases.nixos.org/nixos/21.11/nixos-21.11.335883.7adc9c14ec7/nixexprs.tar.xz",
- "sha256": "0r0sgphydyv0xggzyfas5v7dznf42sdgib8s8zmf7dbi09yb4y1l"
+ "url": "https://releases.nixos.org/nixos/21.11/nixos-21.11.336045.4afca382d80/nixexprs.tar.xz",
+ "sha256": "1af93alrn7iwv6kjm9r941wafjgpx142psjpix45qn2p50lbpxyx"
}
\ No newline at end of file