diff --git a/morph/README.rst b/morph/README.rst
index f43926119b6609fcd0c746dcc1113822b5244e5a..96d03eb3cf522af6f1b0065105a2d57ab5c78f6a 100644
--- a/morph/README.rst
+++ b/morph/README.rst
@@ -59,6 +59,9 @@ An exception is the ``private-keys`` directory in the ``local`` morph grid:
That directory is fully populated, provided as an example, and mostly: not very secret.
Do not deploy these keys to machines reachable via the internet.
+Strictly speaking,
+this path is configurable in the grid's ``config.json`` but all three grids currently use this name.
+
public-keys
~~~~~~~~~~~
@@ -67,6 +70,29 @@ This must be created and populated before the grid can be built or deployed.
This directory contains any public key material necessary for operation of the grid.
This includes the public keys corresponding to any private keys held in ``private-keys``.
+As for ``private-keys``,
+this path can be configured in the grid's ``config.json``.
+
+Star-crossed Keys
+^^^^^^^^^^^^^^^^^
+
+Where the system uses keypairs,
+the public and private parts of those keypairs are stored in different locations
+(``public-keys`` and ``private-keys`` mentioned above).
+This somewhat complicates key management because any key rotation involves changing key material in two location instead of just one.
+
+This complication is balanced against a specific operational goal:
+that our build systems operate without copies of our private keys.
+Our system configurations do currently have build-time dependencies on public keys.
+
+Splitting public keys and private keys across two different storage locations provides a simple mechanism for providing build systems with the public keys but withholding the private keys.
+
+In the future we may:
+* be sufficiently confident in the security of our build systems to let them have our private keys; or
+* remove the dependency upon public keys from the build process.
+
+Either of these directions would let us re-unify public/private-key storage and remove this complication.
+
config.json
~~~~~~~~~~~
diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix
index 18556775c75aeba2d98087881c6b3f76d605c160..4e82fa29db5df7da2c9e977f978e67228e527be5 100644
--- a/morph/grid/local/grid.nix
+++ b/morph/grid/local/grid.nix
@@ -60,7 +60,7 @@ let
gridlib.monitoring
(gridlib.hardware-virtual ({ publicIPv4 = "192.168.67.24"; }))
(gridlib.customize-monitoring {
- inherit hostsMap vpnClientIPs nodeExporterTargets;
+ inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
inherit (config) domain publicKeyPath privateKeyPath;
monitoringvpnIPv4 = "172.23.23.1";
stateVersion = "19.09";
@@ -78,6 +78,7 @@ let
};
vpnClientIPs = [ "172.23.23.11" "172.23.23.12" "172.23.23.13" ];
nodeExporterTargets = [ "monitoring" "payments" "storage1" "storage2" ];
+ paymentExporterTargets = [ "payments" ];
in {
network = {
diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix
index a7b8be201d1c7b86576d9d069c3053984caa66e6..3fa9e287d2cbbb9e5b8e03baaad5912003f99cae 100644
--- a/morph/grid/production/grid.nix
+++ b/morph/grid/production/grid.nix
@@ -37,7 +37,7 @@ let
gridlib.monitoring
gridlib.hardware-aws
(gridlib.customize-monitoring {
- inherit hostsMap vpnClientIPs nodeExporterTargets;
+ inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
inherit (config) domain publicKeyPath privateKeyPath;
monitoringvpnIPv4 = "172.23.23.1";
stateVersion = "19.09";
@@ -116,6 +116,7 @@ let
"storage004"
"storage005"
];
+ paymentExporterTargets = [ "payments" ];
in {
network = {
diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix
index 17fc81ee2c49d990871927bd6cd2378ad6c30493..0c3d7c1c6b297ef9c3695eb438996cd2501536a6 100644
--- a/morph/grid/testing/grid.nix
+++ b/morph/grid/testing/grid.nix
@@ -49,7 +49,7 @@ let
gridlib.monitoring
gridlib.hardware-aws
(gridlib.customize-monitoring {
- inherit hostsMap vpnClientIPs nodeExporterTargets;
+ inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
inherit (config) domain publicKeyPath privateKeyPath;
monitoringvpnIPv4 = "172.23.23.1";
stateVersion = "19.09";
@@ -66,6 +66,7 @@ let
};
vpnClientIPs = [ "172.23.23.11" "172.23.23.12" ];
nodeExporterTargets = [ "monitoring" "payments" "storage001" ];
+ paymentExporterTargets = [ "payments" ];
in {
network = {
diff --git a/morph/lib/customize-monitoring.nix b/morph/lib/customize-monitoring.nix
index f77d26bd817ebb556c1d22d01e290d2838ab9485..05fe45107e44c583c495ee55aeb9e351ba3871f1 100644
--- a/morph/lib/customize-monitoring.nix
+++ b/morph/lib/customize-monitoring.nix
@@ -26,6 +26,10 @@
# which nodes to scrape "nginxExporter" metrics from.
, nginxExporterTargets ? []
+ # A list of VPN clients (IP addresses or hostnames) as strings indicating
+ # which nodes to scrape PaymentServer metrics from.
+, paymentExporterTargets ? []
+
# A string giving the NixOS state version for the system.
, stateVersion
, ...
@@ -52,6 +56,7 @@
services.private-storage.monitoring.prometheus = {
inherit nodeExporterTargets;
inherit nginxExporterTargets;
+ inherit paymentExporterTargets;
};
system.stateVersion = stateVersion;
diff --git a/nixos/modules/monitoring/server/grafana-config/services-overview.json b/nixos/modules/monitoring/server/grafana-config/services-overview.json
new file mode 100644
index 0000000000000000000000000000000000000000..8ee7b130ce917183edfa26a7a53c1a8df1353303
--- /dev/null
+++ b/nixos/modules/monitoring/server/grafana-config/services-overview.json
@@ -0,0 +1,664 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "RED: Requests-Errors-Duration for our services",
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": 2,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": null,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 18,
+ "panels": [],
+ "title": "Payments v1/stripe/charge",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "HTTPS responses per second",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 1
+ },
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(http_responses_total{path=\"v1/stripe/charge\"}[5m])",
+ "instant": false,
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "v1/stripe/charge RPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 1
+ },
+ "id": 15,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": true,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(http_responses_total{path=\"v1/stripe/charge\", status=\"4XX\"}) / sum(http_responses_total{path=\"v1/redeem\"})",
+ "legendFormat": "Client error (4XX) rate",
+ "refId": "A"
+ },
+ {
+ "expr": "sum(http_responses_total{path=\"v1/stripe/charge\", status=\"5XX\"}) / sum(http_responses_total{path=\"v1/redeem\"})",
+ "legendFormat": "Server error (5XX) rate",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "v1/stripe/charge error rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": "100",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "cacheTimeout": null,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "Requests taking longer than 1 s, between 1 sec and 10 msec, and 10 msec and below",
+ "fill": 2,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 1
+ },
+ "id": 12,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pluginVersion": "6.4.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "http_request_duration_seconds_bucket{path=\"v1/stripe/charge\", le=\"0.01\"}",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "=< 0.01s",
+ "refId": "A"
+ },
+ {
+ "expr": "http_request_duration_seconds_bucket{path=\"v1/stripe/charge\", le=\"1.0\"} - ignoring(le) http_request_duration_seconds_bucket{path=\"v1/stripe/charge\", le=\"0.01\"}",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "=< 1s",
+ "refId": "D"
+ },
+ {
+ "expr": "http_request_duration_seconds_bucket{path=\"v1/stripe/charge\", le=\"+Inf\"} - ignoring(le) http_request_duration_seconds_bucket{path=\"v1/stripe/charge\", le=\"1.0\"}",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "> 1s",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "v1/stripe/charge durations",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": null,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
+ "id": 11,
+ "panels": [],
+ "title": "Payments v1/redeem",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "HTTPS responses per second",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 9
+ },
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(http_responses_total{path=\"v1/redeem\"}[5m])",
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "v1/redeem RPS",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 9
+ },
+ "id": 16,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": true,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(http_responses_total{path=\"v1/redeem\", status=\"4XX\"}) / sum(http_responses_total{path=\"v1/redeem\"})",
+ "legendFormat": "Client error (4XX) rate",
+ "refId": "A"
+ },
+ {
+ "expr": "sum(http_responses_total{path=\"v1/redeem\", status=\"5XX\"}) / sum(http_responses_total{path=\"v1/redeem\"})",
+ "legendFormat": "Server error (5XX) rate",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "v1/redeem error rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": "100",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "cacheTimeout": null,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "Requests taking longer than 1 s, between 1 sec and 10 msec, and 10 msec and below",
+ "fill": 2,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 9
+ },
+ "id": 13,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pluginVersion": "6.4.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "http_request_duration_seconds_bucket{path=\"v1/redeem\", le=\"0.01\"}",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "=< 0.01s",
+ "refId": "A"
+ },
+ {
+ "expr": "http_request_duration_seconds_bucket{path=\"v1/redeem\", le=\"1.0\"} - ignoring(le) http_request_duration_seconds_bucket{path=\"v1/redeem\", le=\"0.01\"}",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "=< 1s",
+ "refId": "D"
+ },
+ {
+ "expr": "http_request_duration_seconds_bucket{path=\"v1/redeem\", le=\"+Inf\"} - ignoring(le) http_request_duration_seconds_bucket{path=\"v1/redeem\", le=\"1.0\"}",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "> 1s",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "v1/redeem durations",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 20,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-3h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "Services overview",
+ "uid": "JX3RVEk7k",
+ "version": 6
+}
diff --git a/nixos/modules/monitoring/server/prometheus.nix b/nixos/modules/monitoring/server/prometheus.nix
index 36c2ba6402559771dff8771b1369842e21f7ff7f..c92261ccad2ebdd8dd34dda9027e66962b345be9 100644
--- a/nixos/modules/monitoring/server/prometheus.nix
+++ b/nixos/modules/monitoring/server/prometheus.nix
@@ -26,6 +26,11 @@ in {
example = lib.literalExample "[ node1 node2 ]";
description = "List of nodes (hostnames or IPs) to scrape.";
};
+ paymentExporterTargets = lib.mkOption {
+ type = with lib.types; listOf str;
+ example = lib.literalExample "[ node1 node2 ]";
+ description = "List of nodes (hostnames or IPs) to scrape.";
+ };
};
config = rec {
@@ -49,6 +54,15 @@ in {
}];
relabel_configs = [ dropPortNumber ];
}
+ {
+ job_name = "payment-exporters";
+ scheme = "https";
+ tls_config.insecure_skip_verify = true;
+ static_configs = [{
+ targets = cfg.paymentExporterTargets;
+ }];
+ relabel_configs = [ dropPortNumber ];
+ }
];
};
};
diff --git a/nixpkgs.json b/nixpkgs.json
index 6b98d3d39cffa2eb1a6dfa7fb5f8c3bea50dfb60..e8a900d44fc2a08a09c4e1f6447da3a4a6a52fd1 100644
--- a/nixpkgs.json
+++ b/nixpkgs.json
@@ -1,4 +1,4 @@
{ "name": "nixpkgs"
-, "url": "https://github.com/PrivateStorageio/nixpkgs/archive/7e71ee63a67bd3e2c190abd982b541603f4f86b0.tar.gz"
-, "sha256": "1yy89lc0p7hx7x4r2y5ll851mfn4a2lacj5c9v5w139zz17ky743"
+, "url": "https://github.com/PrivateStorageio/nixpkgs/archive/788cc5806d46b89013ddd59db589b748bc20435e.tar.gz"
+, "sha256": "1mjznn4i4524gl5aiapjpy2jzpac1fzp7jvnkamrh9090ndalhar"
}