diff --git a/DEPLOYMENT-NOTES.rst b/DEPLOYMENT-NOTES.rst
index 5de83386dbb939cc3cfe2a8b68c198f218934933..0a7ea52e0bfb20e77f86797ead1778d614a2a720 100644
--- a/DEPLOYMENT-NOTES.rst
+++ b/DEPLOYMENT-NOTES.rst
@@ -1,6 +1,8 @@
Deployment notes
================
+- 2021-09-30 `Enable alerting <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/185>`_ needs a secret in ``private-keys/grafana-slack-url`` looking like the template in ``morph/grid/local/private-keys/grafana-slack-url`` and pointing to the secret API endpoint URL saved in `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ (or create a new secret URL at https://www.slack.com/apps/A0F7XDUAZ).
+
- 2021-09-07 `Manage access to payment metrics <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/146>`_ requires moving and chown'ing the PaymentServer database on the ``payments`` host::
mkdir /var/lib/zkapissuer
diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix
index 59880af8b91173ec8c4480f459ed6c862474ba38..15afcc8f600018f2cd50fe7f8a0cf4383da27ead 100644
--- a/morph/grid/local/grid.nix
+++ b/morph/grid/local/grid.nix
@@ -108,6 +108,7 @@ let
inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
inherit (grid-config) letsEncryptAdminEmail;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+ enableSlackAlert = false;
monitoringvpnIPv4 = "172.23.23.1";
stateVersion = "19.09";
})
diff --git a/morph/grid/local/private-keys/README.rst b/morph/grid/local/private-keys/README.rst
index 684bf942a8010129f49cfcf79f5df1b60965ae45..91670ac1a0ea6ee2c68df71ff196d010bdba8637 100644
--- a/morph/grid/local/private-keys/README.rst
+++ b/morph/grid/local/private-keys/README.rst
@@ -19,6 +19,13 @@ grafana-admin.password
This is the initial admin password for the Grafana web admin on the monitoring host.
+grafana-slack-url
+-----------------
+
+This file is read by Grafana's systemd service to set an environment variable with a secret Slack WebHook URL to post alerts to.
+The only line in the file should be ``SLACKURL=`` with the secret URL.
+Use the url from `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ or get a new secret URL for your Slack channel at https://www.slack.com/apps/A0F7XDUAZ.
+
stripe.secret
-------------
diff --git a/morph/grid/local/private-keys/grafana-slack-url b/morph/grid/local/private-keys/grafana-slack-url
new file mode 100644
index 0000000000000000000000000000000000000000..cb7dd1aec785a557fef6082a7570bc8c56728f14
--- /dev/null
+++ b/morph/grid/local/private-keys/grafana-slack-url
@@ -0,0 +1,2 @@
+SLACKURL=https://hooks.slack.com/services/x/y/z
+
diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix
index 3e0d41a5e6caabdc2c958f7bad0ae0ff10243c92..ec0c1b37f996dd836f6011425f114471a2ccde0a 100644
--- a/morph/grid/production/grid.nix
+++ b/morph/grid/production/grid.nix
@@ -48,6 +48,7 @@ let
inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
inherit (grid-config) letsEncryptAdminEmail;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+ enableSlackAlert = true;
monitoringvpnIPv4 = "172.23.23.1";
stateVersion = "19.09";
})
diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix
index 383fe740674992042a899ac391c100640ad5dcd7..7a304e11cfd7ec3fbef3f8efa2417a9004132c42 100644
--- a/morph/grid/testing/grid.nix
+++ b/morph/grid/testing/grid.nix
@@ -61,6 +61,7 @@ let
inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
inherit (grid-config) letsEncryptAdminEmail;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+ enableSlackAlert = true;
monitoringvpnIPv4 = "172.23.23.1";
stateVersion = "19.09";
})
diff --git a/morph/lib/customize-monitoring.nix b/morph/lib/customize-monitoring.nix
index 19a800f1fa806c09f132f2bb2769869a30c65ec2..d9842692481777a7e65b70208b5c0246b3209d1a 100644
--- a/morph/lib/customize-monitoring.nix
+++ b/morph/lib/customize-monitoring.nix
@@ -32,6 +32,10 @@
# logins to Grafana.
, googleOAuthClientID
+ # Whether to enable alerting via Slack.
+ # When true requires a grafana-slack-url file (see private-keys/README.rst).
+, enableSlackAlert ? false
+
# A string giving the NixOS state version for the system.
, stateVersion
, ...
@@ -71,12 +75,25 @@ in {
action = ["sudo" "systemctl" "restart" "grafana.service"];
};
};
+ grafanaSlackUrl =
+ if !enableSlackAlert
+ then { }
+ else {
+ "grafana-slack-url" = {
+ source = "${privateKeyPath}/grafana-slack-url";
+ destination = "/run/keys/grafana-slack-url";
+ owner.user = config.systemd.services.grafana.serviceConfig.User;
+ owner.group = config.users.users.grafana.group;
+ permissions = "0400";
+ action = ["sudo" "systemctl" "restart" "grafana.service"];
+ };
+ };
monitoringvpn = {
"monitoringvpn-private-key".source = "${privateKeyPath}/monitoringvpn/server.key";
"monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key";
};
in
- grafanaSSO // monitoringvpn;
+ grafanaSSO // grafanaSlackUrl // monitoringvpn;
networking.hosts = hostsMap;
@@ -96,6 +113,7 @@ in {
services.private-storage.monitoring.grafana = {
inherit letsEncryptAdminEmail;
inherit googleOAuthClientID;
+ inherit enableSlackAlert;
domain = "${config.networking.hostName}.${config.networking.domain}";
};
diff --git a/nixos/modules/monitoring/server/grafana-config/payments.json b/nixos/modules/monitoring/server/grafana-dashboards/payments.json
similarity index 100%
rename from nixos/modules/monitoring/server/grafana-config/payments.json
rename to nixos/modules/monitoring/server/grafana-dashboards/payments.json
diff --git a/nixos/modules/monitoring/server/grafana-config/resources-overview.json b/nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json
similarity index 100%
rename from nixos/modules/monitoring/server/grafana-config/resources-overview.json
rename to nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json
diff --git a/nixos/modules/monitoring/server/grafana.nix b/nixos/modules/monitoring/server/grafana.nix
index c23150238241db561bae52aa50e4878b6961f9e6..1783782ce7e395f9201dd93e2386f4eed4bf003e 100644
--- a/nixos/modules/monitoring/server/grafana.nix
+++ b/nixos/modules/monitoring/server/grafana.nix
@@ -62,12 +62,36 @@ in {
default = /run/keys/grafana-admin.password;
description = "A file containing the password for the Grafana Admin account.";
};
+ enableSlackAlert = lib.mkOption
+ { type = lib.types.bool;
+ default = false;
+ description = ''
+ Enables the slack alerter. Expects a file that contains
+ the definition of an environment variable named SLACKURL
+ pointing to the secret Slack Web Hook URL in
+ grafanaSlackUrlFile (see below).
+ '';
+ };
+ grafanaSlackUrlFile = lib.mkOption
+ { type = lib.types.path;
+ default = /run/keys/grafana-slack-url;
+ description = ''
+ Where to find the Grafana Systemd EnvironmentFile that
+ sets the secret SLACKURL environment variable.
+ '';
+ };
};
config = {
# Port 80 for ACME ssl retrieval only. 443 for nginx -> grafana.
networking.firewall.allowedTCPPorts = [ 80 443 ];
+ # We pass the secret Slack URL using an environment variable.
+ systemd.services.grafana.serviceConfig.EnvironmentFile =
+ if cfg.enableSlackAlert
+ then [ cfg.grafanaSlackUrlFile ]
+ else [ ];
+
services.grafana = {
enable = true;
domain = cfg.domain;
@@ -119,8 +143,23 @@ in {
# See https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards
dashboards = [{
name = "provisioned";
- options.path = ./grafana-config;
+ options.path = ./grafana-dashboards;
}];
+ # See https://grafana.com/docs/grafana/latest/administration/provisioning/#example-alert-notification-channels-config-file
+ notifiers = [ ] ++ (lib.optionals (cfg.enableSlackAlert) [{
+ uid = "slack-notifier-1";
+ name = "Slack";
+ type = "slack";
+ is_default = true;
+ send_reminder = false;
+ settings = {
+ username = "${cfg.domain}";
+ uploadImage = true;
+ };
+ secure_settings = {
+ url = "$SLACKURL";
+ };
+ }]);
};
};