Merge branch '392.enable-alerting' into 'develop'

Enable alerting

Closes privatestorageops#392

See merge request !185

DEPLOYMENT-NOTES.rst

 Deployment notes
 ================
 
+- 2021-09-30 `Enable alerting <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/185>`_ needs a secret in ``private-keys/grafana-slack-url`` looking like the template in ``morph/grid/local/private-keys/grafana-slack-url`` and pointing to the secret API endpoint URL saved in `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ (or create a new secret URL at https://www.slack.com/apps/A0F7XDUAZ).
+
 - 2021-09-07 `Manage access to payment metrics <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/146>`_ requires moving and chown'ing the PaymentServer database on the ``payments`` host::
 
    mkdir /var/lib/zkapissuer

morph/grid/local/grid.nix

@@ -108,6 +108,7 @@ let
         inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
         inherit (grid-config) letsEncryptAdminEmail;
         googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+        enableSlackAlert = false;
         monitoringvpnIPv4 = "172.23.23.1";
         stateVersion = "19.09";
       })

morph/grid/local/private-keys/README.rst

@@ -19,6 +19,13 @@ grafana-admin.password
 
 This is the initial admin password for the Grafana web admin on the monitoring host.
 
+grafana-slack-url
+-----------------
+
+This file is read by Grafana's systemd service to set an environment variable with a secret Slack WebHook URL to post alerts to.
+The only line in the file should be ``SLACKURL=`` with the secret URL.
+Use the url from `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ or get a new secret URL for your Slack channel at https://www.slack.com/apps/A0F7XDUAZ.
+
 stripe.secret
 -------------
 

morph/grid/local/private-keys/grafana-slack-url

+SLACKURL=https://hooks.slack.com/services/x/y/z
+

morph/grid/production/grid.nix

@@ -48,6 +48,7 @@ let
         inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
         inherit (grid-config) letsEncryptAdminEmail;
         googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+        enableSlackAlert = true;
         monitoringvpnIPv4 = "172.23.23.1";
         stateVersion = "19.09";
       })

morph/grid/testing/grid.nix

@@ -61,6 +61,7 @@ let
         inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
         inherit (grid-config) letsEncryptAdminEmail;
         googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+        enableSlackAlert = true;
         monitoringvpnIPv4 = "172.23.23.1";
         stateVersion = "19.09";
       })

morph/lib/customize-monitoring.nix

@@ -32,6 +32,10 @@
   # logins to Grafana.
 , googleOAuthClientID
 
+  # Whether to enable alerting via Slack.
+  # When true requires a grafana-slack-url file (see private-keys/README.rst).
+, enableSlackAlert ? false
+
   # A string giving the NixOS state version for the system.
 , stateVersion
 , ...
@@ -71,12 +75,25 @@ in {
           action = ["sudo" "systemctl" "restart" "grafana.service"];
         };
       };
+    grafanaSlackUrl =
+      if !enableSlackAlert
+      then { }
+      else {
+        "grafana-slack-url" = {
+          source = "${privateKeyPath}/grafana-slack-url";
+          destination = "/run/keys/grafana-slack-url";
+          owner.user = config.systemd.services.grafana.serviceConfig.User;
+          owner.group = config.users.users.grafana.group;
+          permissions = "0400";
+          action = ["sudo" "systemctl" "restart" "grafana.service"];
+        };
+      };
     monitoringvpn = {
       "monitoringvpn-private-key".source = "${privateKeyPath}/monitoringvpn/server.key";
       "monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key";
     };
     in
-      grafanaSSO // monitoringvpn;
+      grafanaSSO // grafanaSlackUrl // monitoringvpn;
 
   networking.hosts = hostsMap;
 
@@ -96,6 +113,7 @@ in {
   services.private-storage.monitoring.grafana = {
     inherit letsEncryptAdminEmail;
     inherit googleOAuthClientID;
+    inherit enableSlackAlert;
     domain = "${config.networking.hostName}.${config.networking.domain}";
   };
 

nixos/modules/monitoring/server/grafana.nix

@@ -62,12 +62,36 @@ in {
       default = /run/keys/grafana-admin.password;
       description = "A file containing the password for the Grafana Admin account.";
     };
+    enableSlackAlert = lib.mkOption
+    { type = lib.types.bool;
+      default = false;
+      description = ''
+        Enables the slack alerter. Expects a file that contains
+        the definition of an environment variable named SLACKURL
+        pointing to the secret Slack Web Hook URL in
+        grafanaSlackUrlFile (see below).
+      '';
+    };
+    grafanaSlackUrlFile = lib.mkOption
+    { type = lib.types.path;
+      default = /run/keys/grafana-slack-url;
+      description = ''
+        Where to find the Grafana Systemd EnvironmentFile that
+        sets the secret SLACKURL environment variable.
+      '';
+    };
   };
 
   config = {
     # Port 80 for ACME ssl retrieval only. 443 for nginx -> grafana.
     networking.firewall.allowedTCPPorts = [ 80 443 ];
 
+    # We pass the secret Slack URL using an environment variable.
+    systemd.services.grafana.serviceConfig.EnvironmentFile =
+      if cfg.enableSlackAlert
+      then [ cfg.grafanaSlackUrlFile ]
+      else [ ];
+
     services.grafana = {
       enable = true;
       domain = cfg.domain;
@@ -119,8 +143,23 @@ in {
         # See https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards
         dashboards = [{
           name = "provisioned";
-          options.path = ./grafana-config;
+          options.path = ./grafana-dashboards;
         }];
+        # See https://grafana.com/docs/grafana/latest/administration/provisioning/#example-alert-notification-channels-config-file
+        notifiers = [ ] ++ (lib.optionals (cfg.enableSlackAlert) [{
+          uid = "slack-notifier-1";
+          name = "Slack";
+          type = "slack";
+          is_default = true;
+          send_reminder = false;
+          settings = {
+            username = "${cfg.domain}";
+            uploadImage = true;
+          };
+          secure_settings = {
+            url = "$SLACKURL";
+          };
+        }]);
       };
     };