From a052f26462d21bcc3190652034b6a096bb71cc2e Mon Sep 17 00:00:00 2001
From: Florian Sesser <florian@private.storage>
Date: Wed, 19 Apr 2023 17:26:00 +0000
Subject: [PATCH] Add Zulip alerting
---
DEPLOYMENT-NOTES.rst | 5 ++++
morph/grid/local/private-keys/README.rst | 7 ++++++
.../grid/local/private-keys/grafana-zulip-url | 1 +
morph/grid/testing/grid.nix | 2 +-
morph/lib/monitoring.nix | 9 +++++++
nixos/modules/monitoring/server/grafana.nix | 25 +++++++++++++++++++
6 files changed, 48 insertions(+), 1 deletion(-)
create mode 100644 morph/grid/local/private-keys/grafana-zulip-url
diff --git a/DEPLOYMENT-NOTES.rst b/DEPLOYMENT-NOTES.rst
index 4337b43b..0fcb897a 100644
--- a/DEPLOYMENT-NOTES.rst
+++ b/DEPLOYMENT-NOTES.rst
@@ -1,6 +1,11 @@
Deployment notes
================
+- 2023-04-19
+
+ The team switched from Slack to Zulip.
+ For the monitoring notifications to reach Zulip, a webhook bot has to be created in Zulip and a secret URL has to be constructed as described in `https://zulip.com/integrations/doc/grafana`_ and added to the ``private_keys`` directory. See ``grid/local/private-keys/grafana-zulip-url`` for an example.
+
- 2021-12-20
`https://whetstone.private.storage/privatestorage/privatestorageops/-/issues/399`_ requires moving the PaymentServer database on the ``payments`` host onto a new dedicated filesystem.
diff --git a/morph/grid/local/private-keys/README.rst b/morph/grid/local/private-keys/README.rst
index 176f0d54..17976e14 100644
--- a/morph/grid/local/private-keys/README.rst
+++ b/morph/grid/local/private-keys/README.rst
@@ -27,6 +27,13 @@ This file is read by Grafana's systemd service to set an environment variable wi
The only line in the file should be the secret URL.
Use the url from `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ or get a new secret URL for your Slack channel at https://www.slack.com/apps/A0F7XDUAZ.
+grafana-zulip-url
+-----------------
+
+This file should contain a single line with the secret Zulip alerting Webhook Bot URL.
+The URLs for Staging and Production are both stored in 1Password.
+See `https://zulip.com/integrations/doc/grafana`_ for documentation and ``grid/local/private-keys/grafana-zulip-url`` for an example.
+
stripe.secret
-------------
diff --git a/morph/grid/local/private-keys/grafana-zulip-url b/morph/grid/local/private-keys/grafana-zulip-url
new file mode 100644
index 00000000..4b83d4f6
--- /dev/null
+++ b/morph/grid/local/private-keys/grafana-zulip-url
@@ -0,0 +1 @@
+https://yourZulipDomain.zulipchat.com/api/v1/external/grafana?api_key=abcdefgh&stream=stream%20name&topic=your%20topic
diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix
index c033da12..5f3ec05f 100644
--- a/morph/grid/testing/grid.nix
+++ b/morph/grid/testing/grid.nix
@@ -70,7 +70,7 @@ let
inherit paymentExporterTargets blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
- enableSlackAlert = true;
+ enableZulipAlert = true;
};
system.stateVersion = "19.09";
};
diff --git a/morph/lib/monitoring.nix b/morph/lib/monitoring.nix
index d2552ebe..ba64dd90 100644
--- a/morph/lib/monitoring.nix
+++ b/morph/lib/monitoring.nix
@@ -77,6 +77,15 @@ in {
When true requires a grafana-slack-url file (see private-keys/README.rst).
'';
};
+
+ enableZulipAlert = lib.mkOption {
+ type = lib.types.bool;
+ default = false;
+ description = ''
+ Whether to enable alerting via Zulip.
+ When true requires a grafana-zulip-url file (see private-keys/README.rst).
+ '';
+ };
};
config = {
diff --git a/nixos/modules/monitoring/server/grafana.nix b/nixos/modules/monitoring/server/grafana.nix
index ab7c7afc..28fcd0e0 100644
--- a/nixos/modules/monitoring/server/grafana.nix
+++ b/nixos/modules/monitoring/server/grafana.nix
@@ -83,6 +83,21 @@ in {
Where to find the file that containts the slack URL.
'';
};
+ enableZulipAlert = lib.mkOption
+ { type = lib.types.bool;
+ default = false;
+ description = ''
+ Enables the Zulip alerter. Expects a file that contains
+ the secret Zulip Web Hook URL in grafanaZulipUrlFile (see below).
+ '';
+ };
+ grafanaZulipUrlFile = lib.mkOption
+ { type = lib.types.path;
+ default = /run/keys/grafana-zulip-url;
+ description = ''
+ Where to find the file that containts the Zulip URL.
+ '';
+ };
};
config =
@@ -165,6 +180,16 @@ in {
# See https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider
url = "$__file{${toString cfg.grafanaSlackUrlFile}}";
};
+ }]) ++ (lib.optionals (cfg.enableZulipAlert) [{
+ # See https://zulip.com/integrations/doc/grafana
+ uid = "zulip-notifier-1";
+ name = "Zulip";
+ type = "webhook";
+ is_default = true;
+ send_reminder = false;
+ secure_settings = {
+ url = "$__file{${toString cfg.grafanaZulipUrlFile}}";
+ };
}]);
};
};
--
GitLab