diff --git a/DEPLOYMENT-NOTES.rst b/DEPLOYMENT-NOTES.rst index 4337b43b8cf365ba37c50ced218cb49e42708a27..9603da2e899d63161486313a7acce7dd05f9b14a 100644 --- a/DEPLOYMENT-NOTES.rst +++ b/DEPLOYMENT-NOTES.rst @@ -1,6 +1,12 @@ Deployment notes ================ +- 2023-04-19 + + The team switched from Slack to Zulip. + For the monitoring notifications to reach Zulip, a webhook bot has to be created in Zulip and a secret URL has to be constructed as described in `https://zulip.com/integrations/doc/grafana`_ and added to the ``private_keys`` directory (See ``grid/local/private-keys/grafana-zulip-url`` for an example). + Find the secret URL for production at `https://my.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/rb22ipb6gvokohzq2d2hhv6t6u`_. + - 2021-12-20 `https://whetstone.private.storage/privatestorage/privatestorageops/-/issues/399`_ requires moving the PaymentServer database on the ``payments`` host onto a new dedicated filesystem. diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix index 088d9e8c79422b82d638a42aeab5da1fcf14f536..0c9f3488eceeee0aee5308441712fe9bdb052ddc 100644 --- a/morph/grid/local/grid.nix +++ b/morph/grid/local/grid.nix @@ -122,7 +122,7 @@ let inherit paymentExporterTargets blackboxExporterHttpsTargets; inherit (grid-config) monitoringDomains; googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; - enableSlackAlert = false; + enableZulipAlert = false; }; system.stateVersion = "19.09"; }; diff --git a/morph/grid/local/private-keys/README.rst b/morph/grid/local/private-keys/README.rst index 176f0d54a9761281b273ab0bdeb710f219807ece..17976e1499e16adaafb982c0360e6ed32ea5c442 100644 --- a/morph/grid/local/private-keys/README.rst +++ b/morph/grid/local/private-keys/README.rst @@ -27,6 +27,13 @@ This file is read by Grafana's systemd service to set an environment variable wi The only line in the file should be the secret URL. Use the url from `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ or get a new secret URL for your Slack channel at https://www.slack.com/apps/A0F7XDUAZ. +grafana-zulip-url +----------------- + +This file should contain a single line with the secret Zulip alerting Webhook Bot URL. +The URLs for Staging and Production are both stored in 1Password. +See `https://zulip.com/integrations/doc/grafana`_ for documentation and ``grid/local/private-keys/grafana-zulip-url`` for an example. + stripe.secret ------------- diff --git a/morph/grid/local/private-keys/grafana-zulip-url b/morph/grid/local/private-keys/grafana-zulip-url new file mode 100644 index 0000000000000000000000000000000000000000..4b83d4f6301b8ede9a19ce2c2b7b89c492deddf6 --- /dev/null +++ b/morph/grid/local/private-keys/grafana-zulip-url @@ -0,0 +1 @@ +https://yourZulipDomain.zulipchat.com/api/v1/external/grafana?api_key=abcdefgh&stream=stream%20name&topic=your%20topic diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix index 06fe07f8277bf81e26e2f9f735783614c117a7b3..cf77dddb4a5b33fbabef9b5eeb40e042f8dd68ff 100644 --- a/morph/grid/production/grid.nix +++ b/morph/grid/production/grid.nix @@ -54,7 +54,7 @@ let inherit paymentExporterTargets blackboxExporterHttpsTargets; inherit (grid-config) monitoringDomains; googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; - enableSlackAlert = true; + enableZulipAlert = true; }; system.stateVersion = "19.09"; }; diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix index c033da1279fa44800e994dc07df3f5febc97d60d..5f3ec05f520d09169e7a5627283b4a05b7fa87f2 100644 --- a/morph/grid/testing/grid.nix +++ b/morph/grid/testing/grid.nix @@ -70,7 +70,7 @@ let inherit paymentExporterTargets blackboxExporterHttpsTargets; inherit (grid-config) monitoringDomains; googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID; - enableSlackAlert = true; + enableZulipAlert = true; }; system.stateVersion = "19.09"; }; diff --git a/morph/lib/monitoring.nix b/morph/lib/monitoring.nix index d2552ebedf026a07a92da783b1de410be9ceca38..a5f2575aaef5fca0cf15f5d125981f150a0f20a3 100644 --- a/morph/lib/monitoring.nix +++ b/morph/lib/monitoring.nix @@ -77,6 +77,15 @@ in { When true requires a grafana-slack-url file (see private-keys/README.rst). ''; }; + + enableZulipAlert = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to enable alerting via Zulip. + When true requires a grafana-zulip-url file (see private-keys/README.rst). + ''; + }; }; config = { @@ -138,6 +147,16 @@ in { action = ["sudo" "systemctl" "restart" "grafana.service"]; }; }) + (lib.mkIf cfg.enableZulipAlert { + "grafana-zulip-url" = { + source = "${privateKeyPath}/grafana-zulip-url"; + destination = "/run/keys/grafana-zulip-url"; + owner.user = config.systemd.services.grafana.serviceConfig.User; + owner.group = config.users.users.grafana.group; + permissions = "0400"; + action = ["sudo" "systemctl" "restart" "grafana.service"]; + }; + }) ]; networking.hosts = hostsMap; @@ -156,7 +175,7 @@ in { }; services.private-storage.monitoring.grafana = { - inherit (cfg) googleOAuthClientID enableSlackAlert ; + inherit (cfg) googleOAuthClientID enableSlackAlert enableZulipAlert; inherit letsEncryptAdminEmail; domains = cfg.monitoringDomains; }; diff --git a/nixos/modules/monitoring/server/grafana.nix b/nixos/modules/monitoring/server/grafana.nix index ab7c7afc5e61113e05df08f5fa39d63f24da5957..91b3641e8d7ae52b4da320e452b0ae0be21652d8 100644 --- a/nixos/modules/monitoring/server/grafana.nix +++ b/nixos/modules/monitoring/server/grafana.nix @@ -83,6 +83,21 @@ in { Where to find the file that containts the slack URL. ''; }; + enableZulipAlert = lib.mkOption + { type = lib.types.bool; + default = false; + description = '' + Enables the Zulip alerter. Expects a file that contains + the secret Zulip Web Hook URL in grafanaZulipUrlFile (see below). + ''; + }; + grafanaZulipUrlFile = lib.mkOption + { type = lib.types.path; + default = /run/keys/grafana-zulip-url; + description = '' + Where to find the file that containts the Zulip URL. + ''; + }; }; config = @@ -165,6 +180,16 @@ in { # See https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider url = "$__file{${toString cfg.grafanaSlackUrlFile}}"; }; + }]) ++ (lib.optionals (cfg.enableZulipAlert) [{ + # See https://zulip.com/integrations/doc/grafana + uid = "zulip-notifier-1"; + name = "Zulip"; + type = "webhook"; + is_default = true; + send_reminder = false; + settings = { + url = "$__file{${toString cfg.grafanaZulipUrlFile}}"; + }; }]); }; };