diff --git a/DEPLOYMENT-NOTES.rst b/DEPLOYMENT-NOTES.rst
index 4337b43b8cf365ba37c50ced218cb49e42708a27..79a98dc5676e397287909470382eb5722ee647de 100644
--- a/DEPLOYMENT-NOTES.rst
+++ b/DEPLOYMENT-NOTES.rst
@@ -1,6 +1,20 @@
Deployment notes
================
+- 2023-06-19
+
+ ZKAPAuthorizer's Tahoe-LAFS plugin name changed from "privatestorageio-zkapauthz-v1" to "privatestorageio-zkapauthz-v2".
+ This causes Tahoe-LAFS to use a different filename to persist the plugin's Foolscap fURL.
+ To preserve the original fURL value (required) each storage node needs this command run before the deployment::
+
+ cp /var/db/tahoe-lafs/storage/private/storage-plugin.privatestorageio-zkapauthz-v{1,2}.furl
+
+- 2023-04-19
+
+ The team switched from Slack to Zulip.
+ For the monitoring notifications to reach Zulip, a webhook bot has to be created in Zulip and a secret URL has to be constructed as described in `https://zulip.com/integrations/doc/grafana`_ and added to the ``private_keys`` directory (See ``grid/local/private-keys/grafana-zulip-url`` for an example).
+ Find the secret URL for production at `https://my.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/rb22ipb6gvokohzq2d2hhv6t6u`_.
+
- 2021-12-20
`https://whetstone.private.storage/privatestorage/privatestorageops/-/issues/399`_ requires moving the PaymentServer database on the ``payments`` host onto a new dedicated filesystem.
diff --git a/docs/conf.py b/docs/conf.py
index 747a90a8cc039e65fd01c3d598170c001599c1c8..f3347073a977bdc170c3e1639c5610bf96b790b0 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -61,7 +61,7 @@ master_doc = 'index'
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
diff --git a/docs/dev/README.rst b/docs/dev/README.rst
index 0c688021dfa0aef8d4e10a6c6501dd5a6a5b6d23..c518c69bfc30025dbe297d3e62b12c5fe8ff9f64 100644
--- a/docs/dev/README.rst
+++ b/docs/dev/README.rst
@@ -24,7 +24,6 @@ The system tests are run using this command::
$ nix-build --attr system-tests
-The system tests boot QEMU VMs which prevents them from running on CI at this time.
The build requires > 10 GB of disk space,
and the VMs might be timing out on slow or busy machines.
If you run into timeouts,
diff --git a/docs/ops/monitoring-architecture.drawio b/docs/ops/monitoring-architecture.drawio
new file mode 100644
index 0000000000000000000000000000000000000000..a9788eeeffdf745a842d28e2cf41b63eb164252b
--- /dev/null
+++ b/docs/ops/monitoring-architecture.drawio
@@ -0,0 +1,127 @@
+<mxfile host="app.diagrams.net" modified="2023-04-20T20:17:44.466Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" etag="8PyLTVr0G94q4Dna4Dsz" version="21.2.1" type="device">
+ <diagram name="Page-1" id="aaaa8250-4180-3840-79b5-4cada1eebb92">
+ <mxGraphModel dx="794" dy="476" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1920" pageHeight="1200" background="#ffffff" math="0" shadow="0">
+ <root>
+ <mxCell id="0" />
+ <mxCell id="1" parent="0" />
+ <mxCell id="vhdg0YFc32S7_3H95Ew1-1" value="<span>Management VPN<br>(Wireshark, TINC...)</span>" style="ellipse;shape=cloud;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+ <mxGeometry x="780" y="720" width="450" height="110" as="geometry" />
+ </mxCell>
+ <mxCell id="2mYkRctJDop23S32jJdh-2" value="" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+ <mxGeometry x="840" y="405.46" width="370" height="304.54" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-3" value="Loki" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.application2;fillColor=#86E83A;strokeColor=#B0F373;aspect=fixed;" parent="1" vertex="1">
+ <mxGeometry x="1116" y="592.9000000000001" width="62" height="53" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-4" value="Prometheus" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.application;fillColor=#4286c5;strokeColor=#57A2D8;aspect=fixed;" parent="1" vertex="1">
+ <mxGeometry x="866" y="585" width="62" height="68.8" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-5" value="Grafana" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.ami2;aspect=fixed;fillColor=#FF9900;strokeColor=#ffffff;" parent="1" vertex="1">
+ <mxGeometry x="996" y="425" width="74" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-6" value="Node 1" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;" parent="1" vertex="1">
+ <mxGeometry x="779" y="845" width="74" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-7" value="Operator" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.end_user;strokeColor=#9673a6;fillColor=#e1d5e7;aspect=fixed;" parent="1" vertex="1">
+ <mxGeometry x="1276" y="305" width="49" height="100.46" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-16" value="Node ..." style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;" parent="1" vertex="1">
+ <mxGeometry x="902" y="845" width="74" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-17" value="Node ..." style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;" parent="1" vertex="1">
+ <mxGeometry x="1025" y="845" width="74" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-18" value="Node N" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;" parent="1" vertex="1">
+ <mxGeometry x="1147.5" y="845" width="74" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-45" value="" style="endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;" parent="1" edge="1">
+ <mxGeometry x="806" y="695" width="50" height="50" as="geometry">
+ <mxPoint x="894.6666666666666" y="695" as="sourcePoint" />
+ <mxPoint x="936" y="835" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-46" value="" style="endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;" parent="1" edge="1">
+ <mxGeometry x="806" y="695" width="50" height="50" as="geometry">
+ <mxPoint x="894.6666666666666" y="695" as="sourcePoint" />
+ <mxPoint x="1046" y="835" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-47" value="" style="endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;" parent="1" edge="1">
+ <mxGeometry x="806" y="695" width="50" height="50" as="geometry">
+ <mxPoint x="894.6666666666666" y="695" as="sourcePoint" />
+ <mxPoint x="1166" y="835" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-50" value="" style="endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;" parent="1" edge="1">
+ <mxGeometry x="818.6666666666667" y="695" width="63.33333333333333" height="75" as="geometry">
+ <mxPoint x="846" y="835" as="sourcePoint" />
+ <mxPoint x="1148" y="695" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-51" value="" style="endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;" parent="1" edge="1">
+ <mxGeometry x="818.6666666666667" y="695" width="63.33333333333333" height="75" as="geometry">
+ <mxPoint x="946" y="835" as="sourcePoint" />
+ <mxPoint x="1148" y="695" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-52" value="" style="endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;" parent="1" edge="1">
+ <mxGeometry x="818.6666666666667" y="695" width="63.33333333333333" height="75" as="geometry">
+ <mxPoint x="1056" y="835" as="sourcePoint" />
+ <mxPoint x="1148" y="695" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-53" value="" style="endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;" parent="1" edge="1">
+ <mxGeometry x="818.6666666666667" y="695" width="63.33333333333333" height="75" as="geometry">
+ <mxPoint x="1186" y="835" as="sourcePoint" />
+ <mxPoint x="1148" y="695" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-57" value="" style="endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;" parent="1" edge="1">
+ <mxGeometry width="50" height="50" relative="1" as="geometry">
+ <mxPoint x="988" y="495" as="sourcePoint" />
+ <mxPoint x="928" y="565" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-58" value="" style="endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;" parent="1" edge="1">
+ <mxGeometry width="50" height="50" relative="1" as="geometry">
+ <mxPoint x="1078" y="495" as="sourcePoint" />
+ <mxPoint x="1136" y="565" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-61" value="View dashboards<br>in browser" style="shape=flexArrow;endArrow=classic;html=1;strokeColor=#9673a6;strokeWidth=1;fillColor=#e1d5e7;spacing=8;" parent="1" edge="1">
+ <mxGeometry width="50" height="50" relative="1" as="geometry">
+ <mxPoint x="1259.5" y="415" as="sourcePoint" />
+ <mxPoint x="1109.5" y="445" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-63" value="<h1>Monitoring architecture&nbsp;</h1><p>Keep it simple, sunshine!<br><br></p><p></p><i>Grafana</i> retrieves metrics from <i>Prometheus</i> and logs from&nbsp;<i>Loki</i>,&nbsp;<span>shows dashboards (web) and does alerting (via eMail? Slack?)</span><br><p><i>Prometheus</i> stores metrics it pulls from various <i>Exporters</i> on nodes</p><p><i>Promtail</i> on nodes pushes logs to <i>Loki<br><br></i></p><p>We try to keep the system as simple as possible: All monitoring and alerting runs on a single machine.</p><h2>Changes</h2><div>v2: Add Github authentication to Grafana. Add management VPN.<br><br>v1: Initial version</div>" style="text;html=1;strokeColor=none;fillColor=none;spacing=5;spacingTop=-20;whiteSpace=wrap;overflow=hidden;rounded=0;shadow=0;comic=0;sketch=0;" parent="1" vertex="1">
+ <mxGeometry x="596" y="315" width="164" height="545" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-65" value="Send alerts" style="shape=flexArrow;endArrow=classic;html=1;strokeColor=#9673a6;strokeWidth=1;fillColor=#e1d5e7;" parent="1" edge="1">
+ <mxGeometry width="50" height="50" relative="1" as="geometry">
+ <mxPoint x="1086" y="405.46000000000004" as="sourcePoint" />
+ <mxPoint x="1236" y="375.46000000000004" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="2mYkRctJDop23S32jJdh-3" value="Monitoring server" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+ <mxGeometry x="845" y="411" width="120" height="20" as="geometry" />
+ </mxCell>
+ <mxCell id="TrmSFti5pUnXIGkjMKb6-44" value="" style="endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;" parent="1" edge="1">
+ <mxGeometry x="806" y="695" width="50" height="50" as="geometry">
+ <mxPoint x="894.6666666666666" y="695" as="sourcePoint" />
+ <mxPoint x="826" y="835" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="vhdg0YFc32S7_3H95Ew1-7" value="GitHub<br>OAuth2" style="ellipse;shape=cloud;whiteSpace=wrap;html=1;" parent="1" vertex="1">
+ <mxGeometry x="984" y="305" width="98" height="60" as="geometry" />
+ </mxCell>
+ <mxCell id="vhdg0YFc32S7_3H95Ew1-8" value="" style="endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;" parent="1" edge="1">
+ <mxGeometry width="50" height="50" relative="1" as="geometry">
+ <mxPoint x="1032.76" y="417" as="sourcePoint" />
+ <mxPoint x="1032.76" y="372" as="targetPoint" />
+ </mxGeometry>
+ </mxCell>
+ </root>
+ </mxGraphModel>
+ </diagram>
+</mxfile>
diff --git a/docs/ops/monitoring-architecture.html b/docs/ops/monitoring-architecture.html
new file mode 100644
index 0000000000000000000000000000000000000000..91a6000fb4e8f6791fb26e8a83bc393024e52ca8
--- /dev/null
+++ b/docs/ops/monitoring-architecture.html
@@ -0,0 +1,12 @@
+<!--[if IE]><meta http-equiv="X-UA-Compatible" content="IE=5,IE=9" ><![endif]-->
+<!DOCTYPE html>
+<html>
+<head>
+<title>monitoring-architecture.html</title>
+<meta charset="utf-8"/>
+</head>
+<body>
+<div class="mxgraph" style="max-width:100%;border:1px solid transparent;" data-mxgraph="{"highlight":"#0000ff","nav":true,"resize":true,"xml":"<mxfile host=\"app.diagrams.net\" modified=\"2023-04-20T20:19:05.428Z\" agent=\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36\" etag=\"4ETb3iIJGh8a_GDWm07E\" version=\"21.2.1\" type=\"device\"><diagram name=\"Page-1\" id=\"aaaa8250-4180-3840-79b5-4cada1eebb92\"><mxGraphModel dx=\"794\" dy=\"476\" grid=\"1\" gridSize=\"10\" guides=\"1\" tooltips=\"1\" connect=\"1\" arrows=\"1\" fold=\"1\" page=\"1\" pageScale=\"1\" pageWidth=\"1920\" pageHeight=\"1200\" background=\"#ffffff\" math=\"0\" shadow=\"0\"><root><mxCell id=\"0\"/><mxCell id=\"1\" parent=\"0\"/><mxCell id=\"vhdg0YFc32S7_3H95Ew1-1\" value=\"&lt;span&gt;Management VPN&lt;br&gt;(WireGuard)&lt;/span&gt;\" style=\"ellipse;shape=cloud;whiteSpace=wrap;html=1;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"780\" y=\"720\" width=\"450\" height=\"110\" as=\"geometry\"/></mxCell><mxCell id=\"2mYkRctJDop23S32jJdh-2\" value=\"\" style=\"rounded=0;whiteSpace=wrap;html=1;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"840\" y=\"405.46\" width=\"370\" height=\"304.54\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-3\" value=\"Loki\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.application2;fillColor=#86E83A;strokeColor=#B0F373;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"1116\" y=\"592.9000000000001\" width=\"62\" height=\"53\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-4\" value=\"Prometheus\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.application;fillColor=#4286c5;strokeColor=#57A2D8;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"866\" y=\"585\" width=\"62\" height=\"68.8\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-5\" value=\"Grafana\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.ami2;aspect=fixed;fillColor=#FF9900;strokeColor=#ffffff;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"996\" y=\"425\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-6\" value=\"Node 1\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"779\" y=\"845\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-7\" value=\"Operator\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.end_user;strokeColor=#9673a6;fillColor=#e1d5e7;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"1276\" y=\"305\" width=\"49\" height=\"100.46\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-16\" value=\"Node ...\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"902\" y=\"845\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-17\" value=\"Node ...\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"1025\" y=\"845\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-18\" value=\"Node N\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"1147.5\" y=\"845\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-45\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"806\" y=\"695\" width=\"50\" height=\"50\" as=\"geometry\"><mxPoint x=\"894.6666666666666\" y=\"695\" as=\"sourcePoint\"/><mxPoint x=\"936\" y=\"835\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-46\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"806\" y=\"695\" width=\"50\" height=\"50\" as=\"geometry\"><mxPoint x=\"894.6666666666666\" y=\"695\" as=\"sourcePoint\"/><mxPoint x=\"1046\" y=\"835\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-47\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"806\" y=\"695\" width=\"50\" height=\"50\" as=\"geometry\"><mxPoint x=\"894.6666666666666\" y=\"695\" as=\"sourcePoint\"/><mxPoint x=\"1166\" y=\"835\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-50\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"818.6666666666667\" y=\"695\" width=\"63.33333333333333\" height=\"75\" as=\"geometry\"><mxPoint x=\"846\" y=\"835\" as=\"sourcePoint\"/><mxPoint x=\"1148\" y=\"695\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-51\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"818.6666666666667\" y=\"695\" width=\"63.33333333333333\" height=\"75\" as=\"geometry\"><mxPoint x=\"946\" y=\"835\" as=\"sourcePoint\"/><mxPoint x=\"1148\" y=\"695\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-52\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"818.6666666666667\" y=\"695\" width=\"63.33333333333333\" height=\"75\" as=\"geometry\"><mxPoint x=\"1056\" y=\"835\" as=\"sourcePoint\"/><mxPoint x=\"1148\" y=\"695\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-53\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"818.6666666666667\" y=\"695\" width=\"63.33333333333333\" height=\"75\" as=\"geometry\"><mxPoint x=\"1186\" y=\"835\" as=\"sourcePoint\"/><mxPoint x=\"1148\" y=\"695\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-57\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"988\" y=\"495\" as=\"sourcePoint\"/><mxPoint x=\"928\" y=\"565\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-58\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"1078\" y=\"495\" as=\"sourcePoint\"/><mxPoint x=\"1136\" y=\"565\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-61\" value=\"View dashboards&lt;br&gt;in browser\" style=\"shape=flexArrow;endArrow=classic;html=1;strokeColor=#9673a6;strokeWidth=1;fillColor=#e1d5e7;spacing=8;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"1259.5\" y=\"415\" as=\"sourcePoint\"/><mxPoint x=\"1109.5\" y=\"445\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-63\" value=\"&lt;h1&gt;Monitoring architecture&amp;nbsp;&lt;/h1&gt;&lt;p&gt;Keep it simple, sunshine!&lt;br&gt;&lt;br&gt;&lt;/p&gt;&lt;p&gt;&lt;/p&gt;&lt;i&gt;Grafana&lt;/i&gt; retrieves metrics from &lt;i&gt;Prometheus&lt;/i&gt; and logs from&amp;nbsp;&lt;i&gt;Loki&lt;/i&gt;,&amp;nbsp;&lt;span&gt;shows dashboards (in a web browser) and does alerting (via Zulip)&lt;/span&gt;&lt;br&gt;&lt;p&gt;&lt;i&gt;Prometheus&lt;/i&gt; stores metrics it pulls from various &lt;i&gt;Exporters&lt;/i&gt; on nodes&lt;/p&gt;&lt;p&gt;&lt;i&gt;Promtail&lt;/i&gt; on nodes pushes logs to &lt;i&gt;Loki&lt;br&gt;&lt;br&gt;&lt;/i&gt;&lt;/p&gt;&lt;p&gt;We try to keep the system as simple as possible: All monitoring and alerting runs on a single machine.&lt;/p&gt;&lt;h2&gt;Changes&lt;/h2&gt;&lt;div&gt;v3: Fix WireGuard/Wireshark braino, Update Auth (Google, not GitHub)&lt;/div&gt;&lt;div&gt;&lt;br&gt;&lt;/div&gt;&lt;div&gt;v2: Add Github authentication to Grafana. Add management VPN.&lt;br&gt;&lt;br&gt;v1: Initial version&lt;/div&gt;\" style=\"text;html=1;strokeColor=none;fillColor=none;spacing=5;spacingTop=-20;whiteSpace=wrap;overflow=hidden;rounded=0;shadow=0;comic=0;sketch=0;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"596\" y=\"315\" width=\"164\" height=\"605\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-65\" value=\"Send alerts\" style=\"shape=flexArrow;endArrow=classic;html=1;strokeColor=#9673a6;strokeWidth=1;fillColor=#e1d5e7;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"1086\" y=\"405.46000000000004\" as=\"sourcePoint\"/><mxPoint x=\"1236\" y=\"375.46000000000004\" as=\"targetPoint\"/></mxGeometry></mxCell><UserObject label=\"Monitoring server\" link=\"https://monitoring.private.storage/\" id=\"2mYkRctJDop23S32jJdh-3\"><mxCell style=\"text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"845\" y=\"411\" width=\"120\" height=\"20\" as=\"geometry\"/></mxCell></UserObject><mxCell id=\"TrmSFti5pUnXIGkjMKb6-44\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"806\" y=\"695\" width=\"50\" height=\"50\" as=\"geometry\"><mxPoint x=\"894.6666666666666\" y=\"695\" as=\"sourcePoint\"/><mxPoint x=\"826\" y=\"835\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"vhdg0YFc32S7_3H95Ew1-7\" value=\"GSuite&lt;br&gt;OAuth2\" style=\"ellipse;shape=cloud;whiteSpace=wrap;html=1;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"984\" y=\"305\" width=\"98\" height=\"60\" as=\"geometry\"/></mxCell><mxCell id=\"vhdg0YFc32S7_3H95Ew1-8\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"1032.76\" y=\"417\" as=\"sourcePoint\"/><mxPoint x=\"1032.76\" y=\"372\" as=\"targetPoint\"/></mxGeometry></mxCell></root></mxGraphModel></diagram></mxfile>","toolbar":"pages zoom layers lightbox","page":0}"></div>
+<script type="text/javascript" src="https://app.diagrams.net/js/viewer-static.min.js"></script>
+</body>
+</html>
diff --git a/docs/ops/monitoring.rst b/docs/ops/monitoring.rst
index e30831ade4ef71a0abd101e383065f706f586b63..53bd914aed57762302f5f762e59bd65ef54ba8d1 100644
--- a/docs/ops/monitoring.rst
+++ b/docs/ops/monitoring.rst
@@ -17,6 +17,19 @@ Analyzing long-term trends
How big is my database and how fast is it growing? How quickly is my daily-active user count growing?
+Architecture
+````````````
+
+Below you find a diagram of the software and systems that comprise our monitoring and alerting infrastructure.
+It has intentionally been kept simple, yet is already surprisingly complex (at least if you are new to the monitoring world).
+The software stack is industry standard and chosen so it would be easy to find solutions to problems and people who can help out.
+
+Log in to `staging <https://monitoring.privatestorage-staging.com/>`_ and `production <https://monitoring.private.storage/>`_ Grafana via your GSuite Private.Storage session.
+
+.. raw:: html
+ :file: monitoring-architecture.html
+
+
Introduction to our dashboards
``````````````````````````````
diff --git a/morph/grid/local/README.rst b/morph/grid/local/README.rst
index 48f395cb82fc272481a61f0d1ab425ffbd20cd02..75bc685852a65673bbb1e572249d34eb5b482db4 100644
--- a/morph/grid/local/README.rst
+++ b/morph/grid/local/README.rst
@@ -8,14 +8,18 @@ Issues with networking that looked like guest misconfigurations vanished after c
This requires `NixOS <https://nixos.org/>`_.
Nix without the OS will not work.
+
Use the local development environment
`````````````````````````````````````
-0. Add VirtualBox to your NixOs system configuration at ``/etc/nixos/configuration.nix``::
+0. Add to your NixOS system configuration at ``/etc/nixos/configuration.nix`` (and rebuild)::
- virtualisation.virtualbox.host.enable = true;
- # Save bytes and build time, optional but recommended:
- virtualisation.virtualbox.host.headless = true;
+ # Enable libvirt - likely incompatible with virtualisation.virtualbox!
+ virtualisation.libvirtd.enable = true;
+ # Required for LibVirt
+ security.polkit.enable = true;
+ # Enable HW acceleration if (nested virtualisation is) available
+ #boot.kernelModules = [ "kvm-amd" "kvm-intel" ];
1. Enter the morph local grid directory::
@@ -27,19 +31,27 @@ Use the local development environment
3. Build and start the VMs::
- VAGRANT_DEFAULT_PROVIDER=virtualbox vagrant up
+ vagrant up --provider=libvirt
+
+ Optionally, to switch from QEMU to KVM virtualization, edit the virtual machine definition of all the machines and replace the "qemu" on the first line with "kvm"::
+
+ sudo virsh list
+ sudo virsh edit <machine id> (once for every machine)
+ vagrant halt
+ vagrant up
+
4. Then, add the Vagrant SSH configuration to your user's ``~/.ssh/config`` file::
install -d ~/.ssh ; vagrant ssh-config >> ~/.ssh/config
- Latest Morph honors the ``SSH_CONFIG_FILE`` environment variable (`since 3f90aa88 (March 2020, v 1.5.0) <https://github.com/DBCDK/morph/commit/3f90aa885fac1c29fce9242452fa7c0c505744ef#diff-d155ad793bd62e6ea4c44ba985049ecb13a4f4f32f799791b2bce695a16c0101>`_), so in the future this should get a bit more convenient.
+ Latest Morph honors the ``SSH_CONFIG_FILE`` environment variable (`since 3f90aa88 (March 2020, v 1.5.0) <https://github.com/DBCDK/morph/commit/3f90aa885fac1c29fce9242452fa7c0c505744ef#diff-d155ad793bd62e6ea4c44ba985049ecb13a4f4f32f799791b2bce695a16c0101>`_), so in the future this should get a bit more convenient.
-6. Create a ``public-keys/users.nix`` file with your SSH key (see ``public-keys/users.nix.example`` for the format) so you'll be able to log in after deploying the new configuration::
+5. Create a ``public-keys/users.nix`` file with your SSH key (see ``public-keys/users.nix.example`` for the format) so you'll be able to log in after deploying the new configuration::
$EDITOR public-keys/users.nix
-7. Then, build and deploy our software to the Vagrant VMs::
+6. Then, build and deploy our software to the Vagrant VMs::
morph build grid.nix
morph push grid.nix
@@ -48,4 +60,4 @@ Use the local development environment
vagrant up
morph upload-secrets grid.nix
- You should now be able to log in with the users and keys you set in your ``users.nix`` file.
+You should now be able to log in with the users and keys you set in your ``users.nix`` file.
diff --git a/morph/grid/local/Vagrantfile b/morph/grid/local/Vagrantfile
index 64d4aec5aadc67e48c91cb0b8154b1107c23f1bb..911dd3f7570834060ed0879b738bb3ea2a61420d 100644
--- a/morph/grid/local/Vagrantfile
+++ b/morph/grid/local/Vagrantfile
@@ -1,29 +1,61 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
-# This Vagrantfile worked for Florian Sesser using Vagrant 2.2.16dev and
-# the VirtualBox Hypervisor. Earlier Vagrant and LibVirt did not work.
+# This Vagrantfile worked for Florian Sesser using Vagrant 2.2.19 and
+# the LibVirt with QEmu Hypervisor. Earlier Vagrant and VirtualBox did worked too.
+
+# Get a dedicated LibVirt pool name or use default one
+pool_name = ENV.has_key?('POOL_NAME') ? ENV['POOL_NAME'] : 'default'
+# For instance, one could create such pool beforehand as follows:
+# export POOL_NAME=morph_local_$(id -un)
+# POOL_PATH="/path/to/your/storage"
+# mkdir -p "${POOL_PATH}"
+# sudo virsh pool-define-as ${POOL_NAME} --type dir --target "${POOL_PATH}"
+# sudo virsh pool-autostart ${POOL_NAME}
+# sudo virsh pool-start ${POOL_NAME}
Vagrant.configure("2") do |config|
# For a complete reference, please see the online documentation at
# https://docs.vagrantup.com.
- config.vm.define "payments.localdev" do |config|
- config.vm.hostname = "payments"
- config.vm.box = "esselius/nixos"
- config.vm.box_version = "20.09"
- config.vm.box_check_update = false
+ # Select the base image
+ config.vm.box = "esselius/nixos"
+ config.vm.box_version = "20.09"
+ config.vm.box_check_update = false
+
+ # No need to sync the working dir. with the guest boxess
+ # Better use SFTP to transfer
+ config.vm.synced_folder ".", "/vagrant", disabled: true
+
+ # Tune LibVirt/QEmu guests
+ config.vm.provider :libvirt do |domain|
+ # The default of one CPU should work
+ # Increase to speed up boot/push/deploy
+ # domain.cpus = 1
# To use the self-updating deployment system you need more memory. Giving
# all of the VMs enough memory for this is rather taxing, though, and the
# self-updating deployment system is not particularly useful for local
# dev. But should you want to:
#
- # config.vm.provider "virtualbox" do |v|
- # v.memory = 4096
- # end
+ # domain.memory = 4096
+ #
+ # Meanwhile, 1024 was apparently the default with VirtualBox
+ domain.memory = 1024
+
+ # Using a specific pool may help to manage the disk space
+ domain.storage_pool_name = pool_name
+ domain.snapshot_pool_name = pool_name
+
+ # No need of graphics - better use serial
+ domain.graphics_type = "none"
+ domain.video_type = "none"
+ end
+
+ config.vm.define "payments.localdev" do |config|
+ config.vm.hostname = "payments"
- # Assign a static IP address inside the VirtualBox host-only (Vagrant
+ # Assign a static IP address inside the box host-only (Vagrant
# calls it "private") network. The address must be in the range
# VirtualBox allows.
# https://www.virtualbox.org/manual/ch06.html#network_hostonly says some
@@ -37,31 +69,26 @@ Vagrant.configure("2") do |config|
config.vm.define "storage1.localdev" do |config|
config.vm.hostname = "storage1"
- config.vm.box = "esselius/nixos"
- config.vm.box_version = "20.09"
- config.vm.box_check_update = false
config.vm.network "private_network", ip: "192.168.56.22"
end
config.vm.define "storage2.localdev" do |config|
config.vm.hostname = "storage2"
- config.vm.box = "esselius/nixos"
- config.vm.box_version = "20.09"
- config.vm.box_check_update = false
config.vm.network "private_network", ip: "192.168.56.23"
end
config.vm.define "monitoring.localdev" do |config|
config.vm.hostname = "monitoring"
- config.vm.box = "esselius/nixos"
- config.vm.box_version = "20.09"
- config.vm.box_check_update = false
config.vm.network "private_network", ip: "192.168.56.24"
end
# To make the VMs assign the static IPs to the network interfaces we need a rebuild:
- config.vm.provision "shell", inline: "echo '{nix.trustedUsers = [ \"@wheel\" \"root\" \"vagrant\" ];}' > /etc/nixos/custom-configuration.nix"
+ ## Rename to 'nix.settings.trusted-users' after 20.09 or so:
+ config.vm.provision "shell",
+ inline: "echo '{ nix.trustedUsers = [ \"@wheel\" \"root\" \"vagrant\" ]; boot.kernelParams = [ \"console=tty0\" \"console=ttyS0,115200\" ]; }' > /etc/nixos/custom-configuration.nix"
config.vm.provision "shell", inline: "nixos-rebuild switch"
+ config.vm.provision "shell", inline: "systemctl stop firewall.service"
+ config.vm.provision "shell", inline: "systemctl start serial-getty@ttyS0.service"
config.trigger.after :up do |trigger|
trigger.info = "Hostname and IP address this host actually uses:"
diff --git a/morph/grid/local/grid.nix b/morph/grid/local/grid.nix
index 088d9e8c79422b82d638a42aeab5da1fcf14f536..0c9f3488eceeee0aee5308441712fe9bdb052ddc 100644
--- a/morph/grid/local/grid.nix
+++ b/morph/grid/local/grid.nix
@@ -122,7 +122,7 @@ let
inherit paymentExporterTargets blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
- enableSlackAlert = false;
+ enableZulipAlert = false;
};
system.stateVersion = "19.09";
};
diff --git a/morph/grid/local/private-keys/README.rst b/morph/grid/local/private-keys/README.rst
index 176f0d54a9761281b273ab0bdeb710f219807ece..17976e1499e16adaafb982c0360e6ed32ea5c442 100644
--- a/morph/grid/local/private-keys/README.rst
+++ b/morph/grid/local/private-keys/README.rst
@@ -27,6 +27,13 @@ This file is read by Grafana's systemd service to set an environment variable wi
The only line in the file should be the secret URL.
Use the url from `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ or get a new secret URL for your Slack channel at https://www.slack.com/apps/A0F7XDUAZ.
+grafana-zulip-url
+-----------------
+
+This file should contain a single line with the secret Zulip alerting Webhook Bot URL.
+The URLs for Staging and Production are both stored in 1Password.
+See `https://zulip.com/integrations/doc/grafana`_ for documentation and ``grid/local/private-keys/grafana-zulip-url`` for an example.
+
stripe.secret
-------------
diff --git a/morph/grid/local/private-keys/grafana-zulip-url b/morph/grid/local/private-keys/grafana-zulip-url
new file mode 100644
index 0000000000000000000000000000000000000000..4b83d4f6301b8ede9a19ce2c2b7b89c492deddf6
--- /dev/null
+++ b/morph/grid/local/private-keys/grafana-zulip-url
@@ -0,0 +1 @@
+https://yourZulipDomain.zulipchat.com/api/v1/external/grafana?api_key=abcdefgh&stream=stream%20name&topic=your%20topic
diff --git a/morph/grid/production/grid.nix b/morph/grid/production/grid.nix
index 06fe07f8277bf81e26e2f9f735783614c117a7b3..cf77dddb4a5b33fbabef9b5eeb40e042f8dd68ff 100644
--- a/morph/grid/production/grid.nix
+++ b/morph/grid/production/grid.nix
@@ -54,7 +54,7 @@ let
inherit paymentExporterTargets blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
- enableSlackAlert = true;
+ enableZulipAlert = true;
};
system.stateVersion = "19.09";
};
diff --git a/morph/grid/testing/grid.nix b/morph/grid/testing/grid.nix
index c033da1279fa44800e994dc07df3f5febc97d60d..5f3ec05f520d09169e7a5627283b4a05b7fa87f2 100644
--- a/morph/grid/testing/grid.nix
+++ b/morph/grid/testing/grid.nix
@@ -70,7 +70,7 @@ let
inherit paymentExporterTargets blackboxExporterHttpsTargets;
inherit (grid-config) monitoringDomains;
googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
- enableSlackAlert = true;
+ enableZulipAlert = true;
};
system.stateVersion = "19.09";
};
diff --git a/morph/grid/testing/public-keys/users.nix b/morph/grid/testing/public-keys/users.nix
index 04d324309028a2c1c73b69b549982f958607548f..502dd255957e642f9bc0dece09cfa613c6cec7c0 100644
--- a/morph/grid/testing/public-keys/users.nix
+++ b/morph/grid/testing/public-keys/users.nix
@@ -1,8 +1,8 @@
let
jcalderone = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon"];
- flo = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la"];
+ flo = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII78HGtpjFxQo7wol85hqfoCqjdK9Nk7+82rwttyLHpe flo@la-staging"];
andreas = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILkZzBoIPpoDSVis3HZId+lOI+3VTQfmz1uc4Yau8p/5 andreas@leastauthority.com"];
- bdonneaux = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIRJNhnYYllhiNhTNQg+IcfbAudxxWzk/VF45E9G9dMn benoit@leastauthority.com"];
+ bdonneaux = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGgpTXgxEqQPSl17NzJkAJgeDSFS1Ke/qjCuVMTZLlna benoit@leastauthority.com" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIZtWY7t8HVnaz6bluYsrAlzZC3MZtb8g0nO5L5fCQKR benoit@leastauthority.com"];
shae = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICBODAgs7pGHfxkIZ8mZABUd1LlS9WhxGy0/6FvhlPYq shae@scannedinavian.com"];
chris = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDT3/sSNoJP3E17oFTYjHN+uOwTY1wVox9tff97iueIo4V88eAc/oXKETiwPkr33qbGwuoXKkxIXJVz8rnNtO6IjTm9qfgzPiRAUJjew8bunL+V7SbhQIv7nk1fyV/efaENElG8bdmzTEpgwGcEnyibqvHJSYX6W+dMCz3G1t/lv97if3ohZKENHuMC5hLfJbGHSGKFO5XdjEjeda9lDd9Ac8XyruaL7iqEefsC7GuUgNRn8V83vwuJMDAC2xXC2V11M65VkGs6WPAct2+llzTtYbsxjxVZXC4yU42eXJYfBZEcCTPtJsKJxQCqSgFOEUnOYiuS6p4Q7a97BfHJ9S9oOV8U/e7YeE4b9Q8TPNzvKTPBAsuKyLyNYekBDB7fOTFziuJy/L578EaDv2BxrsfyCQqtjLko6TIAUbbHvce8urWNvj7H+fNXaURLIQmSTOv/mMl+omkvbP3MNgSFdENpCZaHSTiDxjygf52xcinj6Ijf3uDvPY2UjIRrbWSNV4MYpZDfkqt9THY4QibmxhER/YGvY+0zfiYGqQpQMMbTUB9hhoO5AHnnhMszNG2V9i70VqWyEMsS+Sr1+gOVAPraLp/tqHaqZk7/c4DDpILjA+4davTL6lgaiewx8a0ZEPAKZCZkOMovZKwkVIjyMvfekUkf1cF+QigJPZzcWSWEjQ== cardno:000608671823"];
meejah = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEfCDWivT0SCWoMyxUslX0upuhR4X3rNFh5rc/lCcBbe meejah@buyan" "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDkDNUL9OPvTNTijHovvLwdmgATvezS9tkToKrO6U9Gq17SBfFcb2a1nAADt9nmHtu3KExGqGrJeNkoMqGsbo+Y/BCgAz7yutL0PkoDQ4xRcl88kkk+4NtpWFhXelITIJopaNOW5E2qzkvt8FNXKnUfJpmJh+0v1wYseGKMSUncSYTb3vEViVj3DwgLgzQi/YxI/OrEKML0B+vA+n8t0XrqiHh5Ryathk5DFpss5P+0dfWC4PoJZuWbAdQsxqTm7fqmPrX7IfahZpvFHpru2OUICc2sxzoJI7//3bdXfFGkMh0cKG2pRIy2KSJ0IOnLiaACRHeIG2zcnKJLkx6Xbzfr mike@mantle"];
diff --git a/morph/lib/borgbackup.nix b/morph/lib/borgbackup.nix
index 945d058ddc84f1d1fa02226e5ec33c0bf1a413af..16df515caa5b8ed5e4bfe4eb051153abd8a80b9b 100644
--- a/morph/lib/borgbackup.nix
+++ b/morph/lib/borgbackup.nix
@@ -80,7 +80,7 @@ in {
BORG_RSH = "ssh -i /run/keys/borgbackup/ssh-key -o StrictHostKeyChecking=accept-new";
BORG_REPO = lib.fileContents "${publicKeyPath}/borgbackup/${config.networking.hostName}.repopath";
};
- script = ''${pkgs.borgbackup}/bin/borg check'';
+ script = ''${pkgs.borgbackup}/bin/borg check --verbose --log-json'';
};
};
}
diff --git a/morph/lib/hardware-vagrant.nix b/morph/lib/hardware-vagrant.nix
index 6c41af4923861e89d144303d129d7babde494363..c13cef856552e43e1bdfcab8bffce487dd4c0887 100644
--- a/morph/lib/hardware-vagrant.nix
+++ b/morph/lib/hardware-vagrant.nix
@@ -15,18 +15,19 @@
};
config = {
- virtualisation.virtualbox.guest.enable = true;
+ services.qemuGuest.enable = true;
- boot.loader.grub.device = "/dev/sda";
+ boot.loader.grub.device = "/dev/vda";
- boot.initrd.availableKernelModules = [ "ata_piix" "sd_mod" "sr_mod" ];
+ boot.initrd.availableKernelModules = [ "ata_piix" "virtio_pci" "virtio_blk" "sd_mod" "sr_mod" ];
boot.kernel.sysctl = { "vm.swappiness" = 0; };
+ boot.kernelParams = [ "console=tty0" "console=ttyS0,115200" ];
# remove the fsck that runs at startup. It will always fail to run, stopping
# your boot until you press *.
boot.initrd.checkJournalingFS = false;
- networking.interfaces.enp0s8.ipv4.addresses = [{
+ networking.interfaces.ens5.ipv4.addresses = [{
address = config.grid.publicIPv4;
prefixLength = 24;
}];
@@ -47,11 +48,11 @@
fileSystems."/storage" = { fsType = "tmpfs"; };
fileSystems."/" =
- { device = "/dev/sda1";
+ { device = "/dev/vda1";
fsType = "ext4";
};
# We want to push packages with morph without having to sign them
- nix.trustedUsers = [ "@wheel" "root" "vagrant" ];
+ nix.settings.trusted-users = [ "@wheel" "root" "vagrant" ];
};
}
diff --git a/morph/lib/issuer-aws.nix b/morph/lib/issuer-aws.nix
index 80495e2dc7bafbc9bfbbe174e1a2f75f66942dfe..8a7c14ecaa73cd7aebadebee5f534988baef94b6 100644
--- a/morph/lib/issuer-aws.nix
+++ b/morph/lib/issuer-aws.nix
@@ -13,6 +13,12 @@
randomEncryption = true;
} ];
+ # If we don't manually and explicitly early-load the loop module, crypt-swap
+ # setup fails with the not very helpful message: "loop device with autoclear
+ # flag is required"
+ # See https://unix.stackexchange.com/a/554500/81275
+ boot.kernelModules = [ "loop" ];
+
# Break the tie between AWS and morph for the hostname by forcing the
# morph-supplied name. See also
# <https://github.com/DBCDK/morph/issues/146>.
@@ -35,5 +41,5 @@
# Turn on automatic optimization of nix store
# https://nixos.wiki/wiki/Storage_optimization
- nix.autoOptimiseStore = true;
+ nix.settings.auto-optimise-store = true;
}
diff --git a/morph/lib/monitoring.nix b/morph/lib/monitoring.nix
index d2552ebedf026a07a92da783b1de410be9ceca38..a5f2575aaef5fca0cf15f5d125981f150a0f20a3 100644
--- a/morph/lib/monitoring.nix
+++ b/morph/lib/monitoring.nix
@@ -77,6 +77,15 @@ in {
When true requires a grafana-slack-url file (see private-keys/README.rst).
'';
};
+
+ enableZulipAlert = lib.mkOption {
+ type = lib.types.bool;
+ default = false;
+ description = ''
+ Whether to enable alerting via Zulip.
+ When true requires a grafana-zulip-url file (see private-keys/README.rst).
+ '';
+ };
};
config = {
@@ -138,6 +147,16 @@ in {
action = ["sudo" "systemctl" "restart" "grafana.service"];
};
})
+ (lib.mkIf cfg.enableZulipAlert {
+ "grafana-zulip-url" = {
+ source = "${privateKeyPath}/grafana-zulip-url";
+ destination = "/run/keys/grafana-zulip-url";
+ owner.user = config.systemd.services.grafana.serviceConfig.User;
+ owner.group = config.users.users.grafana.group;
+ permissions = "0400";
+ action = ["sudo" "systemctl" "restart" "grafana.service"];
+ };
+ })
];
networking.hosts = hostsMap;
@@ -156,7 +175,7 @@ in {
};
services.private-storage.monitoring.grafana = {
- inherit (cfg) googleOAuthClientID enableSlackAlert ;
+ inherit (cfg) googleOAuthClientID enableSlackAlert enableZulipAlert;
inherit letsEncryptAdminEmail;
domains = cfg.monitoringDomains;
};
diff --git a/nixos/lib/testing.nix b/nixos/lib/testing.nix
index d89717a0a76f93bb6062ad63c6cfdbb91c12c746..6ed4b733bffad29fbc52a492e295bc5ee7dddbe2 100644
--- a/nixos/lib/testing.nix
+++ b/nixos/lib/testing.nix
@@ -14,7 +14,8 @@
''
# The driver runs pyflakes on this script before letting it
# run... Convince pyflakes that there is a `test` name.
- test = None
+ def test():
+ pass
with open("${testpath}") as testfile:
exec(testfile.read(), globals())
# For simple types, JSON is compatible with Python syntax!
diff --git a/nixos/modules/deployment.nix b/nixos/modules/deployment.nix
index 41381ce5d33e62f4e569b87709d591f3586804df..17eb095f836d0ef75a23bad41a318b4a516eda2f 100755
--- a/nixos/modules/deployment.nix
+++ b/nixos/modules/deployment.nix
@@ -35,11 +35,11 @@ in {
config = {
# Configure the system to use our binary cache so that deployment updates
# only require downloading pre-built software, not building it ourselves.
- nix = {
- binaryCachePublicKeys = [
+ nix.settings = {
+ trusted-public-keys = [
"saxtons.private.storage:MplOcEH8G/6mRlhlKkbA8GdeFR3dhCFsSszrspE/ZwY="
];
- binaryCaches = [
+ substituters = [
"http://saxtons.private.storage"
];
};
diff --git a/nixos/modules/issuer.nix b/nixos/modules/issuer.nix
index 375f064c63c65b1b0b0c5fa94a121a4d68bba781..5537850b4dfc72038d8ba75f92881dd4762b222c 100644
--- a/nixos/modules/issuer.nix
+++ b/nixos/modules/issuer.nix
@@ -254,7 +254,7 @@ in {
];
# NGINX reverse proxy
- security.acme.email = cfg.letsEncryptAdminEmail;
+ security.acme.defaults.email = cfg.letsEncryptAdminEmail;
security.acme.acceptTerms = true;
services.nginx = {
enable = true;
diff --git a/nixos/modules/monitoring/exporters/node.nix b/nixos/modules/monitoring/exporters/node.nix
index 407011069ec0cfdec129244b37a60edd09a57f2b..4a9e41b5c840852323aaf952859e79b4c878a870 100644
--- a/nixos/modules/monitoring/exporters/node.nix
+++ b/nixos/modules/monitoring/exporters/node.nix
@@ -69,7 +69,7 @@ in {
] ++ (
optionals (config.services.nfs.server.enable) [ "nfsd" ]
) ++ (
- optionals ("" != config.boot.initrd.mdadmConf) [ "mdadm" ]
+ optionals ("" != config.boot.initrd.services.swraid.mdadmConf) [ "mdadm" ]
) ++ (
optionals ({} != config.networking.bonds) [ "bonding" ]
) ++ (
diff --git a/nixos/modules/monitoring/server/grafana-dashboards/meta-monitoring.json b/nixos/modules/monitoring/server/grafana-dashboards/meta-monitoring.json
index 5cd8ca5609b656bb0a8c0955d51d48485d16c371..d280cd9b208d4cc5485fd72a3bd4f3d5e47468fd 100644
--- a/nixos/modules/monitoring/server/grafana-dashboards/meta-monitoring.json
+++ b/nixos/modules/monitoring/server/grafana-dashboards/meta-monitoring.json
@@ -186,7 +186,7 @@
"for": "5m",
"frequency": "1m",
"handler": 1,
- "message": "A metrics text file is oder than 10 minutes.",
+ "message": "A metrics text file is older than 10 minutes.",
"name": "Textcollector staleness alert",
"noDataState": "no_data",
"notifications": []
diff --git a/nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json b/nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json
index 310041781486d8d260c29404bed60f2207012994..98e0e1e1157f0e0e9277c4e0f6eff40c38f08f67 100644
--- a/nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json
+++ b/nixos/modules/monitoring/server/grafana-dashboards/resources-overview.json
@@ -116,7 +116,7 @@
"pluginVersion": "8.3.5",
"targets": [
{
- "expr": "1 - (max by (instance) (irate(node_cpu_seconds_total{mode=\"idle\"}[5m])))",
+ "expr": "1 - (min by (instance) (irate(node_cpu_seconds_total{mode=\"idle\"}[5m])))",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{instance}}",
@@ -155,7 +155,7 @@
}
],
"executionErrorState": "alerting",
- "for": "5m",
+ "for": "2h",
"frequency": "1m",
"handler": 1,
"name": "15 min load average alert",
diff --git a/nixos/modules/monitoring/server/grafana-service.nix b/nixos/modules/monitoring/server/grafana-service.nix
deleted file mode 100644
index fd4055ee396ab4ea450a102402782045920bc851..0000000000000000000000000000000000000000
--- a/nixos/modules/monitoring/server/grafana-service.nix
+++ /dev/null
@@ -1,739 +0,0 @@
-# This is the NixOS 21.11 Grafana service definition module
-# with the backported UID setting for data sources, so that
-# we can have the same dashboards in our dev, test and prod
-# environments.
-#
-# The change from nixpkgs 81291cc793cf88bd6eff3fd8512e5eb9d037066c
-# will land in NixOS 22.11.
-#
-# When upgrading PrivateStorageio to 22.05, this file will
-# need an upgrade too. When upgrading PrivateStorageio to
-# 22.11, it can be removed.
-
-{ options, config, lib, pkgs, ... }:
-
-with lib;
-
-let
- cfg = config.services.grafana;
- opt = options.services.grafana;
- declarativePlugins = pkgs.linkFarm "grafana-plugins" (builtins.map (pkg: { name = pkg.pname; path = pkg; }) cfg.declarativePlugins);
- useMysql = cfg.database.type == "mysql";
- usePostgresql = cfg.database.type == "postgres";
-
- envOptions = {
- PATHS_DATA = cfg.dataDir;
- PATHS_PLUGINS = if builtins.isNull cfg.declarativePlugins then "${cfg.dataDir}/plugins" else declarativePlugins;
- PATHS_LOGS = "${cfg.dataDir}/log";
-
- SERVER_PROTOCOL = cfg.protocol;
- SERVER_HTTP_ADDR = cfg.addr;
- SERVER_HTTP_PORT = cfg.port;
- SERVER_SOCKET = cfg.socket;
- SERVER_DOMAIN = cfg.domain;
- SERVER_ROOT_URL = cfg.rootUrl;
- SERVER_STATIC_ROOT_PATH = cfg.staticRootPath;
- SERVER_CERT_FILE = cfg.certFile;
- SERVER_CERT_KEY = cfg.certKey;
-
- DATABASE_TYPE = cfg.database.type;
- DATABASE_HOST = cfg.database.host;
- DATABASE_NAME = cfg.database.name;
- DATABASE_USER = cfg.database.user;
- DATABASE_PASSWORD = cfg.database.password;
- DATABASE_PATH = cfg.database.path;
- DATABASE_CONN_MAX_LIFETIME = cfg.database.connMaxLifetime;
-
- SECURITY_ADMIN_USER = cfg.security.adminUser;
- SECURITY_ADMIN_PASSWORD = cfg.security.adminPassword;
- SECURITY_SECRET_KEY = cfg.security.secretKey;
-
- USERS_ALLOW_SIGN_UP = boolToString cfg.users.allowSignUp;
- USERS_ALLOW_ORG_CREATE = boolToString cfg.users.allowOrgCreate;
- USERS_AUTO_ASSIGN_ORG = boolToString cfg.users.autoAssignOrg;
- USERS_AUTO_ASSIGN_ORG_ROLE = cfg.users.autoAssignOrgRole;
-
- AUTH_ANONYMOUS_ENABLED = boolToString cfg.auth.anonymous.enable;
- AUTH_ANONYMOUS_ORG_NAME = cfg.auth.anonymous.org_name;
- AUTH_ANONYMOUS_ORG_ROLE = cfg.auth.anonymous.org_role;
- AUTH_GOOGLE_ENABLED = boolToString cfg.auth.google.enable;
- AUTH_GOOGLE_ALLOW_SIGN_UP = boolToString cfg.auth.google.allowSignUp;
- AUTH_GOOGLE_CLIENT_ID = cfg.auth.google.clientId;
-
- ANALYTICS_REPORTING_ENABLED = boolToString cfg.analytics.reporting.enable;
-
- SMTP_ENABLED = boolToString cfg.smtp.enable;
- SMTP_HOST = cfg.smtp.host;
- SMTP_USER = cfg.smtp.user;
- SMTP_PASSWORD = cfg.smtp.password;
- SMTP_FROM_ADDRESS = cfg.smtp.fromAddress;
- } // cfg.extraOptions;
-
- datasourceConfiguration = {
- apiVersion = 1;
- datasources = cfg.provision.datasources;
- };
-
- datasourceFile = pkgs.writeText "datasource.yaml" (builtins.toJSON datasourceConfiguration);
-
- dashboardConfiguration = {
- apiVersion = 1;
- providers = cfg.provision.dashboards;
- };
-
- dashboardFile = pkgs.writeText "dashboard.yaml" (builtins.toJSON dashboardConfiguration);
-
- notifierConfiguration = {
- apiVersion = 1;
- notifiers = cfg.provision.notifiers;
- };
-
- notifierFile = pkgs.writeText "notifier.yaml" (builtins.toJSON notifierConfiguration);
-
- provisionConfDir = pkgs.runCommand "grafana-provisioning" { } ''
- mkdir -p $out/{datasources,dashboards,notifiers}
- ln -sf ${datasourceFile} $out/datasources/datasource.yaml
- ln -sf ${dashboardFile} $out/dashboards/dashboard.yaml
- ln -sf ${notifierFile} $out/notifiers/notifier.yaml
- '';
-
- # Get a submodule without any embedded metadata:
- _filter = x: filterAttrs (k: v: k != "_module") x;
-
- # http://docs.grafana.org/administration/provisioning/#datasources
- grafanaTypes.datasourceConfig = types.submodule {
- options = {
- name = mkOption {
- type = types.str;
- description = "Name of the datasource. Required.";
- };
- type = mkOption {
- type = types.str;
- description = "Datasource type. Required.";
- };
- access = mkOption {
- type = types.enum ["proxy" "direct"];
- default = "proxy";
- description = "Access mode. proxy or direct (Server or Browser in the UI). Required.";
- };
- orgId = mkOption {
- type = types.int;
- default = 1;
- description = "Org id. will default to orgId 1 if not specified.";
- };
- uid = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = "Custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically.";
- };
- url = mkOption {
- type = types.str;
- description = "Url of the datasource.";
- };
- password = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = "Database password, if used.";
- };
- user = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = "Database user, if used.";
- };
- database = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = "Database name, if used.";
- };
- basicAuth = mkOption {
- type = types.nullOr types.bool;
- default = null;
- description = "Enable/disable basic auth.";
- };
- basicAuthUser = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = "Basic auth username.";
- };
- basicAuthPassword = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = "Basic auth password.";
- };
- withCredentials = mkOption {
- type = types.bool;
- default = false;
- description = "Enable/disable with credentials headers.";
- };
- isDefault = mkOption {
- type = types.bool;
- default = false;
- description = "Mark as default datasource. Max one per org.";
- };
- jsonData = mkOption {
- type = types.nullOr types.attrs;
- default = null;
- description = "Datasource specific configuration.";
- };
- secureJsonData = mkOption {
- type = types.nullOr types.attrs;
- default = null;
- description = "Datasource specific secure configuration.";
- };
- version = mkOption {
- type = types.int;
- default = 1;
- description = "Version.";
- };
- editable = mkOption {
- type = types.bool;
- default = false;
- description = "Allow users to edit datasources from the UI.";
- };
- };
- };
-
- # http://docs.grafana.org/administration/provisioning/#dashboards
- grafanaTypes.dashboardConfig = types.submodule {
- options = {
- name = mkOption {
- type = types.str;
- default = "default";
- description = "Provider name.";
- };
- orgId = mkOption {
- type = types.int;
- default = 1;
- description = "Organization ID.";
- };
- folder = mkOption {
- type = types.str;
- default = "";
- description = "Add dashboards to the specified folder.";
- };
- type = mkOption {
- type = types.str;
- default = "file";
- description = "Dashboard provider type.";
- };
- disableDeletion = mkOption {
- type = types.bool;
- default = false;
- description = "Disable deletion when JSON file is removed.";
- };
- updateIntervalSeconds = mkOption {
- type = types.int;
- default = 10;
- description = "How often Grafana will scan for changed dashboards.";
- };
- options = {
- path = mkOption {
- type = types.path;
- description = "Path grafana will watch for dashboards.";
- };
- };
- };
- };
-
- grafanaTypes.notifierConfig = types.submodule {
- options = {
- name = mkOption {
- type = types.str;
- default = "default";
- description = "Notifier name.";
- };
- type = mkOption {
- type = types.enum ["dingding" "discord" "email" "googlechat" "hipchat" "kafka" "line" "teams" "opsgenie" "pagerduty" "prometheus-alertmanager" "pushover" "sensu" "sensugo" "slack" "telegram" "threema" "victorops" "webhook"];
- description = "Notifier type.";
- };
- uid = mkOption {
- type = types.str;
- description = "Unique notifier identifier.";
- };
- org_id = mkOption {
- type = types.int;
- default = 1;
- description = "Organization ID.";
- };
- org_name = mkOption {
- type = types.str;
- default = "Main Org.";
- description = "Organization name.";
- };
- is_default = mkOption {
- type = types.bool;
- description = "Is the default notifier.";
- default = false;
- };
- send_reminder = mkOption {
- type = types.bool;
- default = true;
- description = "Should the notifier be sent reminder notifications while alerts continue to fire.";
- };
- frequency = mkOption {
- type = types.str;
- default = "5m";
- description = "How frequently should the notifier be sent reminders.";
- };
- disable_resolve_message = mkOption {
- type = types.bool;
- default = false;
- description = "Turn off the message that sends when an alert returns to OK.";
- };
- settings = mkOption {
- type = types.nullOr types.attrs;
- default = null;
- description = "Settings for the notifier type.";
- };
- secure_settings = mkOption {
- type = types.nullOr types.attrs;
- default = null;
- description = "Secure settings for the notifier type.";
- };
- };
- };
-in {
- options.services.grafana = {
- enable = mkEnableOption "grafana";
-
- protocol = mkOption {
- description = "Which protocol to listen.";
- default = "http";
- type = types.enum ["http" "https" "socket"];
- };
-
- addr = mkOption {
- description = "Listening address.";
- default = "127.0.0.1";
- type = types.str;
- };
-
- port = mkOption {
- description = "Listening port.";
- default = 3000;
- type = types.port;
- };
-
- socket = mkOption {
- description = "Listening socket.";
- default = "/run/grafana/grafana.sock";
- type = types.str;
- };
-
- domain = mkOption {
- description = "The public facing domain name used to access grafana from a browser.";
- default = "localhost";
- type = types.str;
- };
-
- rootUrl = mkOption {
- description = "Full public facing url.";
- default = "%(protocol)s://%(domain)s:%(http_port)s/";
- type = types.str;
- };
-
- certFile = mkOption {
- description = "Cert file for ssl.";
- default = "";
- type = types.str;
- };
-
- certKey = mkOption {
- description = "Cert key for ssl.";
- default = "";
- type = types.str;
- };
-
- staticRootPath = mkOption {
- description = "Root path for static assets.";
- default = "${cfg.package}/share/grafana/public";
- defaultText = literalExpression ''"''${package}/share/grafana/public"'';
- type = types.str;
- };
-
- package = mkOption {
- description = "Package to use.";
- default = pkgs.grafana;
- defaultText = literalExpression "pkgs.grafana";
- type = types.package;
- };
-
- declarativePlugins = mkOption {
- type = with types; nullOr (listOf path);
- default = null;
- description = "If non-null, then a list of packages containing Grafana plugins to install. If set, plugins cannot be manually installed.";
- example = literalExpression "with pkgs.grafanaPlugins; [ grafana-piechart-panel ]";
- # Make sure each plugin is added only once; otherwise building
- # the link farm fails, since the same path is added multiple
- # times.
- apply = x: if isList x then lib.unique x else x;
- };
-
- dataDir = mkOption {
- description = "Data directory.";
- default = "/var/lib/grafana";
- type = types.path;
- };
-
- database = {
- type = mkOption {
- description = "Database type.";
- default = "sqlite3";
- type = types.enum ["mysql" "sqlite3" "postgres"];
- };
-
- host = mkOption {
- description = "Database host.";
- default = "127.0.0.1:3306";
- type = types.str;
- };
-
- name = mkOption {
- description = "Database name.";
- default = "grafana";
- type = types.str;
- };
-
- user = mkOption {
- description = "Database user.";
- default = "root";
- type = types.str;
- };
-
- password = mkOption {
- description = ''
- Database password.
- This option is mutual exclusive with the passwordFile option.
- '';
- default = "";
- type = types.str;
- };
-
- passwordFile = mkOption {
- description = ''
- File that containts the database password.
- This option is mutual exclusive with the password option.
- '';
- default = null;
- type = types.nullOr types.path;
- };
-
- path = mkOption {
- description = "Database path.";
- default = "${cfg.dataDir}/data/grafana.db";
- type = types.path;
- };
-
- connMaxLifetime = mkOption {
- description = ''
- Sets the maximum amount of time (in seconds) a connection may be reused.
- For MySQL this setting should be shorter than the `wait_timeout' variable.
- '';
- default = "unlimited";
- example = 14400;
- type = types.either types.int (types.enum [ "unlimited" ]);
- };
- };
-
- provision = {
- enable = mkEnableOption "provision";
- datasources = mkOption {
- description = "Grafana datasources configuration.";
- default = [];
- type = types.listOf grafanaTypes.datasourceConfig;
- apply = x: map _filter x;
- };
- dashboards = mkOption {
- description = "Grafana dashboard configuration.";
- default = [];
- type = types.listOf grafanaTypes.dashboardConfig;
- apply = x: map _filter x;
- };
- notifiers = mkOption {
- description = "Grafana notifier configuration.";
- default = [];
- type = types.listOf grafanaTypes.notifierConfig;
- apply = x: map _filter x;
- };
- };
-
- security = {
- adminUser = mkOption {
- description = "Default admin username.";
- default = "admin";
- type = types.str;
- };
-
- adminPassword = mkOption {
- description = ''
- Default admin password.
- This option is mutual exclusive with the adminPasswordFile option.
- '';
- default = "admin";
- type = types.str;
- };
-
- adminPasswordFile = mkOption {
- description = ''
- Default admin password.
- This option is mutual exclusive with the <literal>adminPassword</literal> option.
- '';
- default = null;
- type = types.nullOr types.path;
- };
-
- secretKey = mkOption {
- description = "Secret key used for signing.";
- default = "SW2YcwTIb9zpOOhoPsMm";
- type = types.str;
- };
-
- secretKeyFile = mkOption {
- description = "Secret key used for signing.";
- default = null;
- type = types.nullOr types.path;
- };
- };
-
- smtp = {
- enable = mkEnableOption "smtp";
- host = mkOption {
- description = "Host to connect to.";
- default = "localhost:25";
- type = types.str;
- };
- user = mkOption {
- description = "User used for authentication.";
- default = "";
- type = types.str;
- };
- password = mkOption {
- description = ''
- Password used for authentication.
- This option is mutual exclusive with the passwordFile option.
- '';
- default = "";
- type = types.str;
- };
- passwordFile = mkOption {
- description = ''
- Password used for authentication.
- This option is mutual exclusive with the password option.
- '';
- default = null;
- type = types.nullOr types.path;
- };
- fromAddress = mkOption {
- description = "Email address used for sending.";
- default = "admin@grafana.localhost";
- type = types.str;
- };
- };
-
- users = {
- allowSignUp = mkOption {
- description = "Disable user signup / registration.";
- default = false;
- type = types.bool;
- };
-
- allowOrgCreate = mkOption {
- description = "Whether user is allowed to create organizations.";
- default = false;
- type = types.bool;
- };
-
- autoAssignOrg = mkOption {
- description = "Whether to automatically assign new users to default org.";
- default = true;
- type = types.bool;
- };
-
- autoAssignOrgRole = mkOption {
- description = "Default role new users will be auto assigned.";
- default = "Viewer";
- type = types.enum ["Viewer" "Editor"];
- };
- };
-
- auth = {
- anonymous = {
- enable = mkOption {
- description = "Whether to allow anonymous access.";
- default = false;
- type = types.bool;
- };
- org_name = mkOption {
- description = "Which organization to allow anonymous access to.";
- default = "Main Org.";
- type = types.str;
- };
- org_role = mkOption {
- description = "Which role anonymous users have in the organization.";
- default = "Viewer";
- type = types.str;
- };
- };
- google = {
- enable = mkOption {
- description = "Whether to allow Google OAuth2.";
- default = false;
- type = types.bool;
- };
- allowSignUp = mkOption {
- description = "Whether to allow sign up with Google OAuth2.";
- default = false;
- type = types.bool;
- };
- clientId = mkOption {
- description = "Google OAuth2 client ID.";
- default = "";
- type = types.str;
- };
- clientSecretFile = mkOption {
- description = "Google OAuth2 client secret.";
- default = null;
- type = types.nullOr types.path;
- };
- };
- };
-
- analytics.reporting = {
- enable = mkOption {
- description = "Whether to allow anonymous usage reporting to stats.grafana.net.";
- default = true;
- type = types.bool;
- };
- };
-
- extraOptions = mkOption {
- description = ''
- Extra configuration options passed as env variables as specified in
- <link xlink:href="http://docs.grafana.org/installation/configuration/">documentation</link>,
- but without GF_ prefix
- '';
- default = {};
- type = with types; attrsOf (either str path);
- };
- };
-
- config = mkIf cfg.enable {
- warnings = flatten [
- (optional (
- cfg.database.password != opt.database.password.default ||
- cfg.security.adminPassword != opt.security.adminPassword.default
- ) "Grafana passwords will be stored as plaintext in the Nix store!")
- (optional (
- any (x: x.password != null || x.basicAuthPassword != null || x.secureJsonData != null) cfg.provision.datasources
- ) "Datasource passwords will be stored as plaintext in the Nix store!")
- (optional (
- any (x: x.secure_settings != null) cfg.provision.notifiers
- ) "Notifier secure settings will be stored as plaintext in the Nix store!")
- ];
-
- environment.systemPackages = [ cfg.package ];
-
- assertions = [
- {
- assertion = cfg.database.password != opt.database.password.default -> cfg.database.passwordFile == null;
- message = "Cannot set both password and passwordFile";
- }
- {
- assertion = cfg.security.adminPassword != opt.security.adminPassword.default -> cfg.security.adminPasswordFile == null;
- message = "Cannot set both adminPassword and adminPasswordFile";
- }
- {
- assertion = cfg.security.secretKey != opt.security.secretKey.default -> cfg.security.secretKeyFile == null;
- message = "Cannot set both secretKey and secretKeyFile";
- }
- {
- assertion = cfg.smtp.password != opt.smtp.password.default -> cfg.smtp.passwordFile == null;
- message = "Cannot set both password and passwordFile";
- }
- ];
-
- systemd.services.grafana = {
- description = "Grafana Service Daemon";
- wantedBy = ["multi-user.target"];
- after = ["networking.target"] ++ lib.optional usePostgresql "postgresql.service" ++ lib.optional useMysql "mysql.service";
- environment = {
- QT_QPA_PLATFORM = "offscreen";
- } // mapAttrs' (n: v: nameValuePair "GF_${n}" (toString v)) envOptions;
- script = ''
- set -o errexit -o pipefail -o nounset -o errtrace
- shopt -s inherit_errexit
-
- ${optionalString (cfg.auth.google.clientSecretFile != null) ''
- GF_AUTH_GOOGLE_CLIENT_SECRET="$(<${escapeShellArg cfg.auth.google.clientSecretFile})"
- export GF_AUTH_GOOGLE_CLIENT_SECRET
- ''}
- ${optionalString (cfg.database.passwordFile != null) ''
- GF_DATABASE_PASSWORD="$(<${escapeShellArg cfg.database.passwordFile})"
- export GF_DATABASE_PASSWORD
- ''}
- ${optionalString (cfg.security.adminPasswordFile != null) ''
- GF_SECURITY_ADMIN_PASSWORD="$(<${escapeShellArg cfg.security.adminPasswordFile})"
- export GF_SECURITY_ADMIN_PASSWORD
- ''}
- ${optionalString (cfg.security.secretKeyFile != null) ''
- GF_SECURITY_SECRET_KEY="$(<${escapeShellArg cfg.security.secretKeyFile})"
- export GF_SECURITY_SECRET_KEY
- ''}
- ${optionalString (cfg.smtp.passwordFile != null) ''
- GF_SMTP_PASSWORD="$(<${escapeShellArg cfg.smtp.passwordFile})"
- export GF_SMTP_PASSWORD
- ''}
- ${optionalString cfg.provision.enable ''
- export GF_PATHS_PROVISIONING=${provisionConfDir};
- ''}
- exec ${cfg.package}/bin/grafana-server -homepath ${cfg.dataDir}
- '';
- serviceConfig = {
- WorkingDirectory = cfg.dataDir;
- User = "grafana";
- RuntimeDirectory = "grafana";
- RuntimeDirectoryMode = "0755";
- # Hardening
- AmbientCapabilities = lib.mkIf (cfg.port < 1024) [ "CAP_NET_BIND_SERVICE" ];
- CapabilityBoundingSet = if (cfg.port < 1024) then [ "CAP_NET_BIND_SERVICE" ] else [ "" ];
- DeviceAllow = [ "" ];
- LockPersonality = true;
- NoNewPrivileges = true;
- PrivateDevices = true;
- PrivateTmp = true;
- ProtectClock = true;
- ProtectControlGroups = true;
- ProtectHome = true;
- ProtectHostname = true;
- ProtectKernelLogs = true;
- ProtectKernelModules = true;
- ProtectKernelTunables = true;
- ProtectProc = "invisible";
- ProtectSystem = "full";
- RemoveIPC = true;
- RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ];
- RestrictNamespaces = true;
- RestrictRealtime = true;
- RestrictSUIDSGID = true;
- SystemCallArchitectures = "native";
- # Upstream grafana is not setting SystemCallFilter for compatibility
- # reasons, see https://github.com/grafana/grafana/pull/40176
- SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
- UMask = "0027";
- };
- preStart = ''
- ln -fs ${cfg.package}/share/grafana/conf ${cfg.dataDir}
- ln -fs ${cfg.package}/share/grafana/tools ${cfg.dataDir}
- '';
- };
-
- users.users.grafana = {
- uid = config.ids.uids.grafana;
- description = "Grafana user";
- home = cfg.dataDir;
- createHome = true;
- group = "grafana";
- };
- users.groups.grafana = {};
- };
-}
diff --git a/nixos/modules/monitoring/server/grafana.nix b/nixos/modules/monitoring/server/grafana.nix
index ab7c7afc5e61113e05df08f5fa39d63f24da5957..4da836c10975d56438fee3961912da20b4788c7b 100644
--- a/nixos/modules/monitoring/server/grafana.nix
+++ b/nixos/modules/monitoring/server/grafana.nix
@@ -7,24 +7,9 @@
let
cfg = config.services.private-storage.monitoring.grafana;
- grafanaAuth = if (cfg.googleOAuthClientID == "") then {
- anonymous.enable = true;
- } else {
- google.enable = true;
- # Grafana considers it "sign up" to let in a user it has
- # never seen before.
- google.allowSignUp = true;
- google.clientSecretFile = cfg.googleOAuthClientSecretFile;
- google.clientId = cfg.googleOAuthClientID;
- };
in {
- # Override Grafana module so we can specify datasource UIDs
- # Copied from https://nixos.org/manual/nixos/stable/#sec-replace-modules
- disabledModules = [ "services/monitoring/grafana.nix" ];
- imports = [ ./grafana-service.nix ];
-
options.services.private-storage.monitoring.grafana = {
domains = lib.mkOption
{ type = lib.types.listOf lib.types.str;
@@ -83,6 +68,21 @@ in {
Where to find the file that containts the slack URL.
'';
};
+ enableZulipAlert = lib.mkOption
+ { type = lib.types.bool;
+ default = false;
+ description = ''
+ Enables the Zulip alerter. Expects a file that contains
+ the secret Zulip Web Hook URL in grafanaZulipUrlFile (see below).
+ '';
+ };
+ grafanaZulipUrlFile = lib.mkOption
+ { type = lib.types.path;
+ default = /run/keys/grafana-zulip-url;
+ description = ''
+ Where to find the file that containts the Zulip URL.
+ '';
+ };
};
config =
@@ -96,41 +96,59 @@ in {
services.grafana = {
enable = true;
- inherit domain;
- port = 2342;
- addr = "127.0.0.1";
-
- # No phoning home
- analytics.reporting.enable = false;
-
- # Force Grafana to believe it is reachable via https on the default port
- # number because that's where the nginx that forwards traffic to it is
- # listening. Grafana's own server listens on an internal address that
- # doesn't matter to anyone except our nginx instance.
- rootUrl = "https://%(domain)s/";
-
- extraOptions = {
- # Defend against DNS rebinding attacks.
- SERVER_ENFORCE_DOMAIN = "true";
- # Same time zone for all users by default
- DATE_FORMATS_DEFAULT_TIMEZONE = "UTC";
- };
- auth = {
- anonymous.org_role = "Admin";
- anonymous.org_name = "Main Org.";
- } // grafanaAuth;
+ settings = {
+
+ server = {
+ domain = "${toString domain}";
+ http_port = 2342;
+ http_addr = "127.0.0.1";
- # Give users that come through GSuite SSO the highest possible privileges:
- users.autoAssignOrgRole = "Editor";
+ # Defend against DNS rebinding attacks.
+ enforce_domain = true;
- # Read the admin password from a file in our secrets folder:
- security.adminPasswordFile = cfg.adminPasswordFile;
+ # Force Grafana to believe it is reachable via https on the default port
+ # number because that's where the nginx that forwards traffic to it is
+ # listening. Grafana's own server listens on an internal address that
+ # doesn't matter to anyone except our nginx instance.
+ root_url = "https://%(domain)s/";
+ };
+
+ # No phoning home
+ analytics.reporting_enabled = false;
+
+ # Same time zone for all users by default
+ date_formats.default_timezone = "UTC";
+
+ # The auth sections since NixOS 22.11 are named a bit funky with a dot in the name
+ #
+ # https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/grafana/#anonymous-authentication
+ # https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/google/
+ "auth.anonymous" = lib.mkIf (cfg.googleOAuthClientID == "") {
+ enabled = true;
+ org_role = "Admin";
+ org_name = "Main Org.";
+ };
+ "auth.google" = lib.mkIf (cfg.googleOAuthClientID != "") {
+ enabled = true;
+ # Grafana considers it "sign up" to let in a user it has
+ # never seen before.
+ allow_sign_up = true;
+ client_secret = "$__file{${toString cfg.googleOAuthClientSecretFile}}";
+ client_id = cfg.googleOAuthClientID;
+ };
+
+ # Give users that come through GSuite SSO the highest possible privileges:
+ users.auto_assign_org_role = "Editor";
+
+ # Read the admin password from a file in our secrets folder:
+ security.admin_password = "$__file{${toString cfg.adminPasswordFile}}";
+ };
provision = {
enable = true;
# See https://grafana.com/docs/grafana/latest/administration/provisioning/#datasources
- datasources = [{
+ datasources.settings.datasources = [{
name = "Prometheus";
type = "prometheus";
uid = "LocalPrometheus";
@@ -145,7 +163,7 @@ in {
url = cfg.lokiUrl;
}];
# See https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards
- dashboards = [{
+ dashboards.settings.providers = [{
name = "provisioned";
options.path = ./grafana-dashboards;
}];
@@ -165,12 +183,22 @@ in {
# See https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider
url = "$__file{${toString cfg.grafanaSlackUrlFile}}";
};
+ }]) ++ (lib.optionals (cfg.enableZulipAlert) [{
+ # See https://zulip.com/integrations/doc/grafana
+ uid = "zulip-notifier-1";
+ name = "Zulip";
+ type = "webhook";
+ is_default = true;
+ send_reminder = false;
+ settings = {
+ url = "$__file{${toString cfg.grafanaZulipUrlFile}}";
+ };
}]);
};
};
# nginx reverse proxy
- security.acme.email = cfg.letsEncryptAdminEmail;
+ security.acme.defaults.email = cfg.letsEncryptAdminEmail;
security.acme.acceptTerms = true;
services.nginx = {
enable = true;
@@ -188,7 +216,7 @@ in {
enableACME = true;
forceSSL = true;
locations."/" = {
- proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+ proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
proxyWebsockets = true;
};
locations."/metrics" = {
@@ -200,7 +228,7 @@ in {
allow ::1;
deny all;
'';
- proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+ proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
};
};
};
diff --git a/nixos/modules/private-storage.nix b/nixos/modules/private-storage.nix
index 3a716cf05eeb8a5bd5b2aafb6f6c2b3aa54da894..f9521e44455bac9b751b7cd55ff748c1d3704e5a 100644
--- a/nixos/modules/private-storage.nix
+++ b/nixos/modules/private-storage.nix
@@ -67,6 +67,14 @@ in
The port number on which to service storage clients.
'';
};
+ services.private-storage.publicReadOnlyStoragePort = lib.mkOption
+ { default = 8899;
+ type = lib.types.int;
+ example = 8099;
+ description = ''
+ The port number on which to service read-only storage clients.
+ '';
+ };
services.private-storage.issuerRootURL = lib.mkOption
{ default = "https://issuer.${config.networking.domain}/";
type = lib.types.str;
@@ -96,68 +104,96 @@ in
# Define configuration based on values given for our options - starting with
# the option that says whether this is even turned on.
config = lib.mkIf cfg.enable
- { services.tahoe.nodes."${storage-node-name}" =
- { package = cfg.tahoe.package;
- # Each attribute in this set corresponds to a section in the tahoe.cfg
- # file. Attributes on those sets correspond to individual assignments
- # in those sections.
- #
- # We just populate this according to policy/preference of Private
- # Storage.
- sections =
- { client = if cfg.introducerFURL == null then {} else
- { "introducer.furl" = cfg.introducerFURL;
+ {
+ # A read-only storage service. This allows read-only access for clients
+ # that use Great Black Swamp. There is no ZKAP/GBS integration yet so
+ # this is the most we can do at the moment.
+ services.tahoe.nodes."ro-${storage-node-name}" =
+ { package = cfg.tahoe.package;
+ sections =
+ { client = if cfg.introducerFURL == null then {} else
+ { "introducer.furl" = cfg.introducerFURL;
+ };
+ node =
+ { nickname = "ro-${storage-node-name}";
+ "tub.port" = "tcp:${toString cfg.publicReadOnlyStoragePort}";
+ "tub.location" = "tcp:${cfg.publicAddress}:${toString cfg.publicReadOnlyStoragePort}";
+ };
+ storage =
+ { enabled = true;
+ storage_dir = "/storage";
+ readonly = true;
+ force_foolscap = false;
+ };
+ };
};
- node =
- # XXX Should try to name that is unique across the grid.
- { nickname = "${storage-node-name}";
+ # Tahoe nixos module brings along a single socket for the web api.
+ # That's for the other storage node though. Turn off the integration
+ # with this one.
+ systemd.services."tahoe.ro-storage".unitConfig.Requires = [];
- # We have the web port active because the CLI uses it and because it
- # exposes a metrics endpoint for our monitoring system. The actual
- # port configuration lives in systemd so that it can order binding
- # the socket correctly with other dependencies (which we can't
- # reliably do with Tahoe without a bunch of other work).
- "web.port" = "systemd:domain=INET:index=0";
+ services.tahoe.nodes."${storage-node-name}" =
+ { package = cfg.tahoe.package;
+ # Each attribute in this set corresponds to a section in the
+ # tahoe.cfg file. Attributes on those sets correspond to individual
+ # assignments in those sections.
+ #
+ # We just populate this according to policy/preference of Private
+ # Storage.
+ sections =
+ { client = if cfg.introducerFURL == null then {} else
+ { "introducer.furl" = cfg.introducerFURL;
+ };
+ node =
+ # XXX Should try to name that is unique across the grid.
+ { nickname = "${storage-node-name}";
- # We have to tell Tahoe-LAFS where to listen for Foolscap
- # connections for the storage protocol. We have to tell it twice.
- # First, in the syntax which it uses to listen.
- "tub.port" = "tcp:${toString cfg.publicStoragePort}";
+ # We have the web port active because the CLI uses it and
+ # because it exposes a metrics endpoint for our monitoring
+ # system. The actual port configuration lives in systemd so
+ # that it can order binding the socket correctly with other
+ # dependencies (which we can't reliably do with Tahoe
+ # without a bunch of other work).
+ "web.port" = "systemd:domain=INET:index=0";
- # Second, in the syntax it advertises to in the fURL.
- "tub.location" = "tcp:${cfg.publicAddress}:${toString cfg.publicStoragePort}";
- };
- storage =
- { enabled = true;
- # Put the storage where we have a lot of space configured.
- storage_dir = "/storage";
- # Turn on our plugin.
- plugins = "privatestorageio-zkapauthz-v1";
- };
- "storageserver.plugins.privatestorageio-zkapauthz-v1" =
- { "ristretto-issuer-root-url" = cfg.issuerRootURL;
- "ristretto-signing-key-path" = cfg.ristrettoSigningKeyPath;
- } // (
- if cfg.passValue == null
- then {}
- else { "pass-value" = (toString cfg.passValue); }
- );
- };
- };
+ # We have to tell Tahoe-LAFS where to listen for Foolscap
+ # connections for the storage protocol. We have to tell it twice.
+ # First, in the syntax which it uses to listen.
+ "tub.port" = "tcp:${toString cfg.publicStoragePort}";
- # Let traffic destined for the storage node's Foolscap server through.
- networking.firewall.allowedTCPPorts = [ cfg.publicStoragePort ];
+ # Second, in the syntax it advertises to in the fURL.
+ "tub.location" = "tcp:${cfg.publicAddress}:${toString cfg.publicStoragePort}";
+ };
+ storage =
+ { enabled = true;
+ # Put the storage where we have a lot of space configured.
+ storage_dir = "/storage";
+ # Turn on our plugin.
+ plugins = "privatestorageio-zkapauthz-v2";
+ };
+ "storageserver.plugins.privatestorageio-zkapauthz-v2" =
+ { "ristretto-issuer-root-url" = cfg.issuerRootURL;
+ "ristretto-signing-key-path" = cfg.ristrettoSigningKeyPath;
+ } // (
+ if cfg.passValue == null
+ then {}
+ else { "pass-value" = (toString cfg.passValue); }
+ );
+ };
+ };
- systemd.tmpfiles.rules =
- # Add a rule to prevent incident reports from accumulating indefinitely.
- # See tmpfiles.d(5) for the syntax.
- [ "d ${incidents-dir} 0755 root root ${max-incident-age} -"
- ];
+ # Let traffic destined for the storage node's Foolscap server through.
+ networking.firewall.allowedTCPPorts = [ cfg.publicStoragePort cfg.publicReadOnlyStoragePort ];
- environment.systemPackages = [
- # Provide a useful tool for reporting about shares.
- ourpkgs.leasereport
- ];
+ systemd.tmpfiles.rules =
+ # Add a rule to prevent incident reports from accumulating indefinitely.
+ # See tmpfiles.d(5) for the syntax.
+ [ "d ${incidents-dir} 0755 root root ${max-incident-age} -"
+ ];
- };
+ environment.systemPackages = [
+ # Provide a useful tool for reporting about shares.
+ ourpkgs.leasereport
+ ];
+ };
}
diff --git a/nixos/modules/ssh.nix b/nixos/modules/ssh.nix
index 8d5d5766ae3b30c4801b6ce200fa58c1460f6ca7..90fd34b002c607965038a574334fb0fc370d146c 100644
--- a/nixos/modules/ssh.nix
+++ b/nixos/modules/ssh.nix
@@ -25,11 +25,8 @@
services.openssh = {
enable = true;
- # We don't use SFTP for anything. No reason to expose it.
- allowSFTP = false;
-
# We only allow key-based authentication.
- challengeResponseAuthentication = false;
+ kbdInteractiveAuthentication = false;
passwordAuthentication = false;
extraConfig = ''
diff --git a/nixos/modules/tahoe.nix b/nixos/modules/tahoe.nix
index 51c8695420fc6e0b1b9bf2cc92fe0e6e0128ca6d..b53435080d104bccfed5e7e3004f7891f14159bf 100644
--- a/nixos/modules/tahoe.nix
+++ b/nixos/modules/tahoe.nix
@@ -221,10 +221,12 @@ in
# arguments to $(tahoe run). The node directory must come first,
# and arguments which alter Twisted's behavior come afterwards.
ExecStart = ''
- ${settings.package}/bin/tahoe --eliot-destination ${eliotLog} run ${nodedir} -n -l- --pidfile=${pidfile}
+ ${settings.package}/bin/tahoe --eliot-destination ${eliotLog} run --allow-stdin-close ${nodedir} -n -l- --pidfile=${pidfile}
'';
+
# Twisted wants non-blocking sockets:
NonBlocking = true;
+
# The rlimit on number of open files controls how many
# connections a particular storage server can accept (factoring
# in the number of non-connection files the server needs open -
diff --git a/nixos/modules/update-deployment b/nixos/modules/update-deployment
index a0d233a63595a9f838f48243b14ef98fa79a240d..cd41a2363c699c7e551a33d73f1eb83996c1ca85 100755
--- a/nixos/modules/update-deployment
+++ b/nixos/modules/update-deployment
@@ -79,7 +79,11 @@ ssh -o StrictHostKeyChecking=no "$(hostname).$(domainname)" ":"
#
# So instead, import our nixpkgs which forces it to be instantiated in the
# store, then ask for its path, then set NIX_PATH to that.
-export NIX_PATH="nixpkgs=$(nix eval "(import ${CHECKOUT}/nixpkgs.nix { }).path")"
+
+# Two lines since 'export' masks 'set -e'
+# See https://mywiki.wooledge.org/BashFAQ/105#line-204
+NIX_PATH="nixpkgs=$(nix --extra-experimental-features nix-command eval --impure --expr "(import ${CHECKOUT}/nixpkgs.nix { }).path")"
+export NIX_PATH
# Attempt to update just this host. Choose the morph grid definition matching
# the grid we belong to and limit the morph deployment update to the host
diff --git a/nixos/pkgs/privatestorage/default.nix b/nixos/pkgs/privatestorage/default.nix
index 3bbbd3dbcf0b974e6e1997e20773cddbd9ea59c0..f2c7ddea1a2eb35259ca5dbbff3ccd86bdac3a04 100644
--- a/nixos/pkgs/privatestorage/default.nix
+++ b/nixos/pkgs/privatestorage/default.nix
@@ -2,7 +2,9 @@
let
repo-data = lib.importJSON ./repo.json;
repo = fetchFromGitHub (builtins.removeAttrs repo-data [ "branch" ]);
- privatestorage = callPackage repo { python = "python39"; };
+ zk = import repo;
+ # XXX package version choice here
+ zkapauthorizer = zk.outputs.packages.x86_64-linux.zkapauthorizer-python39-tahoe_dev;
+ python = zkapauthorizer.passthru.python;
in
- privatestorage.privatestorage
-
+ python.withPackages (ps: [ zkapauthorizer ] )
diff --git a/nixos/pkgs/privatestorage/repo.json b/nixos/pkgs/privatestorage/repo.json
index 4113e150a72c907dce4ee0d7345361eadb248041..a0aab4861c244c1b6663453c15a3e6a31f99ba0d 100644
--- a/nixos/pkgs/privatestorage/repo.json
+++ b/nixos/pkgs/privatestorage/repo.json
@@ -2,7 +2,7 @@
"owner": "PrivateStorageio",
"branch": "main",
"repo": "ZKAPAuthorizer",
- "rev": "744a063ab76a677b259aa9022711113ffbab2545",
+ "rev": "fb89e91a6c7f595cd0b1c7aa7055cbd32c482180",
"outputHashAlgo": "sha512",
- "outputHash": "293j4469iy69d2hz3gwxwyj0flqb1cncl938s5w5jmfgbvkm1w0yfg1y06nx89zis1rvwqpcly3vxp94pz1dx28d74wiianqks11p54"
+ "outputHash": "3f44znykq8f7mcgdwdyhgf2dvnx7yydmlrjcr17mxfwya4jqmx8zb59mxkxvar0ahn639y2nq3bcqxdyipljfxilfi1cz21li908kkw"
}
\ No newline at end of file
diff --git a/nixos/system-tests.nix b/nixos/system-tests.nix
index 819b5c738eca08b95d3c85b14088a2bf6c000dbf..eafd712cc141bd3ba7c9f6f824d06cb950745ec0 100644
--- a/nixos/system-tests.nix
+++ b/nixos/system-tests.nix
@@ -6,6 +6,13 @@ let
pkgs' = pkgs.extend (self: super: { ourpkgs = self.callPackage ./pkgs {}; });
in {
private-storage = pkgs'.nixosTest ./tests/private-storage.nix;
- spending = pkgs'.nixosTest ./tests/spending.nix;
+
+ # The spending service is not deployed so it doesn't seem *necessary* to run
+ # its test suite here. The packaging still uses mach-nix which is
+ # incompatible with NixOS 22.11 so we can't actually load the ZKAP spending
+ # service derivation anymore. So ... disable the test suite.
+ #
+ # spending = pkgs'.nixosTest ./tests/spending.nix;
+
tahoe = pkgs'.nixosTest ./tests/tahoe.nix;
}
diff --git a/nixos/tests/exercise-storage.py b/nixos/tests/exercise-storage.py
index e3a1d4d2ec7674042487cc0c6dabc670fcd6561d..a4e177b5aa9db7372a41214d2ab4afeef1d23c13 100755
--- a/nixos/tests/exercise-storage.py
+++ b/nixos/tests/exercise-storage.py
@@ -47,7 +47,12 @@ def block_until_connected(api_root):
in servers
if server["connection_status"].startswith("Connected to ")
)
- if len(connected) >= 1:
+ # There is a read-only server and a read-write server! The easiest
+ # way to be sure we've connected to the read-write server is to wait
+ # until we're connected to both. Also, if we manage to connect to two
+ # servers this gives us some confidence that both the read-only and
+ # read-write servers are running.
+ if len(connected) >= 2:
print(
"Connected to a server:\n"
"\t{nodeid}\n"
@@ -85,10 +90,13 @@ def get_api_root(path):
return hyperlink.URL.from_text(f.read().strip())
def tahoe_put(api_root, data, **kwargs):
+ uri = api_root.child(u"uri").to_uri()
response = requests.put(
- api_root.child(u"uri").to_uri(),
+ uri,
BytesIO(data),
+ headers={"accept": "text/plain"},
)
+ print(f"PUT {uri} responded:\n{response.text}\n")
response.raise_for_status()
return response.text
diff --git a/nixos/tests/get-passes.py b/nixos/tests/get-passes.py
index 206e8900e496f7b08967fb11715043fedeaa3f5d..6f9263345521fa9e3977015231a06060f83bd912 100755
--- a/nixos/tests/get-passes.py
+++ b/nixos/tests/get-passes.py
@@ -29,7 +29,7 @@ def main():
if issuerAPIRoot is not None and not issuerAPIRoot.endswith("/"):
issuerAPIRoot += "/"
- zkapauthz = clientAPIRoot + "storage-plugins/privatestorageio-zkapauthz-v1"
+ zkapauthz = clientAPIRoot + "storage-plugins/privatestorageio-zkapauthz-v2"
with open(clientAPITokenPath) as p:
clientAPIToken = p.read().strip()
diff --git a/nixos/tests/private-storage.nix b/nixos/tests/private-storage.nix
index b593a18ef84947bdfe299e1ec6388987f4296491..d7a5aaafe9908b0f3a36c4cdfb041e371eef6bfe 100644
--- a/nixos/tests/private-storage.nix
+++ b/nixos/tests/private-storage.nix
@@ -1,4 +1,4 @@
-{ pkgs }:
+{ pkgs, ... }:
let
ourpkgs = pkgs.callPackage ../pkgs { };
@@ -66,6 +66,8 @@ let
networking.dhcpcd.enable = false;
};
in {
+ name = "private-storage";
+
# https://nixos.org/nixos/manual/index.html#sec-nixos-tests
# https://nixos.mayflower.consulting/blog/2019/07/11/leveraging-nixos-tests-in-your-project/
nodes = rec {
diff --git a/nixos/tests/run-client.py b/nixos/tests/run-client.py
index 8d3d82720ec94b4b91e6af8791aadc58cd7ce2ad..403e47977ca675358dcdab6d0296ba006903b78c 100755
--- a/nixos/tests/run-client.py
+++ b/nixos/tests/run-client.py
@@ -29,12 +29,12 @@ def main():
with open("/tmp/client/tahoe.cfg") as cfg:
config.read_file(cfg)
- config.set(u"client", u"storage.plugins", u"privatestorageio-zkapauthz-v1")
- config.add_section(u"storageclient.plugins.privatestorageio-zkapauthz-v1")
- config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v1", u"redeemer", u"ristretto")
- config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v1", u"ristretto-issuer-root-url", issuerURL)
- config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v1", u"allowed-public-keys", publicKey)
- config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v1", u"default-token-count", tokenCount)
+ config.set(u"client", u"storage.plugins", u"privatestorageio-zkapauthz-v2")
+ config.add_section(u"storageclient.plugins.privatestorageio-zkapauthz-v2")
+ config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v2", u"redeemer", u"ristretto")
+ config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v2", u"ristretto-issuer-root-url", issuerURL)
+ config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v2", u"allowed-public-keys", publicKey)
+ config.set(u"storageclient.plugins.privatestorageio-zkapauthz-v2", u"default-token-count", tokenCount)
with open("/tmp/client/tahoe.cfg", "wt") as cfg:
config.write(cfg)
@@ -43,7 +43,7 @@ def main():
"daemonize",
"-o", "/tmp/stdout",
"-e", "/tmp/stderr",
- which("tahoe"), "run", "/tmp/client",
+ which("tahoe"), "run", "--allow-stdin-close", "/tmp/client",
])
def run(argv):
diff --git a/nixos/tests/run-introducer.py b/nixos/tests/run-introducer.py
index 33c3ec10369477e39c1461b3e59149e015f03ce9..9062c43243f3a5a672ae41d53ace636d7698843a 100755
--- a/nixos/tests/run-introducer.py
+++ b/nixos/tests/run-introducer.py
@@ -31,11 +31,11 @@ def main():
"daemonize",
"-o", "/tmp/stdout",
"-e", "/tmp/stderr",
- which("tahoe"), "run", "/tmp/introducer",
+ which("tahoe"), "run", "--allow-stdin-close", "/tmp/introducer",
])
retry(
- "waiting for open introducer port",
+ f"connect to introducer (port {introducerPort})",
lambda: checkOpen(int(introducerPort)),
)
diff --git a/nixos/tests/tahoe.nix b/nixos/tests/tahoe.nix
index a007e65efd2d6bee8ab4adba9df3cb2901f53526..5b3c5c3827d11798f3656a4912de7d648883e624 100644
--- a/nixos/tests/tahoe.nix
+++ b/nixos/tests/tahoe.nix
@@ -1,8 +1,9 @@
-{ pkgs }:
+{ pkgs, ... }:
let
ourpkgs = pkgs.callPackage ../pkgs { };
in
{
+ name = "tahoe";
nodes = {
storage = { config, pkgs, ourpkgs, ... }: {
imports = [
diff --git a/nixos/tests/test_privatestorage.py b/nixos/tests/test_privatestorage.py
index 0429ac9257f7b587f4efd1d1ecc53375ed9ee4d3..724b99c3667074bc97af13ebe5de71dd58405f24 100644
--- a/nixos/tests/test_privatestorage.py
+++ b/nixos/tests/test_privatestorage.py
@@ -274,7 +274,7 @@ def test(
# It should be possible to restart the storage service without the
# storage node fURL changing.
- furlfile = '/var/db/tahoe-lafs/storage/private/storage-plugin.privatestorageio-zkapauthz-v1.furl'
+ furlfile = '/var/db/tahoe-lafs/storage/private/storage-plugin.privatestorageio-zkapauthz-v2.furl'
before = storage.execute('cat ' + furlfile)
runOnNode(storage, [["systemctl", "restart", "tahoe.storage"]])
after = storage.execute('cat ' + furlfile)
diff --git a/nixpkgs.json b/nixpkgs.json
index 87445ee44c04baa66c0973d665194fa494481973..cb1a73e104de18dddc11467dd4740ea85d009de1 100644
--- a/nixpkgs.json
+++ b/nixpkgs.json
@@ -1,5 +1,5 @@
{
"name": "source",
- "url": "https://releases.nixos.org/nixos/21.11/nixos-21.11.337975.eabc3821918/nixexprs.tar.xz",
- "sha256": "1fq3zz7qfavksdbqvicns7hg61q3hhbxs2ibm818gy629hwkvsvm"
+ "url": "https://releases.nixos.org/nixos/22.11/nixos-22.11.2523.1b82144edfc/nixexprs.tar.xz",
+ "sha256": "12i6lznqz2wxwnjzm1ak5j15dvm2jny1c8998720nnf0qds7180r"
}
\ No newline at end of file
diff --git a/tools/update-nixpkgs b/tools/update-nixpkgs
index 12752892fab880582857d43c5cc4632a41c96d0f..58dedd9c0894b074f41bfa7b2a1b7c3ec1812616 100755
--- a/tools/update-nixpkgs
+++ b/tools/update-nixpkgs
@@ -10,7 +10,7 @@ from ps_tools import get_url_hash
# We pass this to builtins.fetchTarball which only supports sha256
HASH_TYPE = "sha256"
-DEFAULT_CHANNEL = "nixos-21.11"
+DEFAULT_CHANNEL = "nixos-22.11"
CHANNEL_URL_TEMPLATE = "https://channels.nixos.org/{channel}/nixexprs.tar.xz"
@@ -24,9 +24,8 @@ def get_nixos_channel_url(*, channel):
the release.
"""
response = httpx.head(
- CHANNEL_URL_TEMPLATE.format(channel=channel), allow_redirects=False
+ CHANNEL_URL_TEMPLATE.format(channel=channel), follow_redirects=False
)
- response.raise_for_status()
assert response.is_redirect
return str(response.next_request.url)