diff --git a/docs/ops/monitoring-architecture.drawio b/docs/ops/monitoring-architecture.drawio new file mode 100644 index 0000000000000000000000000000000000000000..a9788eeeffdf745a842d28e2cf41b63eb164252b --- /dev/null +++ b/docs/ops/monitoring-architecture.drawio @@ -0,0 +1,127 @@ +<mxfile host="app.diagrams.net" modified="2023-04-20T20:17:44.466Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" etag="8PyLTVr0G94q4Dna4Dsz" version="21.2.1" type="device"> + <diagram name="Page-1" id="aaaa8250-4180-3840-79b5-4cada1eebb92"> + <mxGraphModel dx="794" dy="476" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1920" pageHeight="1200" background="#ffffff" math="0" shadow="0"> + <root> + <mxCell id="0" /> + <mxCell id="1" parent="0" /> + <mxCell id="vhdg0YFc32S7_3H95Ew1-1" value="<span>Management VPN<br>(Wireshark, TINC...)</span>" style="ellipse;shape=cloud;whiteSpace=wrap;html=1;" parent="1" vertex="1"> + <mxGeometry x="780" y="720" width="450" height="110" as="geometry" /> + </mxCell> + <mxCell id="2mYkRctJDop23S32jJdh-2" value="" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1"> + <mxGeometry x="840" y="405.46" width="370" height="304.54" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-3" value="Loki" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.application2;fillColor=#86E83A;strokeColor=#B0F373;aspect=fixed;" parent="1" vertex="1"> + <mxGeometry x="1116" y="592.9000000000001" width="62" height="53" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-4" value="Prometheus" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.application;fillColor=#4286c5;strokeColor=#57A2D8;aspect=fixed;" parent="1" vertex="1"> + <mxGeometry x="866" y="585" width="62" height="68.8" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-5" value="Grafana" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.ami2;aspect=fixed;fillColor=#FF9900;strokeColor=#ffffff;" parent="1" vertex="1"> + <mxGeometry x="996" y="425" width="74" height="50" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-6" value="Node 1" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;" parent="1" vertex="1"> + <mxGeometry x="779" y="845" width="74" height="50" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-7" value="Operator" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.end_user;strokeColor=#9673a6;fillColor=#e1d5e7;aspect=fixed;" parent="1" vertex="1"> + <mxGeometry x="1276" y="305" width="49" height="100.46" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-16" value="Node ..." style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;" parent="1" vertex="1"> + <mxGeometry x="902" y="845" width="74" height="50" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-17" value="Node ..." style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;" parent="1" vertex="1"> + <mxGeometry x="1025" y="845" width="74" height="50" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-18" value="Node N" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;" parent="1" vertex="1"> + <mxGeometry x="1147.5" y="845" width="74" height="50" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-45" value="" style="endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;" parent="1" edge="1"> + <mxGeometry x="806" y="695" width="50" height="50" as="geometry"> + <mxPoint x="894.6666666666666" y="695" as="sourcePoint" /> + <mxPoint x="936" y="835" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-46" value="" style="endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;" parent="1" edge="1"> + <mxGeometry x="806" y="695" width="50" height="50" as="geometry"> + <mxPoint x="894.6666666666666" y="695" as="sourcePoint" /> + <mxPoint x="1046" y="835" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-47" value="" style="endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;" parent="1" edge="1"> + <mxGeometry x="806" y="695" width="50" height="50" as="geometry"> + <mxPoint x="894.6666666666666" y="695" as="sourcePoint" /> + <mxPoint x="1166" y="835" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-50" value="" style="endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;" parent="1" edge="1"> + <mxGeometry x="818.6666666666667" y="695" width="63.33333333333333" height="75" as="geometry"> + <mxPoint x="846" y="835" as="sourcePoint" /> + <mxPoint x="1148" y="695" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-51" value="" style="endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;" parent="1" edge="1"> + <mxGeometry x="818.6666666666667" y="695" width="63.33333333333333" height="75" as="geometry"> + <mxPoint x="946" y="835" as="sourcePoint" /> + <mxPoint x="1148" y="695" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-52" value="" style="endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;" parent="1" edge="1"> + <mxGeometry x="818.6666666666667" y="695" width="63.33333333333333" height="75" as="geometry"> + <mxPoint x="1056" y="835" as="sourcePoint" /> + <mxPoint x="1148" y="695" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-53" value="" style="endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;" parent="1" edge="1"> + <mxGeometry x="818.6666666666667" y="695" width="63.33333333333333" height="75" as="geometry"> + <mxPoint x="1186" y="835" as="sourcePoint" /> + <mxPoint x="1148" y="695" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-57" value="" style="endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;" parent="1" edge="1"> + <mxGeometry width="50" height="50" relative="1" as="geometry"> + <mxPoint x="988" y="495" as="sourcePoint" /> + <mxPoint x="928" y="565" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-58" value="" style="endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;" parent="1" edge="1"> + <mxGeometry width="50" height="50" relative="1" as="geometry"> + <mxPoint x="1078" y="495" as="sourcePoint" /> + <mxPoint x="1136" y="565" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-61" value="View dashboards<br>in browser" style="shape=flexArrow;endArrow=classic;html=1;strokeColor=#9673a6;strokeWidth=1;fillColor=#e1d5e7;spacing=8;" parent="1" edge="1"> + <mxGeometry width="50" height="50" relative="1" as="geometry"> + <mxPoint x="1259.5" y="415" as="sourcePoint" /> + <mxPoint x="1109.5" y="445" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-63" value="<h1>Monitoring architecture&nbsp;</h1><p>Keep it simple, sunshine!<br><br></p><p></p><i>Grafana</i> retrieves metrics from <i>Prometheus</i> and logs from&nbsp;<i>Loki</i>,&nbsp;<span>shows dashboards (web) and does alerting (via eMail? Slack?)</span><br><p><i>Prometheus</i> stores metrics it pulls from various <i>Exporters</i> on nodes</p><p><i>Promtail</i> on nodes pushes logs to <i>Loki<br><br></i></p><p>We try to keep the system as simple as possible: All monitoring and alerting runs on a single machine.</p><h2>Changes</h2><div>v2: Add Github authentication to Grafana. Add management VPN.<br><br>v1: Initial version</div>" style="text;html=1;strokeColor=none;fillColor=none;spacing=5;spacingTop=-20;whiteSpace=wrap;overflow=hidden;rounded=0;shadow=0;comic=0;sketch=0;" parent="1" vertex="1"> + <mxGeometry x="596" y="315" width="164" height="545" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-65" value="Send alerts" style="shape=flexArrow;endArrow=classic;html=1;strokeColor=#9673a6;strokeWidth=1;fillColor=#e1d5e7;" parent="1" edge="1"> + <mxGeometry width="50" height="50" relative="1" as="geometry"> + <mxPoint x="1086" y="405.46000000000004" as="sourcePoint" /> + <mxPoint x="1236" y="375.46000000000004" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="2mYkRctJDop23S32jJdh-3" value="Monitoring server" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1"> + <mxGeometry x="845" y="411" width="120" height="20" as="geometry" /> + </mxCell> + <mxCell id="TrmSFti5pUnXIGkjMKb6-44" value="" style="endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;" parent="1" edge="1"> + <mxGeometry x="806" y="695" width="50" height="50" as="geometry"> + <mxPoint x="894.6666666666666" y="695" as="sourcePoint" /> + <mxPoint x="826" y="835" as="targetPoint" /> + </mxGeometry> + </mxCell> + <mxCell id="vhdg0YFc32S7_3H95Ew1-7" value="GitHub<br>OAuth2" style="ellipse;shape=cloud;whiteSpace=wrap;html=1;" parent="1" vertex="1"> + <mxGeometry x="984" y="305" width="98" height="60" as="geometry" /> + </mxCell> + <mxCell id="vhdg0YFc32S7_3H95Ew1-8" value="" style="endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;" parent="1" edge="1"> + <mxGeometry width="50" height="50" relative="1" as="geometry"> + <mxPoint x="1032.76" y="417" as="sourcePoint" /> + <mxPoint x="1032.76" y="372" as="targetPoint" /> + </mxGeometry> + </mxCell> + </root> + </mxGraphModel> + </diagram> +</mxfile> diff --git a/docs/ops/monitoring-architecture.html b/docs/ops/monitoring-architecture.html new file mode 100644 index 0000000000000000000000000000000000000000..91a6000fb4e8f6791fb26e8a83bc393024e52ca8 --- /dev/null +++ b/docs/ops/monitoring-architecture.html @@ -0,0 +1,12 @@ +<!--[if IE]><meta http-equiv="X-UA-Compatible" content="IE=5,IE=9" ><![endif]--> +<!DOCTYPE html> +<html> +<head> +<title>monitoring-architecture.html</title> +<meta charset="utf-8"/> +</head> +<body> +<div class="mxgraph" style="max-width:100%;border:1px solid transparent;" data-mxgraph="{"highlight":"#0000ff","nav":true,"resize":true,"xml":"<mxfile host=\"app.diagrams.net\" modified=\"2023-04-20T20:19:05.428Z\" agent=\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36\" etag=\"4ETb3iIJGh8a_GDWm07E\" version=\"21.2.1\" type=\"device\"><diagram name=\"Page-1\" id=\"aaaa8250-4180-3840-79b5-4cada1eebb92\"><mxGraphModel dx=\"794\" dy=\"476\" grid=\"1\" gridSize=\"10\" guides=\"1\" tooltips=\"1\" connect=\"1\" arrows=\"1\" fold=\"1\" page=\"1\" pageScale=\"1\" pageWidth=\"1920\" pageHeight=\"1200\" background=\"#ffffff\" math=\"0\" shadow=\"0\"><root><mxCell id=\"0\"/><mxCell id=\"1\" parent=\"0\"/><mxCell id=\"vhdg0YFc32S7_3H95Ew1-1\" value=\"&lt;span&gt;Management VPN&lt;br&gt;(WireGuard)&lt;/span&gt;\" style=\"ellipse;shape=cloud;whiteSpace=wrap;html=1;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"780\" y=\"720\" width=\"450\" height=\"110\" as=\"geometry\"/></mxCell><mxCell id=\"2mYkRctJDop23S32jJdh-2\" value=\"\" style=\"rounded=0;whiteSpace=wrap;html=1;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"840\" y=\"405.46\" width=\"370\" height=\"304.54\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-3\" value=\"Loki\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.application2;fillColor=#86E83A;strokeColor=#B0F373;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"1116\" y=\"592.9000000000001\" width=\"62\" height=\"53\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-4\" value=\"Prometheus\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.application;fillColor=#4286c5;strokeColor=#57A2D8;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"866\" y=\"585\" width=\"62\" height=\"68.8\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-5\" value=\"Grafana\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.ami2;aspect=fixed;fillColor=#FF9900;strokeColor=#ffffff;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"996\" y=\"425\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-6\" value=\"Node 1\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"779\" y=\"845\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-7\" value=\"Operator\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.end_user;strokeColor=#9673a6;fillColor=#e1d5e7;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"1276\" y=\"305\" width=\"49\" height=\"100.46\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-16\" value=\"Node ...\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"902\" y=\"845\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-17\" value=\"Node ...\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"1025\" y=\"845\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-18\" value=\"Node N\" style=\"verticalLabelPosition=bottom;html=1;verticalAlign=top;strokeWidth=1;align=center;outlineConnect=0;dashed=0;outlineConnect=0;shape=mxgraph.aws3d.worker;fillColor=#ECECEC;strokeColor=#5E5E5E;aspect=fixed;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"1147.5\" y=\"845\" width=\"74\" height=\"50\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-45\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"806\" y=\"695\" width=\"50\" height=\"50\" as=\"geometry\"><mxPoint x=\"894.6666666666666\" y=\"695\" as=\"sourcePoint\"/><mxPoint x=\"936\" y=\"835\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-46\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"806\" y=\"695\" width=\"50\" height=\"50\" as=\"geometry\"><mxPoint x=\"894.6666666666666\" y=\"695\" as=\"sourcePoint\"/><mxPoint x=\"1046\" y=\"835\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-47\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"806\" y=\"695\" width=\"50\" height=\"50\" as=\"geometry\"><mxPoint x=\"894.6666666666666\" y=\"695\" as=\"sourcePoint\"/><mxPoint x=\"1166\" y=\"835\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-50\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"818.6666666666667\" y=\"695\" width=\"63.33333333333333\" height=\"75\" as=\"geometry\"><mxPoint x=\"846\" y=\"835\" as=\"sourcePoint\"/><mxPoint x=\"1148\" y=\"695\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-51\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"818.6666666666667\" y=\"695\" width=\"63.33333333333333\" height=\"75\" as=\"geometry\"><mxPoint x=\"946\" y=\"835\" as=\"sourcePoint\"/><mxPoint x=\"1148\" y=\"695\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-52\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"818.6666666666667\" y=\"695\" width=\"63.33333333333333\" height=\"75\" as=\"geometry\"><mxPoint x=\"1056\" y=\"835\" as=\"sourcePoint\"/><mxPoint x=\"1148\" y=\"695\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-53\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#82b366;strokeWidth=1;fillColor=#d5e8d4;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"818.6666666666667\" y=\"695\" width=\"63.33333333333333\" height=\"75\" as=\"geometry\"><mxPoint x=\"1186\" y=\"835\" as=\"sourcePoint\"/><mxPoint x=\"1148\" y=\"695\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-57\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"988\" y=\"495\" as=\"sourcePoint\"/><mxPoint x=\"928\" y=\"565\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-58\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"1078\" y=\"495\" as=\"sourcePoint\"/><mxPoint x=\"1136\" y=\"565\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-61\" value=\"View dashboards&lt;br&gt;in browser\" style=\"shape=flexArrow;endArrow=classic;html=1;strokeColor=#9673a6;strokeWidth=1;fillColor=#e1d5e7;spacing=8;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"1259.5\" y=\"415\" as=\"sourcePoint\"/><mxPoint x=\"1109.5\" y=\"445\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-63\" value=\"&lt;h1&gt;Monitoring architecture&amp;nbsp;&lt;/h1&gt;&lt;p&gt;Keep it simple, sunshine!&lt;br&gt;&lt;br&gt;&lt;/p&gt;&lt;p&gt;&lt;/p&gt;&lt;i&gt;Grafana&lt;/i&gt; retrieves metrics from &lt;i&gt;Prometheus&lt;/i&gt; and logs from&amp;nbsp;&lt;i&gt;Loki&lt;/i&gt;,&amp;nbsp;&lt;span&gt;shows dashboards (in a web browser) and does alerting (via Zulip)&lt;/span&gt;&lt;br&gt;&lt;p&gt;&lt;i&gt;Prometheus&lt;/i&gt; stores metrics it pulls from various &lt;i&gt;Exporters&lt;/i&gt; on nodes&lt;/p&gt;&lt;p&gt;&lt;i&gt;Promtail&lt;/i&gt; on nodes pushes logs to &lt;i&gt;Loki&lt;br&gt;&lt;br&gt;&lt;/i&gt;&lt;/p&gt;&lt;p&gt;We try to keep the system as simple as possible: All monitoring and alerting runs on a single machine.&lt;/p&gt;&lt;h2&gt;Changes&lt;/h2&gt;&lt;div&gt;v3: Fix WireGuard/Wireshark braino, Update Auth (Google, not GitHub)&lt;/div&gt;&lt;div&gt;&lt;br&gt;&lt;/div&gt;&lt;div&gt;v2: Add Github authentication to Grafana. Add management VPN.&lt;br&gt;&lt;br&gt;v1: Initial version&lt;/div&gt;\" style=\"text;html=1;strokeColor=none;fillColor=none;spacing=5;spacingTop=-20;whiteSpace=wrap;overflow=hidden;rounded=0;shadow=0;comic=0;sketch=0;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"596\" y=\"315\" width=\"164\" height=\"605\" as=\"geometry\"/></mxCell><mxCell id=\"TrmSFti5pUnXIGkjMKb6-65\" value=\"Send alerts\" style=\"shape=flexArrow;endArrow=classic;html=1;strokeColor=#9673a6;strokeWidth=1;fillColor=#e1d5e7;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"1086\" y=\"405.46000000000004\" as=\"sourcePoint\"/><mxPoint x=\"1236\" y=\"375.46000000000004\" as=\"targetPoint\"/></mxGeometry></mxCell><UserObject label=\"Monitoring server\" link=\"https://monitoring.private.storage/\" id=\"2mYkRctJDop23S32jJdh-3\"><mxCell style=\"text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"845\" y=\"411\" width=\"120\" height=\"20\" as=\"geometry\"/></mxCell></UserObject><mxCell id=\"TrmSFti5pUnXIGkjMKb6-44\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#6c8ebf;strokeWidth=1;fillColor=#dae8fc;\" parent=\"1\" edge=\"1\"><mxGeometry x=\"806\" y=\"695\" width=\"50\" height=\"50\" as=\"geometry\"><mxPoint x=\"894.6666666666666\" y=\"695\" as=\"sourcePoint\"/><mxPoint x=\"826\" y=\"835\" as=\"targetPoint\"/></mxGeometry></mxCell><mxCell id=\"vhdg0YFc32S7_3H95Ew1-7\" value=\"GSuite&lt;br&gt;OAuth2\" style=\"ellipse;shape=cloud;whiteSpace=wrap;html=1;\" parent=\"1\" vertex=\"1\"><mxGeometry x=\"984\" y=\"305\" width=\"98\" height=\"60\" as=\"geometry\"/></mxCell><mxCell id=\"vhdg0YFc32S7_3H95Ew1-8\" value=\"\" style=\"endArrow=classic;html=1;strokeColor=#d79b00;strokeWidth=1;fillColor=#ffe6cc;\" parent=\"1\" edge=\"1\"><mxGeometry width=\"50\" height=\"50\" relative=\"1\" as=\"geometry\"><mxPoint x=\"1032.76\" y=\"417\" as=\"sourcePoint\"/><mxPoint x=\"1032.76\" y=\"372\" as=\"targetPoint\"/></mxGeometry></mxCell></root></mxGraphModel></diagram></mxfile>","toolbar":"pages zoom layers lightbox","page":0}"></div> +<script type="text/javascript" src="https://app.diagrams.net/js/viewer-static.min.js"></script> +</body> +</html> diff --git a/docs/ops/monitoring.rst b/docs/ops/monitoring.rst index e30831ade4ef71a0abd101e383065f706f586b63..8e204bf9de34ccf34fcd792fb051fdda867c5cca 100644 --- a/docs/ops/monitoring.rst +++ b/docs/ops/monitoring.rst @@ -17,6 +17,15 @@ Analyzing long-term trends How big is my database and how fast is it growing? How quickly is my daily-active user count growing? +Architecture +```````````` + +A high-level overview of the software and systems that comprise our monitoring and alerting infrastructure: + +.. raw:: html + :file: monitoring-architecture.html + + Introduction to our dashboards ``````````````````````````````