update production

Changes from develop since 2022-09-07T12:59:37+00:00

Deployment Notes

Included Merge Requests

• !349 (merged)
• !347 (merged)
• !346 (merged)
• !345 (merged)
• !344 (merged)
• !341 (merged)
• !342 (merged)
• !343 (merged)

Diff Stat

 nixos/modules/monitoring/exporters/tahoe.nix       |  16 +-
 .../server/grafana-dashboards/backups.json         |   8 +-
 .../grafana-dashboards/resources-overview.json     |  55 +-
 .../server/grafana-dashboards/tahoe-lafs.json      | 807 +++++++++++++--------
 nixos/modules/monitoring/server/prometheus.nix     |   2 +
 5 files changed, 560 insertions(+), 328 deletions(-)

nixos/modules/monitoring/exporters/tahoe.nix

 # Tahoe Prometheus metrics collector
 #
-# Scope: Retrieves OpenMetrics from Tahoe and puts them
-#        where textfile collector can find them.
+# Scope: Retrieve metrics from Tahoe and put them where Prometheus'
+#        node-exporter's textfile collector can find them.
 #
 # Usage: Import this to every server running Tahoe.
 #
@@ -55,14 +55,22 @@ in {
         description = "Tahoe metrics gathering service";
         after = [ "tahoe.storage.service" ];
         startAt = cfg.interval;
-        path = [ pkgs.curl ];
+        path = [ pkgs.coreutils pkgs.findutils pkgs.curl ];
         restartIfChanged = false;
 
         # Save to a temp file and then move atomically so the
         # textfile collector won't read a partial file.
         # See https://github.com/prometheus/node_exporter#textfile-collector
         script = ''
-          curl --silent --show-error --fail-with-body --output "${cfg.outFile}.tmp" "${cfg.scrapeEndpoint}"
+          set -euo pipefail
+
+          NUM_CORRUPTION_ADVISORIES=$(find /storage/corruption-advisories/ -type f | wc -l)
+          echo "tahoe_corruption_advisories_total $NUM_CORRUPTION_ADVISORIES" > "${cfg.outFile}.tmp"
+
+          NUM_INCIDENT_REPORTS=$(find /var/db/tahoe-lafs/storage/logs/incidents/ -type f | wc -l)
+          echo "tahoe_incident_reports_total $NUM_INCIDENT_REPORTS" >> "${cfg.outFile}.tmp"
+
+          curl --silent --show-error --fail-with-body "${cfg.scrapeEndpoint}" >> "${cfg.outFile}.tmp"
           mv "${cfg.outFile}.tmp" "${cfg.outFile}"
         '';
       };