Grafana: Payments dashboard log fix & fine tune

@jcalderone This is a JSON-blob-only diff for the "Payments" dashboard. It should fix the log panels, make a couple of things a little bit better and probably nothing worse.

---

• (tryfix) Switch to Loki datasource;
• Filter requests to metrics instead of all GET;
• Show the last week by default
• Show log times

Interesting and not quickly fixable as it looks like: the GET ... line comes after the result. We might be logging this wrong to begin with, but probably this is some next gen AI line ordering intelligence

Update: Later added:

• Turn on shared crosshair for this dashboard (we might want this on the other dashes too?)
• Semi-transparency for the two lastly edited panels, for the looks

nixos/modules/monitoring/server/grafana-dashboards/payments.json

@@ -21,7 +21,7 @@
   "description": "PaymentServer and related metrics",
   "editable": true,
   "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
+  "graphTooltip": 2,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -744,6 +744,7 @@
       "type": "timeseries"
     },
     {
+      "description": "HTTP 4xx and 5xx errors",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -754,7 +755,7 @@
             "axisPlacement": "auto",
             "barAlignment": 0,
             "drawStyle": "line",
-            "fillOpacity": 10,
+            "fillOpacity": 20,
             "gradientMode": "none",
             "hideFrom": {
               "legend": false,
@@ -762,16 +763,19 @@
               "viz": false
             },
             "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
             "lineWidth": 1,
             "pointSize": 5,
             "scaleDistribution": {
               "type": "linear"
             },
             "showPoints": "never",
-            "spanNulls": true,
+            "spanNulls": false,
             "stacking": {
               "group": "A",
-              "mode": "normal"
+              "mode": "none"
             },
             "thresholdsStyle": {
               "mode": "off"
@@ -812,7 +816,7 @@
           "placement": "bottom"
         },
         "tooltip": {
-          "mode": "single"
+          "mode": "multi"
         }
       },
       "pluginVersion": "8.3.4",
@@ -1031,7 +1035,7 @@
     {
       "datasource": {
         "type": "loki",
-        "uid": "P8E80F9AEF21F6940"
+        "uid": "000000002"
       },
       "description": "Exercise in counting maybe interesting lines. This can be alerted on.",
       "fieldConfig": {
@@ -1044,7 +1048,7 @@
             "axisPlacement": "auto",
             "barAlignment": 0,
             "drawStyle": "line",
-            "fillOpacity": 0,
+            "fillOpacity": 20,
             "gradientMode": "none",
             "hideFrom": {
               "legend": false,
@@ -1052,6 +1056,9 @@
               "viz": false
             },
             "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
             "lineWidth": 1,
             "pointSize": 5,
             "scaleDistribution": {
@@ -1107,10 +1114,9 @@
         {
           "datasource": {
             "type": "loki",
-            "uid": "P8E80F9AEF21F6940"
+            "uid": "000000002"
           },
-          "expr": "count_over_time({host=\"payments\",unit=\"zkapissuer.service\"} !~ \"200|GET|Accept\" [5m])",
-          "maxLines": 0,
+          "expr": "count_over_time({host=\"payments\",unit=\"zkapissuer.service\"} !~ \"200|/metrics|Accept\" [5m])",
           "refId": "A"
         }
       ],
@@ -1121,7 +1127,7 @@
     {
       "datasource": {
         "type": "loki",
-        "uid": "P8E80F9AEF21F6940"
+        "uid": "000000002"
       },
       "description": "Exercise in filtering the payment server logs.",
       "gridPos": {
@@ -1137,7 +1143,7 @@
         "prettifyLogMessage": false,
         "showCommonLabels": false,
         "showLabels": false,
-        "showTime": false,
+        "showTime": true,
         "sortOrder": "Descending",
         "wrapLogMessage": false
       },
@@ -1145,9 +1151,9 @@
         {
           "datasource": {
             "type": "loki",
-            "uid": "P8E80F9AEF21F6940"
+            "uid": "000000002"
           },
-          "expr": "{host=\"payments\",unit=\"zkapissuer.service\"} !~ \"200|GET|Accept\"",
+          "expr": "{host=\"payments\",unit=\"zkapissuer.service\"} !~ \"200|/metrics|Accept\"",
           "refId": "A"
         }
       ],
@@ -1163,7 +1169,7 @@
     "list": []
   },
   "time": {
-    "from": "now-3h",
+    "from": "now-7d",
     "to": "now"
   },
   "timepicker": {