morph/README.rst

@@ -36,11 +36,17 @@ lib
 ---
 
 This contains Nix library code for defining the grids.
+It has all the details of how each type of node in our grid is configured.
+It knows about morph (so defines ``deployment.secrets`` and has the logic for collecting data defined by other nodes).
+It defines options (i.e. ``grid.*``) for things specific to how we configure grids (e.g. ``grid.publicKeyPath``).
+It defines metadata about nodes that we use on other nodes (e.g. ``grid.monitoringvpnIPv4`` which is used to define various things on the monitoring node).
+Each top-level module here defines one type of node with all (or at least most) of the configuration necessary for that node.
 
 grid
 ----
 
 Specific grid definitions live in subdirectories beneath this directory.
+They consist almost exclusively setting options defined in ``morph/lib`` (and few options defined elsewhere) and then delegating to the ``morph/lib`` modules.
 
 private-keys
 ~~~~~~~~~~~~

morph/default.nix

+{ pkgs ? import ../nixpkgs.nix {} }:
+let
+  lib = pkgs.lib;
+  gridlib = import ./lib;
+  inherit (gridlib.pkgs) ourpkgs;
+  grids-path = "${builtins.toString ./.}/grid";
+  grid-configs = lib.mapAttrs (n: v: grids-path + "/${n}/grid.nix") (lib.filterAttrs (n: v: v == "directory") (builtins.readDir grids-path));
+  # It would be useful if morph exposed this as a function.
+  # https://github.com/DBCDK/morph/pull/166
+  morph-eval = networkExpr: (import "${pkgs.morph.lib}/eval-machines.nix") { inherit networkExpr; };
+  grids = lib.mapAttrs (n: v: (morph-eval v)) grid-configs;
+  # Derivation with symlinks to the morph output for each grid.
+  output = pkgs.runCommand "privatestorage-morph"
+    { preferLocalBuild = true; allowSubstitutes = false; passthru = { inherit gridlib ourpkgs grids; }; }
+    ''
+      mkdir $out
+      ${lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (
+        name: morph:
+          let
+            output = morph.machines {
+              # It would be nice if we didn't need to write this data to a file.
+              # https://github.com/DBCDK/morph/pull/186
+              argsFile = pkgs.writeText "args" (builtins.toJSON { Names = lib.attrNames morph.nodes; });
+            };
+          in
+            ''
+              ln -s ${output} $out/${lib.escapeShellArg name}
+            ''
+      ) grids
+    )}'';
+in output

morph/grid/local/README.rst

@@ -8,14 +8,18 @@ Issues with networking that looked like guest misconfigurations vanished after c
 This requires `NixOS <https://nixos.org/>`_.
 Nix without the OS will not work.
 
+
 Use the local development environment
 `````````````````````````````````````
 
-0. Add VirtualBox to your NixOs system configuration at ``/etc/nixos/configuration.nix``::
+0. Add to your NixOS system configuration at ``/etc/nixos/configuration.nix`` (and rebuild)::
 
-    virtualisation.virtualbox.host.enable = true;
-    # Save bytes and build time, optional but recommended:
-    virtualisation.virtualbox.host.headless = true;
+    # Enable libvirt - likely incompatible with virtualisation.virtualbox!
+    virtualisation.libvirtd.enable = true;
+    # Required for LibVirt
+    security.polkit.enable = true;
+    # Enable HW acceleration if (nested virtualisation is) available
+    #boot.kernelModules = [ "kvm-amd" "kvm-intel" ];
 
 1. Enter the morph local grid directory::
 
@@ -27,7 +31,15 @@ Use the local development environment
 
 3. Build and start the VMs::
 
-    VAGRANT_DEFAULT_PROVIDER=virtualbox vagrant up
+    vagrant up --provider=libvirt
+
+   Optionally, to switch from QEMU to KVM virtualization, edit the virtual machine definition of all the machines and replace the "qemu" on the first line with "kvm"::
+
+    sudo virsh list
+    sudo virsh edit <machine id> (once for every machine)
+    vagrant halt
+    vagrant up
+
 
 4. Then, add the Vagrant SSH configuration to your user's ``~/.ssh/config`` file::
 
@@ -35,11 +47,11 @@ Use the local development environment
 
    Latest Morph honors the ``SSH_CONFIG_FILE`` environment variable (`since 3f90aa88 (March 2020, v 1.5.0) <https://github.com/DBCDK/morph/commit/3f90aa885fac1c29fce9242452fa7c0c505744ef#diff-d155ad793bd62e6ea4c44ba985049ecb13a4f4f32f799791b2bce695a16c0101>`_), so in the future this should get a bit more convenient.
 
-6. Create a ``public-keys/users.nix`` file with your SSH key (see ``public-keys/users.nix.example`` for the format) so you'll be able to log in after deploying the new configuration::
+5. Create a ``public-keys/users.nix`` file with your SSH key (see ``public-keys/users.nix.example`` for the format) so you'll be able to log in after deploying the new configuration::
 
     $EDITOR public-keys/users.nix
 
-7. Then, build and deploy our software to the Vagrant VMs::
+6. Then, build and deploy our software to the Vagrant VMs::
 
     morph build grid.nix
     morph push grid.nix

morph/grid/local/Vagrantfile

 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-# This Vagrantfile worked for Florian Sesser using Vagrant 2.2.16dev and
-# the VirtualBox Hypervisor. Earlier Vagrant and LibVirt did not work.
+# This Vagrantfile worked for Florian Sesser using Vagrant 2.2.19 and
+# the LibVirt with QEmu Hypervisor. Earlier Vagrant and VirtualBox did worked too.
+
+# Get a dedicated LibVirt pool name or use default one
+pool_name = ENV.has_key?('POOL_NAME') ? ENV['POOL_NAME'] : 'default'
+# For instance, one could create such pool beforehand as follows:
+#   export POOL_NAME=morph_local_$(id -un)
+#   POOL_PATH="/path/to/your/storage"
+#   mkdir -p "${POOL_PATH}"
+#   sudo virsh pool-define-as ${POOL_NAME} --type dir --target "${POOL_PATH}"
+#   sudo virsh pool-autostart ${POOL_NAME}
+#   sudo virsh pool-start ${POOL_NAME}
 
 Vagrant.configure("2") do |config|
   # For a complete reference, please see the online documentation at
   # https://docs.vagrantup.com.
 
-  config.vm.define "payments.localdev" do |config|
-    config.vm.hostname = "payments"
+  # Select the base image
   config.vm.box = "esselius/nixos"
   config.vm.box_version = "20.09"
   config.vm.box_check_update = false
 
+  # No need to sync the working dir. with the guest boxess
+  # Better use SFTP to transfer 
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+
+  # Tune LibVirt/QEmu guests
+  config.vm.provider :libvirt do |domain|
+    # The default of one CPU should work
+    # Increase to speed up boot/push/deploy
+    # domain.cpus = 1
+
     # To use the self-updating deployment system you need more memory.  Giving
     # all of the VMs enough memory for this is rather taxing, though, and the
     # self-updating deployment system is not particularly useful for local
     # dev.  But should you want to:
     #
-    # config.vm.provider "virtualbox" do |v|
-    #   v.memory = 4096
-    # end
+    # domain.memory = 4096
+    #
+    # Meanwhile, 1024 was apparently the default with VirtualBox 
+    domain.memory = 1024
 
-    config.vm.network "private_network", ip: "192.168.67.21"
+    # Using a specific pool may help to manage the disk space
+    domain.storage_pool_name = pool_name
+    domain.snapshot_pool_name = pool_name
+
+    # No need of graphics - better use serial
+    domain.graphics_type = "none"
+    domain.video_type = "none"
+  end
+
+  config.vm.define "payments.localdev" do |config|
+    config.vm.hostname = "payments"
+
+    # Assign a static IP address inside the box host-only (Vagrant
+    # calls it "private") network.  The address must be in the range
+    # VirtualBox allows.
+    # https://www.virtualbox.org/manual/ch06.html#network_hostonly says some
+    # things about this.
+    config.vm.network "private_network", ip: "192.168.56.21"
     # Add self signed SSL key for zkap-issuer:
     config.vm.provision "file", source: "private-keys/payments-localdev-ssl", destination: "/tmp/payments-localdev-ssl"
     config.vm.provision "shell", inline: "sudo mkdir -p /var/lib/letsencrypt/live/payments.localdev/"
@@ -32,35 +69,30 @@ Vagrant.configure("2") do |config|
 
   config.vm.define "storage1.localdev" do |config|
     config.vm.hostname = "storage1"
-    config.vm.box = "esselius/nixos"
-    config.vm.box_version = "20.09"
-    config.vm.box_check_update = false
-    config.vm.network "private_network", ip: "192.168.67.22"
+    config.vm.network "private_network", ip: "192.168.56.22"
   end
 
   config.vm.define "storage2.localdev" do |config|
     config.vm.hostname = "storage2"
-    config.vm.box = "esselius/nixos"
-    config.vm.box_version = "20.09"
-    config.vm.box_check_update = false
-    config.vm.network "private_network", ip: "192.168.67.23"
+    config.vm.network "private_network", ip: "192.168.56.23"
   end
 
   config.vm.define "monitoring.localdev" do |config|
     config.vm.hostname = "monitoring"
-    config.vm.box = "esselius/nixos"
-    config.vm.box_version = "20.09"
-    config.vm.box_check_update = false
-    config.vm.network "private_network", ip: "192.168.67.24"
+    config.vm.network "private_network", ip: "192.168.56.24"
   end
 
   # To make the VMs assign the static IPs to the network interfaces we need a rebuild:
-  config.vm.provision "shell", inline: "echo '{nix.trustedUsers = [ \"@wheel\" \"root\" \"vagrant\" ];}' > /etc/nixos/custom-configuration.nix"
+  ## Rename to 'nix.settings.trusted-users' after 20.09 or so:
+  config.vm.provision "shell",
+    inline: "echo '{ nix.trustedUsers = [ \"@wheel\" \"root\" \"vagrant\" ]; boot.kernelParams = [ \"console=tty0\" \"console=ttyS0,115200\" ]; }' > /etc/nixos/custom-configuration.nix"
   config.vm.provision "shell", inline: "nixos-rebuild switch"
+  config.vm.provision "shell", inline: "systemctl stop firewall.service"
+  config.vm.provision "shell", inline: "systemctl start serial-getty@ttyS0.service"
 
   config.trigger.after :up do |trigger|
     trigger.info = "Hostname and IP address this host actually uses:"
-    trigger.run_remote = {inline: "echo `hostname` `ifconfig | egrep -o '192.168.67.[0-9]* '`"}
+    trigger.run_remote = {inline: "echo `hostname` `ifconfig | egrep -o '192.168.56.[0-9]* '`"}
   end
 
 end

morph/grid/local/config.json

@@ -2,7 +2,7 @@
 , "publicStoragePort": 8898
 , "publicKeyPath": "./public-keys"
 , "privateKeyPath": "./private-keys"
-, "monitoringvpnEndpoint": "192.168.67.24:51820"
+, "monitoringvpnEndpoint": "192.168.56.24:51820"
 , "passValue": 1000000
 , "issuerDomains": ["payments.localdev"]
 , "monitoringDomains": ["monitoring.localdev"]

morph/grid/local/grid.nix

 let
-  pkgs = import <nixpkgs> { };
-
   gridlib = import ../../lib;
-  grid-config = pkgs.lib.trivial.importJSON ./config.json;
+  grid-config = builtins.fromJSON (builtins.readFile ./config.json);
 
   ssh-users = let
     ssh-users-file = ./public-keys/users.nix;
@@ -59,6 +57,7 @@ let
     grid = {
       publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
       privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
+      inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
     };
     # Configure deployment management authorization for all systems in the grid.
     services.private-storage.deployment = {
@@ -70,75 +69,67 @@ let
   payments = {
     imports = [
       gridlib.issuer
-      (gridlib.customize-issuer (grid-config // {
-          monitoringvpnIPv4 = "172.23.23.11";
-      }))
       grid-module
     ];
     config = {
-      grid.publicIPv4 = "192.168.67.21";
+      grid.monitoringvpnIPv4 = "172.23.23.11";
+      grid.publicIPv4 = "192.168.56.21";
+      grid.issuer = {
+        inherit (grid-config) issuerDomains allowedChargeOrigins;
+      };
     };
   };
 
   storage1 = {
     imports = [
       gridlib.storage
-      (gridlib.customize-storage (grid-config // {
-        monitoringvpnIPv4 = "172.23.23.12";
-        stateVersion = "19.09";
-      }))
       grid-module
     ];
     config = {
-      grid.publicIPv4 = "192.168.67.22";
+      grid.monitoringvpnIPv4 = "172.23.23.12";
+      grid.publicIPv4 = "192.168.56.22";
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = "19.09";
     };
   };
 
   storage2 = {
     imports = [
       gridlib.storage
-      (gridlib.customize-storage (grid-config // {
-        monitoringvpnIPv4 = "172.23.23.13";
-        stateVersion = "19.09";
-      }))
       grid-module
     ];
     config = {
-      grid.publicIPv4 = "192.168.67.23";
+      grid.monitoringvpnIPv4 = "172.23.23.13";
+      grid.publicIPv4 = "192.168.56.23";
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = "19.09";
     };
   };
 
   monitoring = {
     imports = [
       gridlib.monitoring
-      (gridlib.customize-monitoring {
-        inherit hostsMap vpnClientIPs
-                nodeExporterTargets
-                paymentExporterTargets
-                blackboxExporterHttpsTargets;
-        inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
-        googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
-        enableSlackAlert = false;
-        monitoringvpnIPv4 = "172.23.23.1";
-        stateVersion = "19.09";
-      })
       grid-module
     ];
     config = {
-      grid.publicIPv4 = "192.168.67.24";
+      grid.monitoringvpnIPv4 = "172.23.23.1";
+      grid.publicIPv4 = "192.168.56.24";
+      grid.monitoring = {
+        inherit paymentExporterTargets blackboxExporterHttpsTargets;
+        inherit (grid-config) monitoringDomains;
+        googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+        enableZulipAlert = false;
+      };
+      system.stateVersion = "19.09";
     };
   };
 
   # TBD: derive these automatically:
-  hostsMap = {
-    "172.23.23.1"  = [ "monitoring" "monitoring.monitoringvpn" ];
-    "172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
-    "172.23.23.12" = [ "storage1" "storage1.monitoringvpn" ];
-    "172.23.23.13" = [ "storage2" "storage2.monitoringvpn" ];
-  };
-  vpnClientIPs = [ "172.23.23.11" "172.23.23.12" "172.23.23.13" ];
-  nodeExporterTargets = [ "monitoring" "payments" "storage1" "storage2" ];
-  paymentExporterTargets = [ "payments" ];
+  paymentExporterTargets = [ "payments.monitoringvpn" ];
   blackboxExporterHttpsTargets = [
     # "https://private.storage/"
     # "https://payments.private.storage/"

morph/grid/local/private-keys/README.rst

@@ -11,7 +11,8 @@ You can find more information about some of these secrets in ``ops/generating-ke
 deploy_key
 ----------
 
-This is an SSH private key which will be authorized to trigger a deployment update on the deployment hosts themselves.
+This SSH private key authenticates an account used by the continuous deployment system.
+Each node authorizes that account to trigger a deployment update on itself. 
 The corresponding SSH public key is kept in the ``public-keys`` location.
 
 grafana-admin.password
@@ -26,6 +27,13 @@ This file is read by Grafana's systemd service to set an environment variable wi
 The only line in the file should be the secret URL.
 Use the url from `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ or get a new secret URL for your Slack channel at https://www.slack.com/apps/A0F7XDUAZ.
 
+grafana-zulip-url
+-----------------
+
+This file should contain a single line with the secret Zulip alerting Webhook Bot URL.
+The URLs for Staging and Production are both stored in 1Password.
+See `https://zulip.com/integrations/doc/grafana`_ for documentation and ``grid/local/private-keys/grafana-zulip-url`` for an example.
+
 stripe.secret
 -------------
 

morph/grid/local/private-keys/borgbackup.passphrase

+The most interesting passphrase in the world.

morph/grid/local/private-keys/borgbackup.ssh-key

+-----BEGIN OPENSSH PRIVATE KEY-----
+ratatatratatatratatatratatatratatatratatatratatatratatatratatatratatat
+ratatatratatatratatatratatatratatatratatatratatatratatatratatatratatat
+ratatatratatatratatatratatatratatatratatatratatatratatatratatatratatat
+ratatatratatatratatatratatatratatatratatatratatatratatatratatatratatat
+ratatatratatatratatatratatatratatatratatatc=
+-----END OPENSSH PRIVATE KEY-----

morph/grid/local/private-keys/grafana-zulip-url

+https://yourZulipDomain.zulipchat.com/api/v1/external/grafana?api_key=abcdefgh&stream=stream%20name&topic=your%20topic

morph/grid/local/private-keys/stripe.webhook-secret

+whsec_12121212121212121212121212121212121212

morph/grid/local/public-keys/borgbackup/storage1.repopath

+abc123de@abc123de.repo.borgbase.com:repo

morph/grid/local/public-keys/borgbackup/storage2.repopath

+vwx789yz@vwx789yz.repo.borgbase.com:repo

morph/grid/local/public-keys/users.nix.example

+let
 # Add your public key. Example:
-# let key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la";
+# key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la";
 # You can use the following to get key from the local machine.
-# let key = builtins.readFile ~/.ssh/id_ed25519.pub;
-let key = undefined;
-in { "root" = key; "vagrant" = key; }
+# key = builtins.readFile ~/.ssh/id_ed25519.pub;
+  key = undefined;
+  keys = [key]
+in {
+  "root" = keys;
+  "vagrant" = keys;
+}

morph/grid/production/config.json

@@ -5,19 +5,16 @@
 , "monitoringvpnEndpoint": "monitoring.private.storage:51820"
 , "passValue": 1000000
 , "issuerDomains": [
-    "payments.privatestorage.io"
-  , "payments.private.storage"
+    "payments.private.storage"
+  , "payments.privatestorage.io"
   ]
 , "monitoringDomains": [
-    "monitoring.privatestorage.io"
-  , "monitoring.private.storage"
+    "monitoring.private.storage"
+  , "monitoring.privatestorage.io"
   ]
 , "letsEncryptAdminEmail": "jean-paul@privatestorage.io"
 , "allowedChargeOrigins": [
-    "https://privatestorage.io"
-  , "https://www.privatestorage.io"
-  , "https://private.storage"
-  , "https://www.private.storage"
+    "https://private.storage"
   ]
 , "monitoringGoogleOAuthClientID": "802959152038-klpkk38sfnqmknn1ucg7pvs4hcc2k8ae.apps.googleusercontent.com"
 }

morph/grid/production/grid.nix

 # See morph/grid/local/grid.nix for additional commentary.
 let
-  pkgs = import <nixpkgs> { };
-
   gridlib = import ../../lib;
-  grid-config = pkgs.lib.trivial.importJSON ./config.json;
+  grid-config = builtins.fromJSON (builtins.readFile ./config.json);
 
   # Module with per-grid configuration
   grid-module = {config, ...}: {
@@ -21,6 +19,7 @@ let
     grid = {
       publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
       privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
+      inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
     };
     # Configure deployment management authorization for all systems in the grid.
     services.private-storage.deployment = {
@@ -33,36 +32,38 @@ let
     imports = [
       gridlib.issuer
       gridlib.hardware-aws
-      (gridlib.customize-issuer (grid-config // {
-        monitoringvpnIPv4 = "172.23.23.11";
-      }))
       grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.11";
+      grid.issuer = {
+        inherit (grid-config) issuerDomains allowedChargeOrigins;
+      };
+    };
   };
 
   monitoring = {
     imports = [
       gridlib.monitoring
       gridlib.hardware-aws
-      (gridlib.customize-monitoring {
-        inherit hostsMap vpnClientIPs
-                nodeExporterTargets
-                paymentExporterTargets
-                blackboxExporterHttpsTargets;
-        inherit (grid-config) letsEncryptAdminEmail monitoringDomains;
-        googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
-        enableSlackAlert = true;
-        monitoringvpnIPv4 = "172.23.23.1";
-        stateVersion = "19.09";
-      })
       grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.1";
+      grid.monitoring = {
+        inherit paymentExporterTargets blackboxExporterHttpsTargets;
+        inherit (grid-config) monitoringDomains;
+        googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+        enableZulipAlert = true;
+      };
+      system.stateVersion = "19.09";
+    };
   };
 
   defineStorageNode = name: { vpnIP, stateVersion }:
   let
-    nodecfg = import "${./.}/${name}-config.nix";
-    hardware ="${./.}/${name}-hardware.nix";
+    nodecfg = import (./. + "/${name}-config.nix");
+    hardware = (./. + "/${name}-hardware.nix");
   in {
     imports = [
       # Get some of the very lowest-level system configuration for this
@@ -79,17 +80,17 @@ let
 
       # Get all of the configuration that is common across all storage nodes.
       gridlib.storage
-
-      # Then customize the storage system a little bit based on this node's particulars.
-      (gridlib.customize-storage (grid-config // nodecfg // {
-        monitoringvpnIPv4 = vpnIP;
-        inherit stateVersion;
-      }))
-
       # Also configure deployment management authorization
       grid-module
     ];
 
+    config = {
+      grid.monitoringvpnIPv4 = vpnIP;
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = stateVersion;
+
       # And supply configuration for those hardware / network / bootloader
       # options.  See the 100tb module for handling of this value.  The module
       # name is quoted because `1` makes `100tb` look an awful lot like a
@@ -98,7 +99,8 @@ let
 
       # Enable statistics gathering for MegaRAID cards.
       # TODO would be nice to enable only on machines that have such a device.
-    services.private-storage.monitoring.megacli2prom.enable = true;
+      services.private-storage.monitoring.exporters.megacli2prom.enable = true;
+    };
   };
 
   # Define all of the storage nodes for this grid.
@@ -110,34 +112,7 @@ let
     storage005 = { vpnIP = "172.23.23.25"; stateVersion = "19.03"; };
   };
 
-  # TBD: derive these automatically:
-  hostsMap = {
-    "172.23.23.1"  = [ "monitoring" "monitoring.monitoringvpn" ];
-    "172.23.23.11" = [   "payments"   "payments.monitoringvpn" ];
-    "172.23.23.21" = [ "storage001" "storage001.monitoringvpn" ];
-    "172.23.23.22" = [ "storage002" "storage002.monitoringvpn" ];
-    "172.23.23.23" = [ "storage003" "storage003.monitoringvpn" ];
-    "172.23.23.24" = [ "storage004" "storage004.monitoringvpn" ];
-    "172.23.23.25" = [ "storage005" "storage005.monitoringvpn" ];
-  };
-  vpnClientIPs = [
-    "172.23.23.11"
-    "172.23.23.21"
-    "172.23.23.22"
-    "172.23.23.23"
-    "172.23.23.24"
-    "172.23.23.25"
-  ];
-  nodeExporterTargets = [
-    "monitoring"
-    "payments"
-    "storage001"
-    "storage002"
-    "storage003"
-    "storage004"
-    "storage005"
-  ];
-  paymentExporterTargets = [ "payments" ];
+  paymentExporterTargets = [ "payments.monitoringvpn" ];
   blackboxExporterHttpsTargets = [
     "https://private.storage/"
     "https://www.private.storage/"

morph/grid/production/public-keys/borgbackup/storage001.repopath

+gye1flhy@gye1flhy.repo.borgbase.com:repo

morph/grid/production/public-keys/borgbackup/storage002.repopath

+l4642x1g@l4642x1g.repo.borgbase.com:repo

morph/grid/production/public-keys/borgbackup/storage003.repopath

+c7400xl6@c7400xl6.repo.borgbase.com:repo

morph/grid/production/public-keys/borgbackup/storage004.repopath

+sbn13vf8@sbn13vf8.repo.borgbase.com:repo