morph/grid/local/grid.nix

 let
-  pkgs = import <nixpkgs> { };
-
   gridlib = import ../../lib;
-  rawConfig = pkgs.lib.trivial.importJSON ./config.json;
-  config = rawConfig // {
-    sshUsers = import ./public-keys/users.nix;
+  grid-config = builtins.fromJSON (builtins.readFile ./config.json);
+
+  ssh-users = let
+    ssh-users-file = ./public-keys/users.nix;
+  in
+    if builtins.pathExists ssh-users-file then
+      import ssh-users-file
+    else
+      # Use builtins.toString so that nix does not add the file
+      # to the nix store before including it in the string.
+      throw ''
+        ssh-keys for local grid are not configured.
+        Refusing to build a possibly inaccessible configuration.
+        Please create ${builtins.toString ssh-users-file} before building.
+        See ${builtins.toString ./README.rst} for more information.
+      '';
+
+  # Module with per-grid configuration
+  grid-module = {config, ...}: {
+    imports = [
+      gridlib.base
+      # Allow us to remotely trigger updates to this system.
+      ../../../nixos/modules/deployment.nix
+      # Give it a good SSH configuration.
+      ../../../nixos/modules/ssh.nix
+      # Configure things specific to the virtualisation environment.
+      gridlib.hardware-vagrant
+    ];
+    services.private-storage.sshUsers = ssh-users;
 
+    # Include the ssh-users config in a form that can be read by nix,
+    # so the self-update deployment system can access it.
+    # nixos/modules/update-deployment imports the nix file into
+    # the checkout of this repository it creates.
+    environment.etc."nixos/ssh-users.json" = {
+      # Output the loaded value, rather than just copying the file, in case the
+      # file has external references.
+      mode = "0666";
+      text = builtins.toJSON ssh-users;
+    };
+    environment.etc."nixos/ssh-users.nix" = {
+      # This is the file that is imported by update-deployment.
+      # We don't directly read the JSON so that the script doesn't
+      # depend on the format we use.
+      mode = "0666";
+      text = ''
+        # Include the ssh-users config
+        builtins.fromJSON (builtins.readFile ./ssh-users.json)
+      '';
+    };
+
+    networking.domain = grid-config.domain;
     # Convert relative paths to absolute so library code can resolve names
     # correctly.
-    publicKeyPath = toString ./. + "/${rawConfig.publicKeyPath}";
-    privateKeyPath = toString ./. + "/${rawConfig.privateKeyPath}";
+    grid = {
+      publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
+      privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
+      inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
     };
-
     # Configure deployment management authorization for all systems in the grid.
-  deployment = {
     services.private-storage.deployment = {
-      authorizedKey = builtins.readFile "${config.publicKeyPath}/deploy_key.pub";
+      authorizedKey = builtins.readFile "${config.grid.publicKeyPath}/deploy_key.pub";
       gridName = "local";
     };
   };
@@ -23,67 +69,76 @@ let
   payments = {
     imports = [
       gridlib.issuer
-      (gridlib.hardware-virtual ({ publicIPv4 = "192.168.67.21"; }))
-      (gridlib.customize-issuer (config // {
-          monitoringvpnIPv4 = "172.23.23.11";
-      }))
-      deployment
+      grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.11";
+      grid.publicIPv4 = "192.168.56.21";
+      grid.issuer = {
+        inherit (grid-config) issuerDomains allowedChargeOrigins;
+      };
+    };
   };
 
   storage1 = {
     imports = [
       gridlib.storage
-      (gridlib.hardware-virtual ({ publicIPv4 = "192.168.67.22"; }))
-      (gridlib.customize-storage (config // {
-        monitoringvpnIPv4 = "172.23.23.12";
-        stateVersion = "19.09";
-      }))
-      deployment
+      grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.12";
+      grid.publicIPv4 = "192.168.56.22";
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = "19.09";
+    };
   };
 
   storage2 = {
     imports = [
       gridlib.storage
-      (gridlib.hardware-virtual ({ publicIPv4 = "192.168.67.23"; }))
-      (gridlib.customize-storage (config // {
-        monitoringvpnIPv4 = "172.23.23.13";
-        stateVersion = "19.09";
-      }))
-      deployment
+      grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.13";
+      grid.publicIPv4 = "192.168.56.23";
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = "19.09";
+    };
   };
 
   monitoring = {
     imports = [
       gridlib.monitoring
-      (gridlib.hardware-virtual ({ publicIPv4 = "192.168.67.24"; }))
-      (gridlib.customize-monitoring {
-        inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
-        inherit (config) domain publicKeyPath privateKeyPath sshUsers letsEncryptAdminEmail;
-        googleOAuthClientID = config.monitoringGoogleOAuthClientID;
-        monitoringvpnIPv4 = "172.23.23.1";
-        stateVersion = "19.09";
-      })
-      deployment
+      grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.1";
+      grid.publicIPv4 = "192.168.56.24";
+      grid.monitoring = {
+        inherit paymentExporterTargets blackboxExporterHttpsTargets;
+        inherit (grid-config) monitoringDomains;
+        googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+        enableZulipAlert = false;
+      };
+      system.stateVersion = "19.09";
+    };
   };
 
   # TBD: derive these automatically:
-  hostsMap = {
-    "172.23.23.1"  = [ "monitoring" "monitoring.monitoringvpn" ];
-    "172.23.23.11" = [ "payments" "payments.monitoringvpn" ];
-    "172.23.23.12" = [ "storage1" "storage1.monitoringvpn" ];
-    "172.23.23.13" = [ "storage2" "storage2.monitoringvpn" ];
-  };
-  vpnClientIPs = [ "172.23.23.11" "172.23.23.12" "172.23.23.13" ];
-  nodeExporterTargets = [ "monitoring" "payments" "storage1" "storage2" ];
-  paymentExporterTargets = [ "payments" ];
+  paymentExporterTargets = [ "payments.monitoringvpn" ];
+  blackboxExporterHttpsTargets = [
+    # "https://private.storage/"
+    # "https://payments.private.storage/"
+  ];
 
 in {
   network = {
     description = "PrivateStorage.io LocalDev Grid";
+    inherit (gridlib) pkgs;
   };
   inherit payments monitoring storage1 storage2;
 }

morph/grid/local/private-keys/README.rst

@@ -11,7 +11,8 @@ You can find more information about some of these secrets in ``ops/generating-ke
 deploy_key
 ----------
 
-This is an SSH private key which will be authorized to trigger a deployment update on the deployment hosts themselves.
+This SSH private key authenticates an account used by the continuous deployment system.
+Each node authorizes that account to trigger a deployment update on itself. 
 The corresponding SSH public key is kept in the ``public-keys`` location.
 
 grafana-admin.password
@@ -19,6 +20,20 @@ grafana-admin.password
 
 This is the initial admin password for the Grafana web admin on the monitoring host.
 
+grafana-slack-url
+-----------------
+
+This file is read by Grafana's systemd service to set an environment variable with a secret Slack WebHook URL to post alerts to.
+The only line in the file should be the secret URL.
+Use the url from `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ or get a new secret URL for your Slack channel at https://www.slack.com/apps/A0F7XDUAZ.
+
+grafana-zulip-url
+-----------------
+
+This file should contain a single line with the secret Zulip alerting Webhook Bot URL.
+The URLs for Staging and Production are both stored in 1Password.
+See `https://zulip.com/integrations/doc/grafana`_ for documentation and ``grid/local/private-keys/grafana-zulip-url`` for an example.
+
 stripe.secret
 -------------
 

morph/grid/local/private-keys/borgbackup.passphrase

+The most interesting passphrase in the world.

morph/grid/local/private-keys/borgbackup.ssh-key

+-----BEGIN OPENSSH PRIVATE KEY-----
+ratatatratatatratatatratatatratatatratatatratatatratatatratatatratatat
+ratatatratatatratatatratatatratatatratatatratatatratatatratatatratatat
+ratatatratatatratatatratatatratatatratatatratatatratatatratatatratatat
+ratatatratatatratatatratatatratatatratatatratatatratatatratatatratatat
+ratatatratatatratatatratatatratatatratatatc=
+-----END OPENSSH PRIVATE KEY-----

morph/grid/local/private-keys/grafana-slack-url

+https://hooks.slack.com/services/x/y/z
+

morph/grid/local/private-keys/grafana-zulip-url

+https://yourZulipDomain.zulipchat.com/api/v1/external/grafana?api_key=abcdefgh&stream=stream%20name&topic=your%20topic

morph/grid/local/private-keys/stripe.webhook-secret

+whsec_12121212121212121212121212121212121212

morph/grid/local/public-keys/borgbackup/storage1.repopath

+abc123de@abc123de.repo.borgbase.com:repo

morph/grid/local/public-keys/borgbackup/storage2.repopath

+vwx789yz@vwx789yz.repo.borgbase.com:repo

morph/grid/local/public-keys/users.nix

-# Add your public key. Example:
-# let key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la";
-let key = undefined;
-in { "root" = key; "vagrant" = key; }

morph/grid/local/public-keys/users.nix.example

+let
+# Add your public key. Example:
+# key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la";
+# You can use the following to get key from the local machine.
+# key = builtins.readFile ~/.ssh/id_ed25519.pub;
+  key = undefined;
+  keys = [key]
+in {
+  "root" = keys;
+  "vagrant" = keys;
+}

morph/grid/production/config.json

@@ -5,15 +5,16 @@
 , "monitoringvpnEndpoint": "monitoring.private.storage:51820"
 , "passValue": 1000000
 , "issuerDomains": [
-    "payments.privatestorage.io"
-  , "payments.private.storage"
+    "payments.private.storage"
+  , "payments.privatestorage.io"
+  ]
+, "monitoringDomains": [
+    "monitoring.private.storage"
+  , "monitoring.privatestorage.io"
   ]
 , "letsEncryptAdminEmail": "jean-paul@privatestorage.io"
 , "allowedChargeOrigins": [
-    "https://privatestorage.io"
-  , "https://www.privatestorage.io"
-  , "https://private.storage"
-  , "https://www.private.storage"
+    "https://private.storage"
   ]
 , "monitoringGoogleOAuthClientID": "802959152038-klpkk38sfnqmknn1ucg7pvs4hcc2k8ae.apps.googleusercontent.com"
 }

morph/grid/production/grid.nix

 # See morph/grid/local/grid.nix for additional commentary.
 let
-  pkgs = import <nixpkgs> { };
-
   gridlib = import ../../lib;
-  rawConfig = pkgs.lib.trivial.importJSON ./config.json;
-  config = rawConfig // {
-    sshUsers = import ./public-keys/users.nix;
+  grid-config = builtins.fromJSON (builtins.readFile ./config.json);
 
+  # Module with per-grid configuration
+  grid-module = {config, ...}: {
+    imports = [
+      gridlib.base
+      # Allow us to remotely trigger updates to this system.
+      ../../../nixos/modules/deployment.nix
+      # Give it a good SSH configuration.
+      ../../../nixos/modules/ssh.nix
+    ];
+    services.private-storage.sshUsers = import ./public-keys/users.nix;
+    networking.domain = grid-config.domain;
     # Convert relative paths to absolute so library code can resolve names
     # correctly.
-    publicKeyPath = toString ./. + "/${rawConfig.publicKeyPath}";
-    privateKeyPath = toString ./. + "/${rawConfig.privateKeyPath}";
+    grid = {
+      publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
+      privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
+      inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
     };
-
     # Configure deployment management authorization for all systems in the grid.
-  deployment = {
     services.private-storage.deployment = {
-      authorizedKey = builtins.readFile "${config.publicKeyPath}/deploy_key.pub";
+      authorizedKey = builtins.readFile "${config.grid.publicKeyPath}/deploy_key.pub";
       gridName = "production";
     };
   };
@@ -25,32 +32,38 @@ let
     imports = [
       gridlib.issuer
       gridlib.hardware-aws
-      (gridlib.customize-issuer (config // {
-        monitoringvpnIPv4 = "172.23.23.11";
-      }))
-      deployment
+      grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.11";
+      grid.issuer = {
+        inherit (grid-config) issuerDomains allowedChargeOrigins;
+      };
+    };
   };
 
   monitoring = {
     imports = [
       gridlib.monitoring
       gridlib.hardware-aws
-      (gridlib.customize-monitoring {
-        inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
-        inherit (config) domain publicKeyPath privateKeyPath sshUsers letsEncryptAdminEmail;
-        googleOAuthClientID = config.monitoringGoogleOAuthClientID;
-        monitoringvpnIPv4 = "172.23.23.1";
-        stateVersion = "19.09";
-      })
-      deployment
+      grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.1";
+      grid.monitoring = {
+        inherit paymentExporterTargets blackboxExporterHttpsTargets;
+        inherit (grid-config) monitoringDomains;
+        googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+        enableZulipAlert = true;
+      };
+      system.stateVersion = "19.09";
+    };
   };
 
   defineStorageNode = name: { vpnIP, stateVersion }:
   let
-    nodecfg = import "${./.}/${name}-config.nix";
-    hardware ="${./.}/${name}-hardware.nix";
+    nodecfg = import (./. + "/${name}-config.nix");
+    hardware = (./. + "/${name}-hardware.nix");
   in {
     imports = [
       # Get some of the very lowest-level system configuration for this
@@ -61,24 +74,33 @@ let
       # Slightly awkwardly, enable some of our hardware / network / bootloader options.
       ../../../nixos/modules/100tb.nix
 
+      # At least some of our storage nodes utilize MegaRAID storage controllers.
+      # Monitor their array status.
+      ../../../nixos/modules/monitoring/exporters/megacli2prom.nix
+
       # Get all of the configuration that is common across all storage nodes.
       gridlib.storage
-
-      # Then customize the storage system a little bit based on this node's particulars.
-      (gridlib.customize-storage (config // nodecfg // {
-        monitoringvpnIPv4 = vpnIP;
-        inherit stateVersion;
-      }))
-
       # Also configure deployment management authorization
-      deployment
+      grid-module
     ];
 
+    config = {
+      grid.monitoringvpnIPv4 = vpnIP;
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = stateVersion;
+
       # And supply configuration for those hardware / network / bootloader
       # options.  See the 100tb module for handling of this value.  The module
       # name is quoted because `1` makes `100tb` look an awful lot like a
       # number.
       "100tb".config = nodecfg;
+
+      # Enable statistics gathering for MegaRAID cards.
+      # TODO would be nice to enable only on machines that have such a device.
+      services.private-storage.monitoring.exporters.megacli2prom.enable = true;
+    };
   };
 
   # Define all of the storage nodes for this grid.
@@ -90,38 +112,22 @@ let
     storage005 = { vpnIP = "172.23.23.25"; stateVersion = "19.03"; };
   };
 
-  # TBD: derive these automatically:
-  hostsMap = {
-    "172.23.23.1"  = [ "monitoring" "monitoring.monitoringvpn" ];
-    "172.23.23.11" = [   "payments"   "payments.monitoringvpn" ];
-    "172.23.23.21" = [ "storage001" "storage001.monitoringvpn" ];
-    "172.23.23.22" = [ "storage002" "storage002.monitoringvpn" ];
-    "172.23.23.23" = [ "storage003" "storage003.monitoringvpn" ];
-    "172.23.23.24" = [ "storage004" "storage004.monitoringvpn" ];
-    "172.23.23.25" = [ "storage005" "storage005.monitoringvpn" ];
-  };
-  vpnClientIPs = [
-    "172.23.23.11"
-    "172.23.23.21"
-    "172.23.23.22"
-    "172.23.23.23"
-    "172.23.23.24"
-    "172.23.23.25"
-  ];
-  nodeExporterTargets = [
-    "monitoring"
-    "payments"
-    "storage001"
-    "storage002"
-    "storage003"
-    "storage004"
-    "storage005"
+  paymentExporterTargets = [ "payments.monitoringvpn" ];
+  blackboxExporterHttpsTargets = [
+    "https://private.storage/"
+    "https://www.private.storage/"
+    "https://privatestorage.io/"
+    "https://www.privatestorage.io/"
+    "https://payments.private.storage/"
+    "https://payments.privatestorage.io/"
+    "https://monitoring.private.storage/"
+    "https://monitoring.privatestorage.io/"
   ];
-  paymentExporterTargets = [ "payments" ];
 
 in {
   network = {
     description = "PrivateStorage.io Production Grid";
+    inherit (gridlib) pkgs;
   };
   inherit payments;
   inherit monitoring;

morph/grid/production/public-keys/borgbackup/storage001.repopath

+gye1flhy@gye1flhy.repo.borgbase.com:repo

morph/grid/production/public-keys/borgbackup/storage002.repopath

+l4642x1g@l4642x1g.repo.borgbase.com:repo

morph/grid/production/public-keys/borgbackup/storage003.repopath

+c7400xl6@c7400xl6.repo.borgbase.com:repo

morph/grid/production/public-keys/borgbackup/storage004.repopath

+sbn13vf8@sbn13vf8.repo.borgbase.com:repo

morph/grid/production/public-keys/borgbackup/storage005.repopath

+wg8x4po7@wg8x4po7.repo.borgbase.com:repo

morph/grid/production/public-keys/users.nix

-let key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGN4VQm3BIQKEFTw6aPrEwNuShf640N+Py2LOKznFCRT exarkun@bottom";
-in { "root" = key; "jcalderone" = key; }
+let
+  flo = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII6EUU/KNDr7y3m5OVWBZAuPiMJ4us3YOBEhxpG29yPN flo@la"];
+  last-resort = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1hy9mPkJI+7mY2Uq6CLpuFMMLOTfiY2sRJHwpihgRt cardno:26 269 859 - Last Resort A-Key"
+                  "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPYMUVNuWr2y+FL1GxW6S6jb3BWYhbzJ2zhvQVKu2ll cardno:23 845 763 - Last Resort C-key"
+                ];
+in {
+  "root" = flo ++ last-resort;
+  inherit  flo    last-resort;
+}

morph/grid/production/storage001-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, ... }:
 
 {
@@ -12,7 +10,7 @@
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/f72c1f46-6723-45bf-9ef7-92f31cc37589";
@@ -38,6 +36,6 @@
   } ];
 
 
-  nix.maxJobs = lib.mkDefault 24;
+  nix.settings.max-jobs = lib.mkDefault 24;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }