Merge pull request #15 from PrivateStorageio/privatestorageops#145.deploy-v1-production-grid

Deploy v1 production grid

morph/grid.nix

@@ -16,9 +16,30 @@ import ./make-grid.nix {
     # doesn't specify one.
     #
     # The names must be unique!
-
-    # Pass the whole grid configuration to the module and let it take what it
-    # wants.
-    "storage000" = import ./storage000.nix cfg;
+    "storage001" = import ./make-storage.nix ({
+        cfg = import ./storage001-config.nix;
+        hardware = ./storage001-hardware.nix;
+        stateVersion = "19.09";
+    } // cfg);
+    "storage002" = import ./make-storage.nix ({
+        cfg = import ./storage002-config.nix;
+        hardware = ./storage002-hardware.nix;
+        stateVersion = "19.09";
+    } // cfg);
+    "storage003" = import ./make-storage.nix ({
+        cfg = import ./storage003-config.nix;
+        hardware = ./storage003-hardware.nix;
+        stateVersion = "19.09";
+    } // cfg);
+    "storage004" = import ./make-storage.nix ({
+        cfg = import ./storage004-config.nix;
+        hardware = ./storage004-hardware.nix;
+        stateVersion = "19.09";
+    } // cfg);
+    "storage005" = import ./make-storage.nix ({
+        cfg = import ./storage005-config.nix;
+        hardware = ./storage005-hardware.nix;
+        stateVersion = "19.03";
+    } // cfg);
   };
 }

morph/storage000.nix → morph/make-storage.nix

-let
-  # Get the configuration that's specific to this node.
-  cfg = import ./storage000-config.nix;
-in
-# Define the function that defines the node.  Accept the public storage server
-# port argument so we can configure Tahoe-LAFS with it.  Accept but ignore any
-# other arguments.
-{ publicStoragePort, ristrettoSigningKeyPath, ... }: rec {
+# Define the function that defines the node.
+{ cfg                        # Get the configuration that's specific to this node.
+, hardware                   # The path to the hardware configuration for this node.
+, publicStoragePort          # The storage port number on which to accept connections.
+, ristrettoSigningKeyPath    # The *local* path to the Ristretto signing key file.
+, stateVersion               # The value for system.stateVersion on this node.
+                             # This value determines the NixOS release with
+                             # which your system is to be compatible, in order
+                             # to avoid breaking some software such as
+                             # database servers. You should change this only
+                             # after NixOS release notes say you should.
 
+}: rec {
   deployment = {
     secrets = {
       "ristretto-signing-key" = {
@@ -26,13 +30,13 @@ in
   # Any extra NixOS modules to load on this server.
   imports = [
     # Include the results of the hardware scan.
-    ./storage000-hardware.nix
+    hardware
     # Configure it as a system operated by 100TB.
     ../nixos/modules/100tb.nix
     # Bring in our module for configuring the Tahoe-LAFS service and other
     # Private Storage-specific things.
     ../nixos/modules/private-storage.nix
-   ];
+  ];
 
   # Pass the configuration specific to this host to the 100TB module to be
   # expanded into a complete system configuration.  See the 100tb module for
@@ -54,9 +58,5 @@ in
     ristrettoSigningKeyPath = deployment.secrets.ristretto-signing-key.destination;
   };
 
-  # This value determines the NixOS release with which your system is to be
-  # compatible, in order to avoid breaking some software such as database
-  # servers. You should change this only after NixOS release notes say you
-  # should.
-  system.stateVersion = "19.03"; # Did you read the comment?
+  system.stateVersion = stateVersion;
 }

morph/storage001-config.nix

+{ "hostId" = "00000001";
+  "interface" = "eno1";
+  "publicIPv4" = "176.113.72.38";
+  "prefixLength" = 30;
+  "gateway" = "176.113.72.37";
+  "gatewayInterface" = "eno1";
+  "grubDeviceID" = "wwn-0x5000cca25cc08d27";
+  "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
+}

morph/storage001-hardware.nix

+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports =
+    [ <nixpkgs/nixos/modules/installer/scan/not-detected.nix>
+    ];
+
+  boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "ehci_pci" "megaraid_sas" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/f72c1f46-6723-45bf-9ef7-92f31cc37589";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/cb3a16e1-d811-4659-be42-15e5e35cd37a";
+      fsType = "ext4";
+    };
+
+  # Manually created using:
+  #   zpool create -f -m legacy -o ashift=12 root raidz /dev/disk/by-id/{wwn-0x5000cca25cc06670,wwn-0x5000cca244c94a41,wwn-0x5000cca24ceb8bb4,wwn-0x500003983bd01315,wwn-0x500003983bd812b2,wwn-0x5000cca25cc09d0c,wwn-0x5000c500921dd6b6}
+  fileSystems."/storage" =
+    { device = "root";
+      fsType = "zfs";
+    };
+
+  swapDevices =
+    [ { device = "/dev/disk/by-uuid/f986a811-4912-4e9a-8bc3-01cb6926c4c6"; }
+    ];
+
+  nix.maxJobs = lib.mkDefault 24;
+  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
+}

morph/storage002-config.nix

+{ "hostId" = "00000002";
+  "interface" = "eno1";
+  "publicIPv4" = "37.120.214.110";
+  "prefixLength" = 30;
+  "gateway" = "37.120.214.109";
+  "gatewayInterface" = "eno1";
+  "grubDeviceID" = "wwn-0x5000cca25dccb3dc";
+  "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
+}

morph/storage002-hardware.nix

+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports =
+    [ <nixpkgs/nixos/modules/installer/scan/not-detected.nix>
+    ];
+
+  boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "ehci_pci" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/9721366a-7f75-45b6-908a-cde68ed82295";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 24;
+  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
+}

morph/storage003-config.nix

+{ "hostId" = "00000003";
+  "interface" = "eno1";
+  "publicIPv4" = "45.83.89.186";
+  "prefixLength" = 30;
+  "gateway" = "45.83.89.185";
+  "gatewayInterface" = "eno1";
+  "grubDeviceID" = "wwn-0x5000cca248c31469";
+  "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
+}

morph/storage003-hardware.nix

+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports =
+    [ <nixpkgs/nixos/modules/installer/scan/not-detected.nix>
+    ];
+
+  boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "ehci_pci" "megaraid_sas" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/daf0b345-97da-46bc-b9df-500d771ec375";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/a1843705-f4e9-4805-924c-19f464d23da7";
+      fsType = "ext4";
+    };
+
+  # Manually created using:
+  #   zpool create -f -m legacy -o ashift=12 root raidz /dev/disk/by-id/{wwn-0x5000cca249d43969,wwn-0x5000cca248dd1f83,wwn-0x5000cca249d44a67,wwn-0x5000cca249d46730,wwn-0x5000cca25dcc719c,wwn-0x5000cca25dcc0241,wwn-0x5000cca24ac2b2df}
+  fileSystems."/storage" =
+    { device = "root";
+      fsType = "zfs";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 24;
+  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
+}

morph/storage004-config.nix

+{ "hostId" = "00000004";
+  "interface" = "eno1";
+  "publicIPv4" = "87.101.93.198";
+  "prefixLength" = 30;
+  "gateway" = "87.101.93.197";
+  "gatewayInterface" = "eno1";
+  "grubDeviceID" = "wwn-0x5000cca249d45533";
+  "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
+}

morph/storage004-hardware.nix

+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports =
+    [ <nixpkgs/nixos/modules/installer/scan/not-detected.nix>
+    ];
+
+  boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "ehci_pci" "megaraid_sas" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/d628122e-05d9-4212-b6a5-4b9516d85dbe";
+      fsType = "ext4";
+    };
+
+  # Manually created using:
+  #   zpool create -f -m legacy -o ashift=12 root raidz /dev/disk/by-id/{wwn-0x5000cca25cc0b6f9,wwn-0x5000cca25cc073af,wwn-0x5000cca25dcca3b5,wwn-0x5000cca25cc0addc,wwn-0x5000cca25cc08772,wwn-0x5000cca25dcc6f5f,wwn-0x5000cca25dcc4491}
+  fileSystems."/storage" =
+    { device = "root";
+      fsType = "zfs";
+    };
+
+  swapDevices = [ ];
+
+  nix.maxJobs = lib.mkDefault 32;
+  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
+}

morph/storage005-config.nix

+{ "hostId" = "00000005";
+  "interface" = "eno1";
+  "publicIPv4" = "193.148.18.206";
+  "prefixLength" = 30;
+  "gateway" = "193.148.18.205";
+  "gatewayInterface" = "eno1";
+  "grubDeviceID" = "wwn-0x5000cca25dcc78b5";
+  "rootPublicKey" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
+}

morph/storage005-hardware.nix

+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, ... }:
+
+{
+  imports =
+    [ <nixpkgs/nixos/modules/installer/scan/not-detected.nix>
+    ];
+
+  boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "ehci_pci" "megaraid_sas" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/29d379b9-a3e2-4efd-8ac1-d55bb5751dc2";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/C95C-B700";
+      fsType = "vfat";
+    };
+
+  swapDevices =
+    [ { device = "/dev/disk/by-uuid/4800c196-ee81-43ea-8e48-ae7e222524de"; }
+    ];
+
+  nix.maxJobs = lib.mkDefault 32;
+  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
+}

nixos/modules/100tb.nix

@@ -32,6 +32,11 @@ let
   # values.  This value is in the `let` to make the code below a little easier
   # to read.  See below where we use it.
   options = {
+    hostId = lib.mkOption
+    { type = lib.types.str;
+      example = lib.literalExample "abcdefab";
+      description = "The 32-bit host ID of the machine, formatted as 8 hexadecimal characters.";
+    };
     interface = lib.mkOption
     { type = lib.types.str;
       example = lib.literalExample "eno0";
@@ -113,6 +118,7 @@ in {
       cfg.rootPublicKey
     ];
 
+    networking.hostId = cfg.hostId;
     networking.dhcpcd.enable = false;
     networking.interfaces = {
       "${cfg.interface}".ipv4.addresses = [

nixos/modules/private-storage.nix

@@ -110,6 +110,8 @@ in
         };
         storage =
         { enabled = true;
+          # Put the storage where we have a lot of space configured.
+          storage_dir = "/storage";
           # Turn on our plugin.
           plugins = "privatestorageio-zkapauthz-v1";
         };

nixos/pkgs/zkapissuer-repo.nix

 let
-  pkgs = import <nixpkgs> { };
+  pkgs = import <nixpkgs> {};
 in
   pkgs.fetchFromGitHub {
     owner = "PrivateStorageio";
     repo = "PaymentServer";
-    rev = "94fb418962abee71fa97c09c76e85ccc13cf4c1e";
-    sha256 = "15v71hqhs3rd8c77igbzbi2lbvrb6yyshasq3ijs51w7pwp10dac";
-  }
+    rev = "028d26152eba4f034aba405caa17627a764c2bbe";
+    sha256 = "06hdln97r2ign7phf661wlzh3z06bk9906lvc0gm3lh1pa23d3gb";
+  }
\ No newline at end of file