local-grid.nix

+let
+  pkgs = import ./nixpkgs-2105.nix {
+    config = {};
+    overlays = [];
+  };
+  lib = pkgs.lib;
+in
+  let
+    grid-file = ./morph/grid/local/grid.nix;
+    grid = import grid-file;
+    nodes = lib.mapAttrs
+      (
+        name: node: {
+          imports = [ node "${pkgs.morph.lib}/options.nix" ./secrets.nix ./tailscale.nix ];
+          _file = grid-file;
+          config = {
+            networking.hostName = lib.mkDefault name;
+            deployment.targetHost = lib.mkDefault name;
+            # nixpkgs.pkgs set by pkgs.nixosTest
+            # documentation.nixos.extraModuleSources is unneeded
+            # TODO: _module.args.nodes
+            #   evalConfig { ...,  extraArgs = { nodes = uncheckedNodes ; name = machineName; };
+            virtualisation.msize = 1024 * 16;
+          };
+        }
+      ) (builtins.removeAttrs grid [ "network" ]);
+    test-pkg = pkgs.nixosTest {
+      name = "local-grid";
+      testScript = "";
+      inherit nodes;
+    };
+  in
+    lib.addMetaAttrs { mainProgram = "nixos-run-vms"; }
+      test-pkg.driver

morph/grid/local/.gitignore

 /.vagrant
 /public-keys/users.nix
+/local-config.json
+/local-grid.json

morph/grid/local/config.json

@@ -10,4 +10,5 @@
     "http://localhost:5000"
   ]
 , "monitoringGoogleOAuthClientID": ""
+, "virtualisation": "vagrant"
 }

morph/grid/local/grid.nix

 let
   pkgs = import <nixpkgs> { };
+  lib = pkgs.lib;
 
   gridlib = import ../../lib;
-  grid-config = pkgs.lib.trivial.importJSON ./config.json;
+  local-grid-config = if builtins.pathExists ./local-config.json then
+    lib.importJSON ./local-config.json else {};
+  grid-config = lib.importJSON ./config.json // local-grid-config;
 
   ssh-users = let
     ssh-users-file = ./public-keys/users.nix;
@@ -28,7 +31,7 @@ let
       # Give it a good SSH configuration.
       ../../../nixos/modules/ssh.nix
       # Configure things specific to the virtualisation environment.
-      gridlib.hardware-vagrant
+      gridlib."hardware-${grid-config.virtualisation}"
     ];
     services.private-storage.sshUsers = ssh-users;
 

morph/lib/default.nix

@@ -6,6 +6,7 @@
 
   hardware-aws = import ./issuer-aws.nix;
   hardware-vagrant = import ./hardware-vagrant.nix;
+  hardware-qemu = import ./hardware-qemu.nix;
 
   issuer = import ./issuer.nix;
   customize-issuer = import ./customize-issuer.nix;

morph/lib/hardware-qemu.nix

+{ config, lib, ... }:
+{
+  imports = [
+    # Options for this module are defined in ./hardware-virtual.nix
+    ./hardware-virtual.nix
+  ];
+
+  config = {
+    networking.primaryIPAddress = lib.mkForce config.grid.publicIPv4;
+    networking.interfaces = lib.mkForce {
+      eth1.ipv4.addresses = [
+        {
+          address = config.grid.publicIPv4;
+          prefixLength = 24;
+        }
+      ];
+    };
+  };
+}

morph/lib/hardware-vagrant.nix

-{ config, lib, modulesPath, ... }:
+{ config, modulesPath, ... }:
 {
   imports = [
+    # Options for this module are defined in ./hardware-virtual.nix
+    ./hardware-virtual.nix
     # modulesPath points at the upstream nixos/modules directory.
     "${modulesPath}/virtualisation/vagrant-guest.nix"
   ];
 
-  options.grid = {
-    publicIPv4 = lib.mkOption {
-      type = lib.types.str;
-      description = ''
-        The primary IPv4 address of the virtual machine.
-      '';
-    };
-  };
-
   config = {
     virtualisation.virtualbox.guest.enable = true;
 
@@ -41,3 +34,6 @@
     nix.trustedUsers = [ "@wheel" "root" "vagrant" ];
   };
 }
+    nix.trustedUsers = [ "@wheel" "root" "vagrant" ];
+  };
+};

morph/lib/hardware-virtual.nix

+{ config, lib, modulesPath, ... }:
+{
+  # These options are used by ./hardware-vagrant.nix and ./hardware-qemu.nix
+  options.grid = {
+    publicIPv4 = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The primary IPv4 address of the virtual machine.
+      '';
+    };
+    virtualisation = lib.mkOption {
+      type = lib.types.enum ["vagrant" "qemu"];
+      description = ''
+        The type of virtualisation to use for the local grid.
+      '';
+    };
+  };
+}

secrets.nix

+{ config, lib, pkgs, ... }@args:
+let
+  cfg = config.deployment;
+
+in {
+  # Force secrets to be in /etc/secrets instead of the default.
+  # Most modules default to `/run/keys` which is deleted on boot.
+  # Since the local private keys are in VCS anyway, this is safe.
+  options = {
+    deployment.secrets = lib.mkOption {
+      apply = lib.mapAttrs (k: v: v // {destination = "/etc/secrets/${k}";});
+    };
+  };
+
+  # Actually put the secrets into /etc/secrets
+  config = {
+    environment.etc = lib.mapAttrs'
+        (k: v: lib.nameValuePair "secrets/${k}" {
+          mode = "0444";
+          text = lib.readFile v.source;
+        })
+        cfg.secrets;
+  };
+}

tailscale.nix

+{ pkgs, config, lib, ... }:
+let
+  unstable = import (
+    builtins.fetchTarball {
+      url = "https://github.com/NixOS/nixpkgs/archive/7fa76be757c8d9feaecb984d8bc0b3b13e40545f.tar.gz";
+      sha256 = "0whjrnfz0n1rbr84ykzp1nqnmsn7d9lpj1h4q76w1kxzx50rqg9r";
+    }
+  ) {};
+  tailscale = unstable.tailscale;
+  tailscale-cert-service = certName: conf: {
+    serviceConfig = {
+      type = "oneshot";
+    };
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    requires = [ "tailscaled.service" ];
+    after = [ "tailscaled.service" ];
+    script = let
+      cert-dir = conf.directory;
+      group = conf.group;
+    in
+      ''
+        set -euxo pipefail
+        ${tailscale}/bin/tailscale cert --cert-file ${cert-dir}/fullchain.pem --key-file ${cert-dir}/key.pem ${certName}
+        chgrp ${group} ${cert-dir}/fullchain.pem ${cert-dir}/key.pem
+        chmod g+r ${cert-dir}/fullchain.pem ${cert-dir}/key.pem
+      '';
+  };
+in
+{
+  services.tailscale.enable = true;
+  services.tailscale.package = tailscale;
+  virtualisation.graphics = false;
+  virtualisation.qemu.options = [
+    "-virtfs local,path=/root/persist/${config.networking.hostName},security_model=none,mount_tag=persist"
+  ];
+  virtualisation.fileSystems."/persist" = let
+    cfg = config.virtualisation;
+  in
+    {
+      device = "persist";
+      fsType = "9p";
+      neededForBoot = true;
+      options = [ "trans=virtio" "version=9p2000.L" ] ++ lib.optional (cfg.msize != null) "msize=${toString cfg.msize}";
+    };
+  systemd.tmpfiles.rules = [
+    "d /persist/tailscale 0700 root root -"
+    "d /persist/ssh 0700 root root -"
+  ];
+  services.openssh = {
+    hostKeys = [
+      {
+        path = "/persist/ssh/ssh_host_ed25519_key";
+        type = "ed25519";
+      }
+    ];
+  };
+  systemd.services = lib.mapAttrs' (cert: conf: lib.nameValuePair "acme-${cert}" (lib.mkForce (tailscale-cert-service cert conf))) config.security.acme.certs
+  // {
+    tailscaled = {
+      serviceConfig.BindPaths = [ "/persist/tailscale:/var/lib/tailscale" ];
+      environment = {
+        TS_DEBUG_RESTUN_STOP_ON_IDLE = "true";
+      };
+    };
+  };
+}