.gitlab-ci.yml

@@ -5,20 +5,26 @@ default:
     - "nixos"
     - "shell"
 
+variables:
+  # https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts
+  GET_SOURCES_ATTEMPTS: 10
+
 docs:
   stage: "build"
   script:
-    - "nix-build docs.nix"
-    - "cp --recursive --no-preserve=mode result/docs/. docs/build/"
+    - "nix-build --attr docs --out-link result-docs"
+    # GitLab wants to lchown artifacts.  It can't do that to store paths.  Get
+    # a copy of the docs outside of the store.
+    - "cp --recursive --no-preserve=mode ./result-docs/docs ./docs-build/"
   artifacts:
     paths:
-      - "docs/build/"
+      - "./docs-build/"
     expose_as: "documentation"
 
 unit-tests:
   stage: "test"
   script:
-    - "nix-shell --run 'nix-build nixos/unit-tests.nix' && cat result"
+    - "nix-build --attr unit-tests && cat result"
 
 .morph-build: &MORPH_BUILD
   stage: "test"
@@ -68,7 +74,7 @@ system-tests:
   stage: "test"
   timeout: "3 hours"
   script:
-    - "nix-shell --run 'nix-build nixos/system-tests.nix'"
+    - "nix-build --attr system-tests"
 
 # A template for a job that can update one of the grids.
 .update-grid: &UPDATE_GRID

DEPLOYMENT-NOTES.rst

 Deployment notes
 ================
 
+- 2021-10-12 The secret in ``private-keys/grafana-slack-url`` needs to be changed to remove the ``SLACKURL=`` prefix.
+
 - 2021-09-30 `Enable alerting <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/185>`_ needs a secret in ``private-keys/grafana-slack-url`` looking like the template in ``morph/grid/local/private-keys/grafana-slack-url`` and pointing to the secret API endpoint URL saved in `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ (or create a new secret URL at https://www.slack.com/apps/A0F7XDUAZ).
 
 - 2021-09-07 `Manage access to payment metrics <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/merge_requests/146>`_ requires moving and chown'ing the PaymentServer database on the ``payments`` host::

ci-tools/vulnerability-scan

@@ -32,6 +32,12 @@ else
 fi
 '
 
+# The version (1.9.6) of vulnix in nixos-21.05 incorrectly collapses
+# derivations with the same name+version, but different sets of patches
+# applied. Therefore, we use a recent nixos-unstable version that has a newer
+# version of vulnix included.
+export NIX_PATH=nixpkgs=https://api.github.com/repos/NixOS/nixpkgs/tarball/ee084c02040e864eeeb4cf4f8538d92f7c675671
+
 # vulnix exits with an error status if there are vulnerabilities.  We told
 # GitLab to allow this by setting `allow_failure` to true in the GitLab CI
 # config.  vulnix exit status indicates what vulnix thinks happened.  If we

default.nix

+{ pkgs ? import ./nixpkgs-2105.nix { } }:
+{
+  # Render the project documentation source to some presentation format (ie,
+  # html) with Sphinx.
+  docs = pkgs.callPackage ./docs.nix { };
+
+  # Run some system integration tests in VMs covering some of the software
+  # we're integrating (ie, application functionality).
+  system-tests = pkgs.callPackage ./nixos/system-tests.nix { };
+
+  # Run some unit tests of the Nix that ties all of these things together (ie,
+  # PrivateStorageio-internal library functionality).
+  unit-tests = pkgs.callPackage ./nixos/unit-tests.nix { };
+}

docs.nix

-{ pkgs ? import ./nixpkgs-2105.nix { } }:
-pkgs.callPackage ./privatestorageio.nix { }
+{ stdenv, lib, graphviz, plantuml, python3, sphinx }:
+let
+  pyenv = python3.withPackages (ps: [ ps.sphinx ps.sphinxcontrib_plantuml ]);
+in
+stdenv.mkDerivation rec {
+  version = "0.0";
+  name = "privatestorageio-${version}";
+  src = lib.cleanSource ./.;
+
+  phases = [ "unpackPhase" "buildPhase" ];
+
+  depsBuildBuild = [
+    graphviz
+    plantuml
+  ];
+
+  buildPhase = ''
+    ${pyenv}/bin/sphinx-build -W docs/ $out/docs
+  '';
+}

docs/source/conf.py → docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'PrivateStorageio'
-copyright = '2019, PrivateStorage.io, LLC'
+copyright = '2021, PrivateStorage.io, LLC'
 author = 'PrivateStorage.io, LLC'
 
 # The short X.Y version
@@ -38,8 +38,10 @@ release = '0.0'
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
+
 extensions = [
     "sphinx.ext.graphviz",
+    "sphinxcontrib.plantuml",
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/source/dev/README.rst → docs/dev/README.rst

@@ -16,13 +16,13 @@ The test system uses `Nix`_ which must be installed before any tests can be run.
 
 Unit tests are run using this command::
 
-  $ nix-build nixos/unit-tests.nix
+  $ nix-build --attr unit-tests
 
 Unit tests are also run on CI.
 
 The system tests are run using this command::
 
-  $ nix-build nixos/system-tests.nix
+  $ nix-build --attr system-tests
 
 The system tests boot QEMU VMs which prevents them from running on CI at this time.
 The build requires > 10 GB of disk space,
@@ -32,7 +32,7 @@ try `raising the number of retries <https://whetstone.privatestorage.io/privates
 
 It is also possible go through the testing script interactively - useful for debugging::
 
-  $ nix-build -A private-storage.driver nixos/system-tests.nix
+  $ nix-build --attr system-tests.private-storage.driver
 
 This will give you a result symlink in the current directory.
 Inside that is bin/nixos-test-driver which gives you a kind of REPL for interacting with the VMs.
@@ -56,7 +56,6 @@ That will update ``nixpkgs-2015.json`` to the latest release on the nixos-21.05
 To update the channel, the script will need to be updated,
 along with the filenames that have the channel in them.
 
-
 Gitlab Repositories
 ```````````````````
 To update the version of packages we import from gitlab, run:
@@ -70,15 +69,119 @@ The command uses branch and repository owner specified in the ``repo.json`` file
 but you can override them by passing the ``--branch`` or ``-owner`` arguments to the command.
 A specific revision can also be pinned, by passing ``-rev``.
 
-
-Architecture overview
----------------------
-
-.. graphviz:: architecture-overview.dot
-
+Interactions
+------------
+
+Storage-Time Purchase (ie Payment)
+``````````````````````````````````
+
+.. uml::
+
+   actor User as User
+   participant GridSync
+   participant ZKAPAuthorizer
+   database    ZKAPAuthzDB as "ZKAPAuthorizer"
+   participant Browser
+   participant PaymentServer as "Payment Server"
+   database    PaymentServerDB as "Payment Server"
+   participant WebServer as "Web Server"
+   participant Stripe
+
+   User           -> GridSync       : buy storage-time
+   activate User
+   GridSync       -> GridSync       : generate voucher
+   GridSync       -> ZKAPAuthorizer : redeem voucher
+   activate ZKAPAuthorizer
+   ZKAPAuthorizer -> ZKAPAuthzDB    : store voucher
+   ZKAPAuthorizer -> GridSync       : acknowledge
+   GridSync       -> Browser        : open payment page
+
+   loop until redeemed
+       GridSync       -> ZKAPAuthorizer : query voucher state
+       ZKAPAuthorizer -> GridSync       : not paid
+   end
+
+   Browser       -> WebServer  : request payment form
+   WebServer     -> Browser    : payment form
+   Browser       -> User       : Payment form displayed
+   activate User
+   User          -> Browser    : Submit payment details
+   Browser       -> Stripe     : Submit payment details
+
+   alt payment details accepted
+       Stripe         -> Browser         : details okay, return card token
+       Browser        -> PaymentServer   : create charge using card token
+       PaymentServer  -> Stripe          : charge card using token
+       note left: the user has now paid for the service
+       Stripe         -> PaymentServer   : acknowledge
+       PaymentServer  -> PaymentServerDB : store voucher paid state
+   else payment details rejected
+       Stripe    -> Browser        : payment failure
+   end
+
+   Browser       -> User       : payment processing results displayed
+   deactivate User
+
+   group repeat for each redemption group
+       ZKAPAuthorizer -> ZKAPAuthzDB    : generate and store random tokens
+       ZKAPAuthorizer -> PaymentServer  : redeem voucher with blinded tokens
+       PaymentServer  -> ZKAPAuthorizer : return signatures for blinded tokens
+       ZKAPAuthorizer -> ZKAPAuthzDB    : store unblinded signatures for tokens
+       note right: the user has now been authorized to use the service
+   end
+   deactivate ZKAPAuthorizer
+
+   loop until redeemed
+       GridSync       -> ZKAPAuthorizer : query voucher state
+       ZKAPAuthorizer -> GridSync       : fully redeemed
+   end
+
+   GridSync           -> User           : storage-time available displayed
+   deactivate User
+
+Storage-Time Spending (ie Use)
+``````````````````````````````
+
+.. uml::
+
+   participant MagicFolder
+   participant TahoeLAFS as "Tahoe-LAFS"
+   participant ZKAPAuthorizer
+   database    ZKAPAuthzDB as "ZKAPAuthorizer"
+   participant StorageNode as "Storage Node"
+   participant SpendingService as "Spending Service"
+
+   [-> MagicFolder: upload triggered
+   activate MagicFolder
+
+   MagicFolder -> TahoeLAFS : store some data
+   activate TahoeLAFS
+
+   TahoeLAFS -> ZKAPAuthorizer : store some data
+   activate ZKAPAuthorizer
+
+   loop until tokens accepted
+       ZKAPAuthorizer <- ZKAPAuthzDB : load some tokens
+       ZKAPAuthorizer -> StorageNode : store some data using these tokens
+       StorageNode -> SpendingService : spend these tokens
+
+       alt spent tokens
+           SpendingService -> StorageNode: already spent, rejected
+           StorageNode -> ZKAPAuthorizer: already spent, rejected
+       else fresh tokens
+           SpendingService -> StorageNode: accepted
+       end
+   end
+
+   StorageNode -> ZKAPAuthorizer: data stored
+   deactivate ZKAPAuthorizer
+   ZKAPAuthorizer -> ZKAPAuthzDB: discard spent tokens
+   ZKAPAuthorizer -> TahoeLAFS: data stored
+   deactivate TahoeLAFS
+   TahoeLAFS -> MagicFolder: data stored
+   deactivate MagicFolder
 
 .. include::
-      ../../../morph/grid/local/README.rst
+      ../../morph/grid/local/README.rst
 
 .. _Nix: https://nixos.org/nix
-

docs/source/ops/README.rst → docs/ops/README.rst

@@ -4,7 +4,7 @@ Administrator documentation
 This contains documentation regarding running PrivateStorageio.
 
 .. include::
-      ../../../morph/README.rst
+      ../../morph/README.rst
 
 .. include::
       monitoring.rst

docs/source/_templates/.gitignore

-!.gitignore

docs/source/dev/architecture-overview.dot

-digraph subscriptions {
-        rankdir=LR
-
-        subgraph cluster_usercontrolled {
-                label = "User Operated"
-                rankdir=LR
-                GridSync                 [label="GridSync",                          shape=circle]
-                Browser                  [label="Browser",                           shape=circle]
-                TahoeLAFS                [label="Tahoe-LAFS",                        shape=circle]
-        }
-
-
-        subgraph cluster_pscontrolled {
-                label = "PrivateStorage.io Operated"
-                rankdir = TB
-                PSWebServer              [label="PrivateStorage.io Web Server",      shape=box]
-                SubscriptionConfigWHPeer [label="Subscription Config Wormhole Peer", shape=box]
-                PaymentServer            [label="Payment Server",                    shape=box]
-                SATIssuer                [label="SAT Issuer",                        shape=box]
-                PSStorageGrid            [label="PrivateStorage.io Storage Grid",    shape=box]
-        }
-
-        User                     [label="User",                              shape=egg]
-        Stripe                   [label="Stripe",                            shape=pentagon]
-
-        User                     -> PSWebServer              [label="1. Get wormhole code",    fontcolor=red, color=red]
-        PSWebServer              -> User                     [label="2. 7-petulant-banana",    fontcolor=blue, color=blue]
-        User                     -> GridSync                 [label="3. 7-petulant-banana",    fontcolor=brown, color=brown]
-        GridSync                 -> SubscriptionConfigWHPeer [label="4. Get configuration",    fontcolor=black, color=black]
-        SubscriptionConfigWHPeer -> GridSync                 [label="5. Grid configuration",   fontcolor=magenta, color=magenta]
-        GridSync                 -> TahoeLAFS                [label="6. Instantiate",          fontcolor=aquamarine3, color=aquamarine3]
-        GridSync                 -> TahoeLAFS                [label="7. Redeem PRN",           fontcolor=crimson, color=crimson]
-        TahoeLAFS                -> PaymentServer            [label="8. Redeem PRN",           fontcolor=crimson, color=crimson]
-        PaymentServer            -> TahoeLAFS                [label="9. Payment required",     fontcolor=gold3, color=gold3]
-        TahoeLAFS                -> GridSync                 [label="10. Payment required",    fontcolor=gold3, color=gold3]
-        GridSync                 -> Browser                  [label="11. Open payment window", fontcolor=gold3, color=gold3]
-        User                     -> Browser                  [label="12. Enter payment info",  fontcolor=blue, color=blue]
-        Browser                  -> Stripe                   [label="13. Submit payment form", fontcolor=brown, color=brown]
-        Stripe                   -> Browser                  [label="14. Payment ok",          fontcolor=black, color=black]
-        Stripe                   -> PaymentServer            [label="15. Payment notification", fontcolor=magenta, color=magenta]
-        GridSync                 -> TahoeLAFS                [label="16. Redeem PRN",             fontcolor=aquamarine3, color=aquamarine3]
-        TahoeLAFS                -> TahoeLAFS                [label="17. Generate blinded tokens",        fontcolor=crimson, color=crimson]
-        TahoeLAFS                -> SATIssuer                [label="18. Redeem PRN, blinded-tokens=xs",  fontcolor=crimson, color=crimson]
-	SATIssuer                -> PaymentServer            [label="19. Check PRN",                      fontcolor=gold3, color=gold3]
-	PaymentServer            -> SATIssuer                [label="20. PRN Valid",                      fontcolor=gold3, color=gold3]
-	SATIssuer                -> TahoeLAFS                [label="21. PRN valid, signed-tokens=ys",    fontcolor=crimson, color=crimson]
-        TahoeLAFS                -> TahoeLAFS                [label="22. Store signed tokens",            fontcolor=crimson, color=crimson]
-        TahoeLAFS                -> GridSync                 [label="23. PRN Redeemed",                   fontcolor=red, color=red]
-        TahoeLAFS                -> PSStorageGrid            [label="24. Use storage, passes=y",          fontcolor=magenta, color=magenta]
-}

morph/grid/local/private-keys/README.rst

@@ -23,7 +23,7 @@ grafana-slack-url
 -----------------
 
 This file is read by Grafana's systemd service to set an environment variable with a secret Slack WebHook URL to post alerts to.
-The only line in the file should be ``SLACKURL=`` with the secret URL.
+The only line in the file should be the secret URL.
 Use the url from `this 1Password entry <https://privatestorage.1password.com/vaults/7flqasy5hhhmlbtp5qozd3j4ga/allitems/cgznskz2oix2tyx5xyntwaos5i>`_ or get a new secret URL for your Slack channel at https://www.slack.com/apps/A0F7XDUAZ.
 
 stripe.secret

morph/grid/local/private-keys/grafana-slack-url

-SLACKURL=https://hooks.slack.com/services/x/y/z
+https://hooks.slack.com/services/x/y/z
 

nixos/modules/issuer.nix

@@ -189,7 +189,7 @@ in {
       extraGroups = [ "keys" ];
     };
 
-    # Open 80 and 443 for the certbot HTTP server and the PaymentServer HTTPS server.
+    # Open 80 and 443 for nginx
     networking.firewall.allowedTCPPorts = [
       80
       443