diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d6b06fae42f6e738725238fac59617aeb161dfd4..4494a1656146337cf7c64c44eb1081ef172d39e1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,20 +5,26 @@ default:
- "nixos"
- "shell"
+variables:
+ # https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts
+ GET_SOURCES_ATTEMPTS: 10
+
docs:
stage: "build"
script:
- - "nix-build docs.nix"
- - "cp --recursive --no-preserve=mode result/docs/. docs/build/"
+ - "nix-build --attr docs --out-link result-docs"
+ # GitLab wants to lchown artifacts. It can't do that to store paths. Get
+ # a copy of the docs outside of the store.
+ - "cp --recursive --no-preserve=mode ./result-docs/docs ./docs-build/"
artifacts:
paths:
- - "docs/build/"
+ - "./docs-build/"
expose_as: "documentation"
unit-tests:
stage: "test"
script:
- - "nix-shell --run 'nix-build nixos/unit-tests.nix' && cat result"
+ - "nix-build --attr unit-tests && cat result"
.morph-build: &MORPH_BUILD
stage: "test"
@@ -68,7 +74,7 @@ system-tests:
stage: "test"
timeout: "3 hours"
script:
- - "nix-shell --run 'nix-build nixos/system-tests.nix'"
+ - "nix-build --attr system-tests"
# A template for a job that can update one of the grids.
.update-grid: &UPDATE_GRID
diff --git a/default.nix b/default.nix
new file mode 100644
index 0000000000000000000000000000000000000000..6441675a243e22e6154267c656652c8d8575940e
--- /dev/null
+++ b/default.nix
@@ -0,0 +1,14 @@
+{ pkgs ? import ./nixpkgs-2105.nix { } }:
+{
+ # Render the project documentation source to some presentation format (ie,
+ # html) with Sphinx.
+ docs = pkgs.callPackage ./docs.nix { };
+
+ # Run some system integration tests in VMs covering some of the software
+ # we're integrating (ie, application functionality).
+ system-tests = pkgs.callPackage ./nixos/system-tests.nix { };
+
+ # Run some unit tests of the Nix that ties all of these things together (ie,
+ # PrivateStorageio-internal library functionality).
+ unit-tests = pkgs.callPackage ./nixos/unit-tests.nix { };
+}
diff --git a/docs.nix b/docs.nix
index 4c8b230a7eddb462bf47a4c3ee591e64fb3ce1ff..b13c7b58c100553c522cb71912089c6fdbfaed4b 100644
--- a/docs.nix
+++ b/docs.nix
@@ -1,2 +1,20 @@
-{ pkgs ? import ./nixpkgs-2105.nix { } }:
-pkgs.callPackage ./privatestorageio.nix { }
+{ stdenv, lib, graphviz, plantuml, python3, sphinx }:
+let
+ pyenv = python3.withPackages (ps: [ ps.sphinx ps.sphinxcontrib_plantuml ]);
+in
+stdenv.mkDerivation rec {
+ version = "0.0";
+ name = "privatestorageio-${version}";
+ src = lib.cleanSource ./.;
+
+ phases = [ "unpackPhase" "buildPhase" ];
+
+ depsBuildBuild = [
+ graphviz
+ plantuml
+ ];
+
+ buildPhase = ''
+ ${pyenv}/bin/sphinx-build -W docs/ $out/docs
+ '';
+}
diff --git a/docs/build/.gitignore b/docs/_static/.gitignore
similarity index 100%
rename from docs/build/.gitignore
rename to docs/_static/.gitignore
diff --git a/docs/source/_static/logo-ps.svg b/docs/_static/logo-ps.svg
similarity index 100%
rename from docs/source/_static/logo-ps.svg
rename to docs/_static/logo-ps.svg
diff --git a/docs/source/_static/.gitignore b/docs/_templates/.gitignore
similarity index 100%
rename from docs/source/_static/.gitignore
rename to docs/_templates/.gitignore
diff --git a/docs/source/conf.py b/docs/conf.py
similarity index 98%
rename from docs/source/conf.py
rename to docs/conf.py
index 66aa921e2ba799e1b1b4d8e7a778ab07ee07a73b..747a90a8cc039e65fd01c3d598170c001599c1c8 100644
--- a/docs/source/conf.py
+++ b/docs/conf.py
@@ -20,7 +20,7 @@
# -- Project information -----------------------------------------------------
project = 'PrivateStorageio'
-copyright = '2019, PrivateStorage.io, LLC'
+copyright = '2021, PrivateStorage.io, LLC'
author = 'PrivateStorage.io, LLC'
# The short X.Y version
@@ -38,8 +38,10 @@ release = '0.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
+
extensions = [
"sphinx.ext.graphviz",
+ "sphinxcontrib.plantuml",
]
# Add any paths that contain templates here, relative to this directory.
diff --git a/docs/dev/README.rst b/docs/dev/README.rst
new file mode 100644
index 0000000000000000000000000000000000000000..af746a3586d2def59050430f756825a87e8c9fcf
--- /dev/null
+++ b/docs/dev/README.rst
@@ -0,0 +1,187 @@
+Developer documentation
+=======================
+
+Building
+--------
+
+The build system uses `Nix`_ which must be installed before anything can be built.
+Start by setting up the development/operations environment::
+
+ $ nix-shell
+
+Testing
+-------
+
+The test system uses `Nix`_ which must be installed before any tests can be run.
+
+Unit tests are run using this command::
+
+ $ nix-build --attr unit-tests
+
+Unit tests are also run on CI.
+
+The system tests are run using this command::
+
+ $ nix-build --attr system-tests
+
+The system tests boot QEMU VMs which prevents them from running on CI at this time.
+The build requires > 10 GB of disk space,
+and the VMs might be timing out on slow or busy machines.
+If you run into timeouts,
+try `raising the number of retries <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/e8233d2/nixos/modules/tests/run-introducer.py#L55-62>`_.
+
+It is also possible go through the testing script interactively - useful for debugging::
+
+ $ nix-build --attr system-tests.private-storage.driver
+
+This will give you a result symlink in the current directory.
+Inside that is bin/nixos-test-driver which gives you a kind of REPL for interacting with the VMs.
+The kind of `Python in this testScript <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/78881a3/nixos/modules/tests/private-storage.nix#L180>`_ is what you can enter into this REPL.
+Consult the `official documentation on NixOS Tests <https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests>`_ for more information.
+
+Updatings Pins
+--------------
+
+Nixpkgs
+```````
+
+To update the version of NixOS we deploy with, run:
+
+.. code: shell
+
+ nix-shell --run 'update-nixpkgs'
+
+That will update ``nixpkgs-2015.json`` to the latest release on the nixos-21.05 channel.
+
+To update the channel, the script will need to be updated,
+along with the filenames that have the channel in them.
+
+Gitlab Repositories
+```````````````````
+To update the version of packages we import from gitlab, run:
+
+.. code: shell
+
+ nix-shell --command 'tools/update-gitlab nixos/pkgs/<package>/repo.json'
+
+That will update the package to point at the latest version of the project.\
+The command uses branch and repository owner specified in the ``repo.json`` file,
+but you can override them by passing the ``--branch`` or ``-owner`` arguments to the command.
+A specific revision can also be pinned, by passing ``-rev``.
+
+Interactions
+------------
+
+Storage-Time Purchase (ie Payment)
+``````````````````````````````````
+
+.. uml::
+
+ actor User as User
+ participant GridSync
+ participant ZKAPAuthorizer
+ database ZKAPAuthzDB as "ZKAPAuthorizer"
+ participant Browser
+ participant PaymentServer as "Payment Server"
+ database PaymentServerDB as "Payment Server"
+ participant WebServer as "Web Server"
+ participant Stripe
+
+ User -> GridSync : buy storage-time
+ activate User
+ GridSync -> GridSync : generate voucher
+ GridSync -> ZKAPAuthorizer : redeem voucher
+ activate ZKAPAuthorizer
+ ZKAPAuthorizer -> ZKAPAuthzDB : store voucher
+ ZKAPAuthorizer -> GridSync : acknowledge
+ GridSync -> Browser : open payment page
+
+ loop until redeemed
+ GridSync -> ZKAPAuthorizer : query voucher state
+ ZKAPAuthorizer -> GridSync : not paid
+ end
+
+ Browser -> WebServer : request payment form
+ WebServer -> Browser : payment form
+ Browser -> User : Payment form displayed
+ activate User
+ User -> Browser : Submit payment details
+ Browser -> Stripe : Submit payment details
+
+ alt payment details accepted
+ Stripe -> Browser : details okay, return card token
+ Browser -> PaymentServer : create charge using card token
+ PaymentServer -> Stripe : charge card using token
+ note left: the user has now paid for the service
+ Stripe -> PaymentServer : acknowledge
+ PaymentServer -> PaymentServerDB : store voucher paid state
+ else payment details rejected
+ Stripe -> Browser : payment failure
+ end
+
+ Browser -> User : payment processing results displayed
+ deactivate User
+
+ group repeat for each redemption group
+ ZKAPAuthorizer -> ZKAPAuthzDB : generate and store random tokens
+ ZKAPAuthorizer -> PaymentServer : redeem voucher with blinded tokens
+ PaymentServer -> ZKAPAuthorizer : return signatures for blinded tokens
+ ZKAPAuthorizer -> ZKAPAuthzDB : store unblinded signatures for tokens
+ note right: the user has now been authorized to use the service
+ end
+ deactivate ZKAPAuthorizer
+
+ loop until redeemed
+ GridSync -> ZKAPAuthorizer : query voucher state
+ ZKAPAuthorizer -> GridSync : fully redeemed
+ end
+
+ GridSync -> User : storage-time available displayed
+ deactivate User
+
+Storage-Time Spending (ie Use)
+``````````````````````````````
+
+.. uml::
+
+ participant MagicFolder
+ participant TahoeLAFS as "Tahoe-LAFS"
+ participant ZKAPAuthorizer
+ database ZKAPAuthzDB as "ZKAPAuthorizer"
+ participant StorageNode as "Storage Node"
+ participant SpendingService as "Spending Service"
+
+ [-> MagicFolder: upload triggered
+ activate MagicFolder
+
+ MagicFolder -> TahoeLAFS : store some data
+ activate TahoeLAFS
+
+ TahoeLAFS -> ZKAPAuthorizer : store some data
+ activate ZKAPAuthorizer
+
+ loop until tokens accepted
+ ZKAPAuthorizer <- ZKAPAuthzDB : load some tokens
+ ZKAPAuthorizer -> StorageNode : store some data using these tokens
+ StorageNode -> SpendingService : spend these tokens
+
+ alt spent tokens
+ SpendingService -> StorageNode: already spent, rejected
+ StorageNode -> ZKAPAuthorizer: already spent, rejected
+ else fresh tokens
+ SpendingService -> StorageNode: accepted
+ end
+ end
+
+ StorageNode -> ZKAPAuthorizer: data stored
+ deactivate ZKAPAuthorizer
+ ZKAPAuthorizer -> ZKAPAuthzDB: discard spent tokens
+ ZKAPAuthorizer -> TahoeLAFS: data stored
+ deactivate TahoeLAFS
+ TahoeLAFS -> MagicFolder: data stored
+ deactivate MagicFolder
+
+.. include::
+ ../../morph/grid/local/README.rst
+
+.. _Nix: https://nixos.org/nix
diff --git a/docs/source/index.rst b/docs/index.rst
similarity index 100%
rename from docs/source/index.rst
rename to docs/index.rst
diff --git a/docs/source/ops/README.rst b/docs/ops/README.rst
similarity index 86%
rename from docs/source/ops/README.rst
rename to docs/ops/README.rst
index b78e5ef82c0ae4dad88e65e9517f3fc6ec7bdfd2..9ef2837548e272fffadc55130ec1f541d46acafa 100644
--- a/docs/source/ops/README.rst
+++ b/docs/ops/README.rst
@@ -4,7 +4,7 @@ Administrator documentation
This contains documentation regarding running PrivateStorageio.
.. include::
- ../../../morph/README.rst
+ ../../morph/README.rst
.. include::
monitoring.rst
diff --git a/docs/source/ops/generating-keys.rst b/docs/ops/generating-keys.rst
similarity index 100%
rename from docs/source/ops/generating-keys.rst
rename to docs/ops/generating-keys.rst
diff --git a/docs/source/ops/monitoring.rst b/docs/ops/monitoring.rst
similarity index 100%
rename from docs/source/ops/monitoring.rst
rename to docs/ops/monitoring.rst
diff --git a/docs/source/ops/service-dag-to-dashboard-order.dot b/docs/ops/service-dag-to-dashboard-order.dot
similarity index 100%
rename from docs/source/ops/service-dag-to-dashboard-order.dot
rename to docs/ops/service-dag-to-dashboard-order.dot
diff --git a/docs/source/_templates/.gitignore b/docs/source/_templates/.gitignore
deleted file mode 100644
index f935021a8f8a7bd22f9d6703cafa5134bb6a57f8..0000000000000000000000000000000000000000
--- a/docs/source/_templates/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-!.gitignore
diff --git a/docs/source/dev/README.rst b/docs/source/dev/README.rst
deleted file mode 100644
index 1a617e0bf8d753a1e64074b49aa9aae221274796..0000000000000000000000000000000000000000
--- a/docs/source/dev/README.rst
+++ /dev/null
@@ -1,84 +0,0 @@
-Developer documentation
-=======================
-
-Building
---------
-
-The build system uses `Nix`_ which must be installed before anything can be built.
-Start by setting up the development/operations environment::
-
- $ nix-shell
-
-Testing
--------
-
-The test system uses `Nix`_ which must be installed before any tests can be run.
-
-Unit tests are run using this command::
-
- $ nix-build nixos/unit-tests.nix
-
-Unit tests are also run on CI.
-
-The system tests are run using this command::
-
- $ nix-build nixos/system-tests.nix
-
-The system tests boot QEMU VMs which prevents them from running on CI at this time.
-The build requires > 10 GB of disk space,
-and the VMs might be timing out on slow or busy machines.
-If you run into timeouts,
-try `raising the number of retries <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/e8233d2/nixos/modules/tests/run-introducer.py#L55-62>`_.
-
-It is also possible go through the testing script interactively - useful for debugging::
-
- $ nix-build -A private-storage.driver nixos/system-tests.nix
-
-This will give you a result symlink in the current directory.
-Inside that is bin/nixos-test-driver which gives you a kind of REPL for interacting with the VMs.
-The kind of `Python in this testScript <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/78881a3/nixos/modules/tests/private-storage.nix#L180>`_ is what you can enter into this REPL.
-Consult the `official documentation on NixOS Tests <https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests>`_ for more information.
-
-Updatings Pins
---------------
-
-Nixpkgs
-```````
-
-To update the version of NixOS we deploy with, run:
-
-.. code: shell
-
- nix-shell --run 'update-nixpkgs'
-
-That will update ``nixpkgs-2015.json`` to the latest release on the nixos-21.05 channel.
-
-To update the channel, the script will need to be updated,
-along with the filenames that have the channel in them.
-
-
-Gitlab Repositories
-```````````````````
-To update the version of packages we import from gitlab, run:
-
-.. code: shell
-
- nix-shell --command 'tools/update-gitlab nixos/pkgs/<package>/repo.json'
-
-That will update the package to point at the latest version of the project.\
-The command uses branch and repository owner specified in the ``repo.json`` file,
-but you can override them by passing the ``--branch`` or ``-owner`` arguments to the command.
-A specific revision can also be pinned, by passing ``-rev``.
-
-
-Architecture overview
----------------------
-
-.. graphviz:: architecture-overview.dot
-
-
-.. include::
- ../../../morph/grid/local/README.rst
-
-.. _Nix: https://nixos.org/nix
-
diff --git a/docs/source/dev/architecture-overview.dot b/docs/source/dev/architecture-overview.dot
deleted file mode 100644
index cc95fbb74a5b67fc290b80d53ac679a1d1c9d972..0000000000000000000000000000000000000000
--- a/docs/source/dev/architecture-overview.dot
+++ /dev/null
@@ -1,50 +0,0 @@
-digraph subscriptions {
- rankdir=LR
-
- subgraph cluster_usercontrolled {
- label = "User Operated"
- rankdir=LR
- GridSync [label="GridSync", shape=circle]
- Browser [label="Browser", shape=circle]
- TahoeLAFS [label="Tahoe-LAFS", shape=circle]
- }
-
-
- subgraph cluster_pscontrolled {
- label = "PrivateStorage.io Operated"
- rankdir = TB
- PSWebServer [label="PrivateStorage.io Web Server", shape=box]
- SubscriptionConfigWHPeer [label="Subscription Config Wormhole Peer", shape=box]
- PaymentServer [label="Payment Server", shape=box]
- SATIssuer [label="SAT Issuer", shape=box]
- PSStorageGrid [label="PrivateStorage.io Storage Grid", shape=box]
- }
-
- User [label="User", shape=egg]
- Stripe [label="Stripe", shape=pentagon]
-
- User -> PSWebServer [label="1. Get wormhole code", fontcolor=red, color=red]
- PSWebServer -> User [label="2. 7-petulant-banana", fontcolor=blue, color=blue]
- User -> GridSync [label="3. 7-petulant-banana", fontcolor=brown, color=brown]
- GridSync -> SubscriptionConfigWHPeer [label="4. Get configuration", fontcolor=black, color=black]
- SubscriptionConfigWHPeer -> GridSync [label="5. Grid configuration", fontcolor=magenta, color=magenta]
- GridSync -> TahoeLAFS [label="6. Instantiate", fontcolor=aquamarine3, color=aquamarine3]
- GridSync -> TahoeLAFS [label="7. Redeem PRN", fontcolor=crimson, color=crimson]
- TahoeLAFS -> PaymentServer [label="8. Redeem PRN", fontcolor=crimson, color=crimson]
- PaymentServer -> TahoeLAFS [label="9. Payment required", fontcolor=gold3, color=gold3]
- TahoeLAFS -> GridSync [label="10. Payment required", fontcolor=gold3, color=gold3]
- GridSync -> Browser [label="11. Open payment window", fontcolor=gold3, color=gold3]
- User -> Browser [label="12. Enter payment info", fontcolor=blue, color=blue]
- Browser -> Stripe [label="13. Submit payment form", fontcolor=brown, color=brown]
- Stripe -> Browser [label="14. Payment ok", fontcolor=black, color=black]
- Stripe -> PaymentServer [label="15. Payment notification", fontcolor=magenta, color=magenta]
- GridSync -> TahoeLAFS [label="16. Redeem PRN", fontcolor=aquamarine3, color=aquamarine3]
- TahoeLAFS -> TahoeLAFS [label="17. Generate blinded tokens", fontcolor=crimson, color=crimson]
- TahoeLAFS -> SATIssuer [label="18. Redeem PRN, blinded-tokens=xs", fontcolor=crimson, color=crimson]
- SATIssuer -> PaymentServer [label="19. Check PRN", fontcolor=gold3, color=gold3]
- PaymentServer -> SATIssuer [label="20. PRN Valid", fontcolor=gold3, color=gold3]
- SATIssuer -> TahoeLAFS [label="21. PRN valid, signed-tokens=ys", fontcolor=crimson, color=crimson]
- TahoeLAFS -> TahoeLAFS [label="22. Store signed tokens", fontcolor=crimson, color=crimson]
- TahoeLAFS -> GridSync [label="23. PRN Redeemed", fontcolor=red, color=red]
- TahoeLAFS -> PSStorageGrid [label="24. Use storage, passes=y", fontcolor=magenta, color=magenta]
-}
diff --git a/nixos/system-tests.nix b/nixos/system-tests.nix
index 7b6d382ada53c1121a1bc3d0edbf82964d644ad2..218132fe2cd3857f4c201085b4df56a411c794d4 100644
--- a/nixos/system-tests.nix
+++ b/nixos/system-tests.nix
@@ -1,7 +1,6 @@
# The overall system test suite for PrivateStorageio NixOS configuration.
-let
- pkgs = import ../nixpkgs-2105.nix { };
-in {
+{ pkgs }:
+{
private-storage = pkgs.nixosTest ./tests/private-storage.nix;
spending = pkgs.nixosTest ./tests/spending.nix;
tahoe = pkgs.nixosTest ./tests/tahoe.nix;
diff --git a/nixos/unit-tests.nix b/nixos/unit-tests.nix
index 75016a17d128fabe11f4ecaad65dba3471ed863d..b9f72bf95901af2668d68d1a09814c2bc2a9cd93 100644
--- a/nixos/unit-tests.nix
+++ b/nixos/unit-tests.nix
@@ -1,7 +1,6 @@
# The overall unit test suite for PrivateStorageio NixOS configuration.
+{ pkgs }:
let
- pkgs = import <nixpkgs> { };
-
# Total the numbers in a list.
sum = builtins.foldl' (a: b: a + b) 0;
diff --git a/privatestorageio.nix b/privatestorageio.nix
deleted file mode 100644
index cde46b16f6ed537bb5ee74c5641409b11716e11a..0000000000000000000000000000000000000000
--- a/privatestorageio.nix
+++ /dev/null
@@ -1,19 +0,0 @@
-{ stdenv, lib, graphviz, python3Packages }:
-stdenv.mkDerivation rec {
- version = "0.0";
- name = "privatestorageio-${version}";
- src = lib.cleanSource ./.;
-
- depsBuildBuild = [
- graphviz
- ];
-
- buildPhase = ''
- ${python3Packages.sphinx}/bin/sphinx-build -W docs/source docs/build
- '';
-
- installPhase = ''
- mkdir $out
- mv docs/build $out/docs
- '';
-}