Skip to content
Snippets Groups Projects
Unverified Commit 570731a0 authored by Jean-Paul Calderone's avatar Jean-Paul Calderone Committed by GitHub
Browse files

Merge pull request #1 from PrivateStorageio/133.nixos-service-module

Add a basic NixOS Private Storage service module.
parents 9caa15d6 6ac24ed6
No related branches found
No related tags found
No related merge requests found
Showing
with 755 additions and 12 deletions
......@@ -15,6 +15,23 @@
version: 2
jobs:
test:
docker:
- image: "nixos/nix:2.2.1"
steps:
- run:
name: "Install Git"
command: |
# Required for the checkout step
nix-env -i git openssh
- "checkout"
- run:
name: "Run Tests"
command: |
nix-build nixos/unit-tests.nix && cat result
build:
docker:
- image: "nixos/nix:2.2.1"
......@@ -24,8 +41,8 @@ jobs:
command: |
# Required for cache and artifact interactions. Though we use a
# nix image, it's actually an alpine base... The CircleCI cache
# management and artifact uploader doesn't know how to use the nix
# ca bundle we could install.
# management (which we stopped using) and artifact uploader don't
# know how to use the nix ca bundle we could install.
apk update
apk add ca-certificates
......@@ -35,21 +52,12 @@ jobs:
# Required for the checkout step
nix-env -i git openssh
- restore_cache:
keys:
- "v1-nix-store"
- "checkout"
- run:
name: "Nix Build"
command: |
nix-build
- save_cache:
key: "v1-nix-store"
paths:
- "/nix/store"
nix-build docs.nix
- store_artifacts:
path: "result/docs"
......@@ -59,4 +67,5 @@ workflows:
version: 2
everything:
jobs:
- "test"
- "build"
......@@ -2,3 +2,36 @@ PrivateStorageio
================
The backend for a private, secure, and end-to-end encrypted storage solution
Building
--------
The build system uses `Nix`_ which must be installed before anything can be built.
Documentation
~~~~~~~~~~~~~
The documentation can be built using this command::
$ nix-build docs.nix
The documentation is also built on and published by CI.
Testing
-------
The test system uses `Nix`_ which must be installed before any tests can be run.
Unit tests are run using this command::
$ nix-build nixos/unit-tests.nix
Unit tests are also run on CI.
The system tests are run using this command::
$ nix-build nixos/system-tests.nix
The system tests boot QEMU VMs which prevents them from running on CI at this time.
.. _Nix: https://nixos.org/nix
File moved
.. include::
../../README.rst
......@@ -10,8 +10,11 @@ Welcome to PrivateStorageio's documentation!
:maxdepth: 2
:caption: Contents:
README
architecture-overview
Indices and tables
==================
......
# Functionality related to writing out ini syntax files (like Tahoe-LAFS'
# tahoe.cfg).
{ pkgs ? import <nixpkgs> { } }:
let lib = pkgs.lib;
in rec {
# Get the .ini-file-appropriate string representation of a simple value.
#
# > toINIString "hello"
# "hello"
# > toINIString true
# "true"
toINIString = v:
if builtins.isBool v then builtins.toJSON v
else builtins.toString v;
# Map a function over an attrset and concatenate the string results.
#
# > concatMapAttrsToList (n: v: "${n} = ${v}\n") { a = "b"; c = "d"; }
# "a = b\nc = d\n"
concatMapAttrsToList = f: a:
lib.strings.concatStrings (lib.attrsets.mapAttrsToList f a);
# Generate one line of configuration defining one item in one section.
#
# > oneConfigItemText "foo" "bar"
# "foo = bar\n"
oneConfigItemText = name: value:
"${name} = ${toINIString value}\n";
# Generate all lines of configuration defining all items in one section.
#
# > allConfigItemsText { foo = "bar"; baz = "quux"; }
# "foo = bar\nbaz = quux"
allConfigItemsText = items:
concatMapAttrsToList oneConfigItemText items;
# Generate all lines of configuration for one section, header
# and items included.
#
# > oneConfigSectionText "foo" { bar = "baz"; }
# "[foo]\nbar = baz\n"
oneConfigSectionText = name: value: ''
[${name}]
${allConfigItemsText value}'';
# Generate all lines of configuration for all sections, headers
# and items included.
#
# > allConfigSectionsText { foo = { bar = "baz"; }; }
# "[foo]\nbar = baz\n"
allConfigSectionsText = sections:
concatMapAttrsToList oneConfigSectionText sections;
}
ini:
{ test_empty =
{ expected = "";
expr = ini.allConfigSectionsText { };
};
test_one_empty_section =
{ expected = ''
[foo]
'';
expr = ini.allConfigSectionsText { foo = { }; };
};
test_one_section_one_item =
{ expected = ''
[foo]
bar = baz
'';
expr = ini.allConfigSectionsText { foo = { bar = "baz"; }; };
};
test_one_section_two_items =
{ expected = ''
[foo]
bar = baz
foobar = quux
'';
expr = ini.allConfigSectionsText { foo = { bar = "baz"; foobar = "quux"; }; };
};
test_two_sections =
{ expected = ''
[alpha]
beta = gamma
[foo]
bar = baz
foobar = quux
'';
expr = ini.allConfigSectionsText
{ foo = { bar = "baz"; foobar = "quux"; };
alpha = { beta = "gamma"; };
};
};
test_true =
{ expected = "x = true\n";
expr = ini.oneConfigItemText "x" true;
};
test_false =
{ expected = "x = false\n";
expr = ini.oneConfigItemText "x" false;
};
test_integer =
{ expected = "x = 12345\n";
expr = ini.oneConfigItemText "x" 12345;
};
test_dotted_key =
{ expected = "x.y = z\n";
expr = ini.oneConfigItemText "x.y" "z";
};
}
self: super: {
python27 = super.python27.override {
packageOverrides = python-self: python-super: {
# Get the newest Tahoe-LAFS as a module instead of an application.
tahoe-lafs = python-super.toPythonModule (python-super.callPackage ../pkgs/tahoe-lafs.nix { });
# Get our ZKAP authorizer plugin package.
zkapauthorizer = python-self.callPackage ../pkgs/zkapauthorizer.nix { };
# new tahoe-lafs has a new dependency on eliot.
eliot = python-super.callPackage ../pkgs/eliot.nix { };
# new tahoe-lafs depends on a very recent autobahn for better websocket
# testing features.
autobahn = python-super.callPackage ../pkgs/autobahn.nix { };
# new autobahn requires a newer cryptography
cryptography = python-super.callPackage ../pkgs/cryptography.nix { };
# new cryptography requires a newer cryptography_vectors
cryptography_vectors = python-super.callPackage ../pkgs/cryptography_vectors.nix { };
# upstream twisted package is missing a recently added dependency.
twisted = python-super.twisted.overrideAttrs (old:
{ propagatedBuildInputs = old.propagatedBuildInputs ++ [ python-super.appdirs ];
checkPhase = ''
${self.python.interpreter} -m twisted.trial twisted
'';
});
};
};
privatestorage = self.python27.buildEnv.override
{ extraLibs =
[ self.python27Packages.tahoe-lafs
self.python27Packages.zkapauthorizer
];
# Twisted's dropin.cache always collides between different
# plugin-providing packages.
ignoreCollisions = true;
};
}
# A NixOS module which can instantiate a Tahoe-LAFS storage server in the
# preferred configuration for the Private Storage grid.
{ pkgs, lib, config, ... }:
let
pspkgs = import pkgs.path
{ overlays = [ (import ./overlays.nix) ];
};
cfg = config.services.private-storage;
in
{
# Upstream tahoe-lafs module conflicts with ours (since ours is a
# copy/paste/edit of upstream's...). Disable
# it.
#
# https://nixos.org/nixos/manual/#sec-replace-modules
disabledModules =
[ "services/network-filesystems/tahoe.nix"
];
# Load our tahoe-lafs module.
imports =
[ ./tahoe.nix
];
options =
{ services.private-storage.enable = lib.mkEnableOption "private storage service";
services.private-storage.tahoe.package = lib.mkOption
{ default = pspkgs.privatestorage;
type = lib.types.package;
example = lib.literalExample "pkgs.tahoelafs";
description = ''
The package to use for the Tahoe-LAFS daemon.
'';
};
};
config = lib.mkIf cfg.enable
{ services.tahoe.nodes."storage" =
{ package = config.services.private-storage.tahoe.package;
sections =
{ node =
# XXX Should try to name that is unique across the grid.
{ nickname = "storage";
"web.port" = "tcp:3456:interface=127.0.0.1";
};
storage =
{ enabled = true;
plugins = "privatestorageio-zkapauthz-v1";
};
"storageserver.plugins.privatestorageio-zkapauthz-v1" =
{
};
};
};
};
}
# Copy/pasted from nixos/modules/services/network-filesystems/tahoe.nix :/ We
# require control over additional configuration options compared to upstream
# and it's not clear how to do this without duplicating everything.
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.tahoe;
ini = pkgs.callPackage ../lib/ini.nix { };
in
{
options.services.tahoe = {
introducers = mkOption {
default = {};
type = with types; attrsOf (submodule {
options = {
nickname = mkOption {
type = types.str;
description = ''
The nickname of this Tahoe introducer.
'';
};
tub.port = mkOption {
default = 3458;
type = types.int;
description = ''
The port on which the introducer will listen.
'';
};
tub.location = mkOption {
default = null;
type = types.nullOr types.str;
description = ''
The external location that the introducer should listen on.
If specified, the port should be included.
'';
};
package = mkOption {
default = pkgs.tahoelafs;
defaultText = "pkgs.tahoelafs";
type = types.package;
example = literalExample "pkgs.tahoelafs";
description = ''
The package to use for the Tahoe LAFS daemon.
'';
};
};
});
description = ''
The Tahoe introducers.
'';
};
nodes = mkOption {
default = {};
type = with types; attrsOf (submodule {
options = {
sections = mkOption {
type = types.attrs;
description = ''
Top-level configuration sections.
'';
default = {
"node" = { };
"client" = { };
"storage" = { };
};
};
package = mkOption {
default = pkgs.tahoelafs;
defaultText = "pkgs.tahoelafs";
type = types.package;
example = literalExample "pkgs.tahoelafs";
description = ''
The package to use for the Tahoe LAFS daemon.
'';
};
};
});
description = ''
The Tahoe nodes.
'';
};
};
config = mkMerge [
(mkIf (cfg.introducers != {}) {
environment = {
etc = flip mapAttrs' cfg.introducers (node: settings:
nameValuePair "tahoe-lafs/introducer-${node}.cfg" {
mode = "0444";
text = ''
# This configuration is generated by Nix. Edit at your own
# peril; here be dragons.
[node]
nickname = ${settings.nickname}
tub.port = ${toString settings.tub.port}
${optionalString (settings.tub.location != null)
"tub.location = ${settings.tub.location}"}
'';
});
# Actually require Tahoe, so that we will have it installed.
systemPackages = flip mapAttrsToList cfg.introducers (node: settings:
settings.package
);
};
# Open up the firewall.
# networking.firewall.allowedTCPPorts = flip mapAttrsToList cfg.introducers
# (node: settings: settings.tub.port);
systemd.services = flip mapAttrs' cfg.introducers (node: settings:
let
pidfile = "/run/tahoe.introducer-${node}.pid";
# This is a directory, but it has no trailing slash. Tahoe commands
# get antsy when there's a trailing slash.
nodedir = "/var/db/tahoe-lafs/introducer-${node}";
in nameValuePair "tahoe.introducer-${node}" {
description = "Tahoe LAFS node ${node}";
wantedBy = [ "multi-user.target" ];
path = [ settings.package ];
restartTriggers = [
config.environment.etc."tahoe-lafs/introducer-${node}.cfg".source ];
serviceConfig = {
Type = "simple";
PIDFile = pidfile;
# Believe it or not, Tahoe is very brittle about the order of
# arguments to $(tahoe run). The node directory must come first,
# and arguments which alter Twisted's behavior come afterwards.
ExecStart = ''
${settings.package}/bin/tahoe run ${lib.escapeShellArg nodedir} -n -l- --pidfile=${lib.escapeShellArg pidfile}
'';
};
preStart = ''
if [ ! -d ${lib.escapeShellArg nodedir} ]; then
mkdir -p /var/db/tahoe-lafs
tahoe create-introducer ${lib.escapeShellArg nodedir}
fi
# Tahoe has created a predefined tahoe.cfg which we must now
# scribble over.
# XXX I thought that a symlink would work here, but it doesn't, so
# we must do this on every prestart. Fixes welcome.
# rm ${nodedir}/tahoe.cfg
# ln -s /etc/tahoe-lafs/introducer-${node}.cfg ${nodedir}/tahoe.cfg
cp /etc/tahoe-lafs/introducer-"${node}".cfg ${lib.escapeShellArg nodedir}/tahoe.cfg
'';
});
users.users = flip mapAttrs' cfg.introducers (node: _:
nameValuePair "tahoe.introducer-${node}" {
description = "Tahoe node user for introducer ${node}";
isSystemUser = true;
});
})
(mkIf (cfg.nodes != {}) {
environment = {
etc = flip mapAttrs' cfg.nodes (node: settings:
nameValuePair "tahoe-lafs/${node}.cfg" {
mode = "0444";
text = ''
# This configuration is generated by Nix. Edit at your own
# peril; here be dragons.
${ini.allConfigSectionsText settings.sections}
'';
});
# Actually require Tahoe, so that we will have it installed.
systemPackages = flip mapAttrsToList cfg.nodes (node: settings:
settings.package
);
};
# Open up the firewall.
# networking.firewall.allowedTCPPorts = flip mapAttrsToList cfg.nodes
# (node: settings: settings.tub.port);
systemd.services = flip mapAttrs' cfg.nodes (node: settings:
let
pidfile = "/run/tahoe.${node}.pid";
# This is a directory, but it has no trailing slash. Tahoe commands
# get antsy when there's a trailing slash.
nodedir = "/var/db/tahoe-lafs/${node}";
in nameValuePair "tahoe.${node}" {
description = "Tahoe LAFS node ${node}";
wantedBy = [ "multi-user.target" ];
path = [ settings.package ];
restartTriggers = [
config.environment.etc."tahoe-lafs/${node}.cfg".source ];
serviceConfig = {
Type = "simple";
PIDFile = pidfile;
# Believe it or not, Tahoe is very brittle about the order of
# arguments to $(tahoe run). The node directory must come first,
# and arguments which alter Twisted's behavior come afterwards.
ExecStart = ''
${settings.package}/bin/tahoe run ${lib.escapeShellArg nodedir} -n -l- --pidfile=${lib.escapeShellArg pidfile}
'';
};
preStart = ''
if [ ! -d ${lib.escapeShellArg nodedir} ]; then
mkdir -p /var/db/tahoe-lafs
tahoe create-node --hostname=localhost ${lib.escapeShellArg nodedir}
fi
# Tahoe has created a predefined tahoe.cfg which we must now
# scribble over.
# XXX I thought that a symlink would work here, but it doesn't, so
# we must do this on every prestart. Fixes welcome.
# rm ${nodedir}/tahoe.cfg
# ln -s /etc/tahoe-lafs/${lib.escapeShellArg node}.cfg ${nodedir}/tahoe.cfg
cp /etc/tahoe-lafs/${lib.escapeShellArg node}.cfg ${lib.escapeShellArg nodedir}/tahoe.cfg
'';
});
users.users = flip mapAttrs' cfg.nodes (node: _:
nameValuePair "tahoe.${node}" {
description = "Tahoe node user for node ${node}";
isSystemUser = true;
});
})
];
}
# https://nixos.org/nixos/manual/index.html#sec-nixos-tests
import <nixpkgs/nixos/tests/make-test.nix> {
# Configure a single machine as a PrivateStorage storage node.
machine =
{ config, pkgs, ... }:
{ imports =
[ ../private-storage.nix
];
services.private-storage.enable = true;
};
# Test the machine with a Perl program (sobbing).
testScript =
''
# Boot the VM.
$machine->start;
# The systemd unit should reach the running state.
$machine->waitForUnit("tahoe.storage.service");
# Some while after that the Tahoe-LAFS node should listen on the web API
# port. The port number here has to agree with the port number set in
# the private-storage.nix module.
$machine->waitForOpenPort(3456);
# Once the web API is listening it should be possible to scrape some
# status from the node if it is really working.
$machine->succeed("tahoe -d /var/db/tahoe-lafs/storage status");
'';
}
{ lib, buildPythonPackage, fetchFromGitHub, isPy3k,
six, txaio, twisted, zope_interface, cffi, trollius, futures, cryptography,
mock, pytest
}:
buildPythonPackage rec {
pname = "autobahn";
version = "19.7.1";
src = fetchFromGitHub {
owner = "crossbario";
repo = "autobahn-python";
rev = "v${version}";
sha256 = "1gl2m18s77hlpiglh44plv3k6b965n66ylnxbzgvzcdl9jf3l3q3";
};
propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography ] ++
(lib.optionals (!isPy3k) [ trollius futures ]);
checkInputs = [ mock pytest ];
checkPhase = ''
runHook preCheck
USE_TWISTED=true py.test $out
runHook postCheck
'';
# XXX Fails for some reason I don't understand.
doCheck = false;
meta = with lib; {
description = "WebSocket and WAMP in Python for Twisted and asyncio.";
homepage = "https://crossbar.io/autobahn";
license = licenses.mit;
maintainers = with maintainers; [ nand0p ];
};
}
{ stdenv
, buildPythonPackage
, fetchFromGitHub
, openssl
, cryptography_vectors
, darwin
, asn1crypto
, packaging
, six
, pythonOlder
, enum34
, ipaddress
, isPyPy
, cffi
, pytest
, pretend
, iso8601
, pytz
, hypothesis
}:
buildPythonPackage rec {
pname = "cryptography";
version = "2.7"; # Also update the hash in vectors.nix
src = fetchFromGitHub {
owner = "pyca";
repo = "cryptography";
rev = "2.7";
sha256 = "145byri5c3b8m6dbhwb6yxrv9jrr652l3z1w16mz205z8dz38qja";
};
outputs = [ "out" "dev" ];
buildInputs = [ openssl ]
++ stdenv.lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
propagatedBuildInputs = [
asn1crypto
packaging
six
] ++ stdenv.lib.optional (pythonOlder "3.4") enum34
++ stdenv.lib.optional (pythonOlder "3.3") ipaddress
++ stdenv.lib.optional (!isPyPy) cffi;
checkInputs = [
cryptography_vectors
hypothesis
iso8601
pretend
pytest
pytz
];
checkPhase = ''
py.test --disable-pytest-warnings tests
'';
# IOKit's dependencies are inconsistent between OSX versions, so this is the best we
# can do until nix 1.11's release
__impureHostDeps = [ "/usr/lib" ];
meta = with stdenv.lib; {
description = "A package which provides cryptographic recipes and primitives";
longDescription = ''
Cryptography includes both high level recipes and low level interfaces to
common cryptographic algorithms such as symmetric ciphers, message
digests, and key derivation functions.
Our goal is for it to be your "cryptographic standard library". It
supports Python 2.7, Python 3.4+, and PyPy 5.3+.
'';
homepage = https://github.com/pyca/cryptography;
license = with licenses; [ asl20 bsd3 psfl ];
maintainers = with maintainers; [ primeos ];
};
}
{ buildPythonPackage, fetchPypi, lib, cryptography }:
buildPythonPackage rec {
pname = "cryptography_vectors";
# The test vectors must have the same version as the cryptography package:
version = cryptography.version;
src = fetchPypi {
inherit pname version;
sha256 = "1g38zw90510azyfrj6mxbslx2gp9yrnv5dac0w2819k9ssdznbgi";
};
# No tests included
doCheck = false;
meta = with lib; {
description = "Test vectors for the cryptography package";
homepage = https://cryptography.io/en/latest/development/test-vectors/;
# Source: https://github.com/pyca/cryptography/tree/master/vectors;
license = with licenses; [ asl20 bsd3 ];
maintainers = with maintainers; [ primeos ];
};
}
{ lib, buildPythonPackage, fetchPypi, zope_interface, pyrsistent, boltons
, hypothesis, testtools, pytest }:
buildPythonPackage rec {
pname = "eliot";
version = "1.7.0";
src = fetchPypi {
inherit pname version;
sha256 = "0ylyycf717s5qsrx8b9n6m38vyj2k8328lfhn8y6r31824991wv8";
};
postPatch = ''
substituteInPlace setup.py \
--replace "boltons >= 19.0.1" boltons
# depends on eliot.prettyprint._main which we don't have here.
rm eliot/tests/test_prettyprint.py
'';
checkInputs = [ testtools pytest hypothesis ];
propagatedBuildInputs = [ zope_interface pyrsistent boltons ];
meta = with lib; {
homepage = https://github.com/itamarst/eliot/;
description = "Logging library that tells you why it happened";
license = licenses.asl20;
};
}
{ fetchFromGitHub, eliot, tahoelafs, plugins ? [ ] }:
tahoelafs.overrideAttrs (old:
{ src = fetchFromGitHub
{ owner = "tahoe-lafs";
repo = "tahoe-lafs";
rev = "6c1a37c95188c1d9a877286ef726280a68d38a4b";
sha256 = "1fd8b6j52wn04bnvnvysws4c713max6k1592lz4nzyjlhrcwawwh";
};
propagatedBuildInputs = old.propagatedBuildInputs ++ [ eliot ] ++ plugins;
doInstallCheck = false;
})
{ pkgs, fetchFromGitHub, tahoe-lafs }:
let
src = fetchFromGitHub
{ owner = "PrivateStorageio";
repo = "ZKAPAuthorizer";
rev = "a14b38f39e48d1560ea10ec26fffad6ce50fd00a";
sha256 = "1v81l0ylx8r8xflhi16m8hb1dm3rlzyfrldiknvggqkyi5psdja4";
};
in
pkgs.python27Packages.callPackage "${src}/zkapauthorizer.nix"
{ inherit tahoe-lafs;
}
# The overall system test suite for PrivateStorageio NixOS configuration.
# There is only one system test so far so I don't have to do anything to
# aggregate multiple tests...
import ./modules/tests/private-storage.nix
# The overall unit test suite for PrivateStorageio NixOS configuration.
let
pkgs = import <nixpkgs> { };
# Total the numbers in a list.
sum = builtins.foldl' (a: b: a + b) 0;
# A helper for loading tests.
loadTest = moduleUnderTest: testModule:
(import testModule (pkgs.callPackage moduleUnderTest { }));
# A list of tests to run. Manually updated for now, alas. Only tests in
# this list will be run!
testModules =
[ (loadTest ./lib/ini.nix ./lib/tests/test_ini.nix)
];
# Count up the tests we're going to run.
numTests = sum (map (s: builtins.length (builtins.attrNames s)) testModules);
# Convert it into a string for interpolation into the shell script.
numTestsStr = builtins.toString numTests;
# Run the tests and collect the failures.
failures = map pkgs.lib.runTests testModules;
# Count the number of failures in each module.
numFailures = sum (map builtins.length failures);
# Convert the total into a string for easy interpolation into the shell script.
numFailuresStr = builtins.toString (numFailures);
# Convert the failure information to a string for reporting.
failuresStr = builtins.toJSON failures;
in
pkgs.runCommand "test-results" {} ''
if [ ${numFailuresStr} -gt 0 ]; then
echo "Failed ${numFailuresStr} tests"
echo '${failuresStr}'
exit 1
else
echo '${numTestsStr} tests OK' > $out
fi
''
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment