diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 467ebe7c1e6199366594768bcb4a246091157651..3c8030a2583f5458fb102726a83a6be2d501c049 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -71,5 +71,5 @@ nixpkgs
We pin to a nixos channel release, which isn't directly supported by niv (`issue <https://github.com/nmattia/niv/issues/225>`_).
Thus, the pin needs to be update manually.
-To do this, copy the ``url`` and ``sha256`` values from PrivateStorageio's `nixpkgs-2105.json <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/develop/nixpkgs-2105.json>`_ into the ``release2015`` entry in ``nix/sources.json``.
+To do this, copy the ``url`` and ``sha256`` values from PrivateStorageio's `nixpkgs-2105.json <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/develop/nixpkgs-2105.json>`_ into the ``release2105`` entry in ``nix/sources.json``.
When this is deployed as part of Privatestorageio, we use the value pinned there, rather than the pin in this repository.
diff --git a/default.nix b/default.nix
index 7f9b277c7290c3a277c7f80ff194ddabbb32c00b..e2b8e609769c6f559bf77fbbe28beee11ae53d58 100644
--- a/default.nix
+++ b/default.nix
@@ -1,7 +1,7 @@
let
sources = import nix/sources.nix;
in
-{ pkgs ? import sources.release2015 {}
+{ pkgs ? import sources.release2105 {}
, pypiData ? sources.pypi-deps-db
, mach-nix ? import sources.mach-nix { inherit pkgs pypiData; }
, tahoe-lafs-source ? "tahoe-lafs"
@@ -30,13 +30,25 @@ in
traceback2 = "wheel";
# - Incorrectly merged extras - https://github.com/DavHau/mach-nix/pull/334
tqdm = "wheel";
+
+ # The version of Klein we get doesn't need / can't have the patch that
+ # comes from the nixpkgs derivation mach-nix picks up from 21.05.
+ klein = "wheel";
};
in
rec {
tahoe-lafs = mach-nix.buildPythonPackage rec {
inherit python providers;
name = "tahoe-lafs";
- version = "1.16.post999";
+ # We add `.post999` here so that we don't accidentally *exactly* match
+ # the upstream Tahoe-LAFS version. This avoids the misleading
+ # circumstance where the version in the Nix packaging *looks* like a
+ # real upstream Tahoe-LAFS revision but we have forgotten to update it
+ # so it is the *wrong* real upstream Tahoe-LAFS revision. Hopefully
+ # the `.post999` looks weird enough that if someone really cares about
+ # the version in use they will notice it and go searching for what's
+ # going on and discover the real version specified by `src` below.
+ version = "1.17.0.post999";
# See https://github.com/DavHau/mach-nix/issues/190
requirementsExtra = ''
pyrsistent < 0.17
diff --git a/nix/sources.json b/nix/sources.json
index 9ce1d8e0cf1793a5db2e50d82e8758ec645578e7..fdde626e48c8f4c98f24532a65da6829345b6f6e 100644
--- a/nix/sources.json
+++ b/nix/sources.json
@@ -35,7 +35,7 @@
"url": "https://github.com/DavHau/pypi-deps-db/archive/96d01556b4597c022647acbf8c3b58d2a99bc963.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
- "release2015": {
+ "release2105": {
"sha256": "112drvixj81vscj8cncmks311rk2ik5gydpd03d3r0yc939zjskg",
"type": "tarball",
"url": "https://releases.nixos.org/nixos/21.05/nixos-21.05.3740.ce7a1190a0f/nixexprs.tar.xz",
@@ -47,10 +47,10 @@
"homepage": "https://tahoe-lafs.org/",
"owner": "tahoe-lafs",
"repo": "tahoe-lafs",
- "rev": "d3c6f58a8ded7db3324ef97c47f5c1921c3d58b7",
- "sha256": "18zr6l53r32pigymsnv10m67kgf981bxl8c3rjhv5bikfnf986q8",
+ "rev": "tahoe-lafs-1.17.0",
+ "sha256": "0vjq7g1lfjd16y0iwnfsccp5k3q3av7wllkyqbsyd877a29nibzi",
"type": "tarball",
- "url": "https://github.com/tahoe-lafs/tahoe-lafs/archive/d3c6f58a8ded7db3324ef97c47f5c1921c3d58b7.tar.gz",
+ "url": "https://github.com/tahoe-lafs/tahoe-lafs/archive/tahoe-lafs-1.17.0.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
}
}
diff --git a/setup.cfg b/setup.cfg
index c1c6250091326d2793a6d2d3ed9843dcde0b0578..6d39a54cf210521314307ad39114e59389be8e12 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -40,13 +40,8 @@ install_requires =
# The pip resolver sometimes finds treq's dependencies first and these are
# incompatible with Tahoe-LAFS'. So duplicate them here (the ones that
# have been observed to cause problems).
- Twisted[tls,conch]>=18.4.0
-
- # Lease renewal changes aren't available from a release of Tahoe yet.
- # Note "Public index servers SHOULD NOT allow the use of direct references
- # in uploaded distributions."
- # https://www.python.org/dev/peps/pep-0440/#direct-references
- tahoe-lafs @ https://github.com/tahoe-lafs/tahoe-lafs/archive/d3c6f58a8ded7db3324ef97c47f5c1921c3d58b7.zip
+ Twisted[tls,conch] >= 19.10.0
+ tahoe-lafs >=1.17,<1.18
treq
pyutil
prometheus-client
diff --git a/shell.nix b/shell.nix
index f34fcf949ca84d7b092cf06761aae4f03db8df51..0f661ebc3692fbf6bfe199abc0cbd8fbd6feaa8e 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,7 +1,7 @@
let
sources = import nix/sources.nix;
in
-{ pkgs ? import sources.release2015 {}
+{ pkgs ? import sources.release2105 {}
, tahoe-lafs-source ? "tahoe-lafs"
}:
let
diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py
index 5fdf3e473b27086abd680481b736ecd9888695f4..809b385118b2a2f27c32e623c5d97bea2b8fc3be 100644
--- a/src/_zkapauthorizer/_storage_server.py
+++ b/src/_zkapauthorizer/_storage_server.py
@@ -33,7 +33,9 @@ from struct import calcsize, unpack
import attr
from allmydata.interfaces import RIStorageServer, TestAndWriteVectorsForShares
from allmydata.storage.common import storage_index_to_dir
+from allmydata.storage.immutable import ShareFile
from allmydata.storage.lease import LeaseInfo
+from allmydata.storage.mutable import MutableShareFile
from allmydata.storage.server import StorageServer
from allmydata.storage.shares import get_share_file
from allmydata.util.base32 import b2a
@@ -721,7 +723,7 @@ def get_storage_index_share_size(sharepath):
# From src/allmydata/storage/immutable.py
#
# The share file has the following layout:
- # 0x00: share file version number, four bytes, current version is 1
+ # 0x00: share file version number, four bytes, current version is 2
# 0x04: share data length, four bytes big-endian = A # See Footnote 1 below.
# 0x08: number of leases, four bytes big-endian
# 0x0c: beginning of share data (see immutable.layout.WriteBucketProxy)
@@ -756,12 +758,14 @@ def get_storage_index_share_size(sharepath):
version, _, number_of_leases = unpack(header_format, header)
- if version != 1:
- raise ValueError(
- "Cannot interpret version {} share file.".format(version),
- )
+ if version in (1, 2):
+ # Version 1 and 2 don't differ in a way that changes the size
+ # calculation.
+ return share_file_size - header_size - (number_of_leases * (4 + 32 + 32 + 4))
- return share_file_size - header_size - (number_of_leases * (4 + 32 + 32 + 4))
+ raise ValueError(
+ "Cannot interpret version {} share file.".format(version),
+ )
def stat_bucket(storage_server, storage_index, sharepath):
@@ -824,10 +828,16 @@ def get_stat(sharepath):
# Figure out if it is a storage index or a slot.
with open(sharepath, "rb") as share_file:
magic = share_file.read(32)
- if magic == "Tahoe mutable container v1\n" + "\x75\x09\x44\x03\x8e":
+ if len(magic) < 32:
+ # Tahoe could check for this.
+ # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3853
+ raise ValueError("Share file has short header")
+ if ShareFile.is_valid_header(magic):
+ return stat_bucket
+ elif MutableShareFile.is_valid_header(magic):
return stat_slot
else:
- return stat_bucket
+ raise ValueError("Cannot interpret share header {!r}".format(magic))
def add_leases_for_writev(storage_server, storage_index, secrets, tw_vectors, now):
@@ -847,6 +857,7 @@ def add_leases_for_writev(storage_server, storage_index, secrets, tw_vectors, no
(write_enabler, renew_secret, cancel_secret) = secrets
share = get_share_file(sharepath)
share.add_or_renew_lease(
+ storage_server.get_available_space(),
LeaseInfo(
owner_num=1,
renew_secret=renew_secret,
diff --git a/src/_zkapauthorizer/tests/fixtures.py b/src/_zkapauthorizer/tests/fixtures.py
index 3a4e4155fb8cbf4f19c36bd6f0e29c22b2ad79bc..e017eadd6b42f10f6f4255caf5c110d64cfca7de 100644
--- a/src/_zkapauthorizer/tests/fixtures.py
+++ b/src/_zkapauthorizer/tests/fixtures.py
@@ -21,7 +21,6 @@ from __future__ import absolute_import
from base64 import b64encode
import attr
-from allmydata import __version__ as allmydata_version
from allmydata.storage.server import StorageServer
from fixtures import Fixture, TempDir
from twisted.internet.task import Clock
@@ -51,18 +50,10 @@ class AnonymousStorageServer(Fixture):
def _setUp(self):
self.tempdir = FilePath(self.useFixture(TempDir()).join(b"storage"))
- if allmydata_version >= "1.16.":
- # This version of Tahoe adds a new StorageServer argument for
- # controlling time.
- timeargs = {"get_current_time": self.clock.seconds}
- else:
- # Older versions just use time.time() and there's not much we can
- # do _here_. Code somewhere else will have to monkey-patch that
- # to control things.
- timeargs = {}
-
self.storage_server = StorageServer(
- self.tempdir.asBytesMode().path, b"x" * 20, **timeargs
+ self.tempdir.asBytesMode().path,
+ b"x" * 20,
+ clock=self.clock,
)
diff --git a/src/_zkapauthorizer/tests/test_storage_protocol.py b/src/_zkapauthorizer/tests/test_storage_protocol.py
index a5b642997cb07c01ced81c33090f59713ef9f34e..89654633b6b9176ec98fb71492571b0467eb42b7 100644
--- a/src/_zkapauthorizer/tests/test_storage_protocol.py
+++ b/src/_zkapauthorizer/tests/test_storage_protocol.py
@@ -21,7 +21,6 @@ from __future__ import absolute_import
from allmydata.storage.common import storage_index_to_dir
from allmydata.storage.shares import get_share_file
from challenge_bypass_ristretto import PublicKey, random_signing_key
-from fixtures import MonkeyPatch
from foolscap.referenceable import LocalReferenceable
from hypothesis import assume, given
from hypothesis.strategies import data as data_strategy
@@ -540,33 +539,22 @@ class ShareTests(TestCase):
self.clock.advance(when)
- # anonymous_storage_server uses time.time() before Tahoe-LAFS 1.16,
- # unfortunately. For Tahoe-LAFS 1.16, AnonymousStorageServer will
- # glue self.clock in for us. For older versions we still need this
- # monkey-patching.
- #
- # And useFixture does not interact very well with Hypothesis.
- patch = MonkeyPatch("time.time", self.clock.seconds)
- try:
- patch.setUp()
- # Create a share we can toy with.
- write_shares(
- self.anonymous_storage_server,
+ # Create a share we can toy with.
+ write_shares(
+ self.anonymous_storage_server,
+ storage_index,
+ {sharenum},
+ size,
+ canary=self.canary,
+ )
+ # Perhaps put some more leases on it. Leases might impact our
+ # ability to determine share data size.
+ for renew_secret in leases:
+ self.anonymous_storage_server.remote_add_lease(
storage_index,
- {sharenum},
- size,
- canary=self.canary,
+ renew_secret,
+ cancel_secret,
)
- # Perhaps put some more leases on it. Leases might impact our
- # ability to determine share data size.
- for renew_secret in leases:
- self.anonymous_storage_server.remote_add_lease(
- storage_index,
- renew_secret,
- cancel_secret,
- )
- finally:
- patch.cleanUp()
expected = [
{
@@ -629,7 +617,7 @@ class ShareTests(TestCase):
If a share file with an unexpected version is found, ``stat_shares``
declines to offer a result (by raising ``ValueError``).
"""
- assume(version != 1)
+ assume(version not in (1, 2))
sharedir = FilePath(self.anonymous_storage_server.sharedir).preauthChild(
# storage_index_to_dir likes to return multiple segments
@@ -771,23 +759,18 @@ class ShareTests(TestCase):
"""
self.clock.advance(when)
- patch = MonkeyPatch("time.time", self.clock.seconds)
- try:
- patch.setUp()
- # Create a share we can toy with.
- wrote, read = extract_result(
- self.client.slot_testv_and_readv_and_writev(
- storage_index,
- secrets=secrets,
- tw_vectors={
- k: v.for_call()
- for (k, v) in test_and_write_vectors_for_shares.items()
- },
- r_vector=[],
- ),
- )
- finally:
- patch.cleanUp()
+ # Create a share we can toy with.
+ wrote, read = extract_result(
+ self.client.slot_testv_and_readv_and_writev(
+ storage_index,
+ secrets=secrets,
+ tw_vectors={
+ k: v.for_call()
+ for (k, v) in test_and_write_vectors_for_shares.items()
+ },
+ r_vector=[],
+ ),
+ )
self.assertThat(
wrote,
Equals(True),
@@ -1052,20 +1035,14 @@ class ShareTests(TestCase):
r_vector=[],
)
- patch = MonkeyPatch("time.time", self.clock.seconds)
- try:
- patch.setUp()
-
- # Create a share we can toy with.
- self.assertThat(write(), is_successful_write())
+ # Create a share we can toy with.
+ self.assertThat(write(), is_successful_write())
- # Advance time by more than a lease period so the lease is no
- # longer valid.
- self.clock.advance(self.server.LEASE_PERIOD.total_seconds() + 1)
+ # Advance time by more than a lease period so the lease is no
+ # longer valid.
+ self.clock.advance(self.server.LEASE_PERIOD.total_seconds() + 1)
- self.assertThat(write(), is_successful_write())
- finally:
- patch.cleanUp()
+ self.assertThat(write(), is_successful_write())
# The spent passes have been reported to the spending service.
self.assertThat(
diff --git a/tests.nix b/tests.nix
index 7023c06a296dfd5be2cb3468c789d7038ac25add..c985ce0f41b39c886b67788243835d2ee8605808 100644
--- a/tests.nix
+++ b/tests.nix
@@ -1,7 +1,7 @@
let
sources = import nix/sources.nix;
in
-{ pkgs ? import sources.release2015 {}
+{ pkgs ? import sources.release2105 {}
, pypiData ? sources.pypi-deps-db
, mach-nix ? import sources.mach-nix { inherit pkgs pypiData; }
, tahoe-lafs-source ? "tahoe-lafs"
@@ -20,7 +20,7 @@ in
inherit (pkgs) lib;
inherit (privatestorage) zkapauthorizer;
hypothesisProfile' = if hypothesisProfile == null then "default" else hypothesisProfile;
- defaultTrialArgs = [ "--rterrors" ] ++ (lib.optional (! collectCoverage) "--jobs=$NIX_BUILD_CORES");
+ defaultTrialArgs = [ "--rterrors" ] ++ (lib.optional (! collectCoverage) "--jobs=$(($NIX_BUILD_CORES > 8 ? 8 : $NIX_BUILD_CORES))");
trialArgs' = if trialArgs == null then defaultTrialArgs else trialArgs;
extraTrialArgs = builtins.concatStringsSep " " trialArgs';
testSuite' = if testSuite == null then "_zkapauthorizer" else testSuite;