diff --git a/setup.cfg b/setup.cfg
index dfc4960747ba2d57bea9b12deff72f59371cd743..42dd1649cf7c1b05292f9e6f17647bec69143f59 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -42,6 +42,7 @@ install_requires =
Twisted[tls,conch]>=18.4.0
tahoe-lafs==1.14.0
treq
+ pyutil
[versioneer]
VCS = git
diff --git a/src/_zkapauthorizer/pricecalculator.py b/src/_zkapauthorizer/pricecalculator.py
new file mode 100644
index 0000000000000000000000000000000000000000..007ec9cdf212839ddabbac9555308b29fb158483
--- /dev/null
+++ b/src/_zkapauthorizer/pricecalculator.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 PrivateStorage.io, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Calculate the price, in ZKAPs, for storing files.
+
+The underlying storage system operates only on individual shares. Thus, it
+*does not* use this file-oriented calculator. However, for end-users,
+file-oriented pricing is often more helpful. This calculator builds on the
+share-oriented price calculation to present file-oriented price information.
+
+It accounts for erasure encoding data expansion. It does not account for the
+real state of the storage system (e.g., if some data is *already* stored then
+storing it "again" is essentially free but this will not be reflected by this
+calculator).
+"""
+
+import attr
+
+from .storage_common import (
+ required_passes,
+ share_size_for_data,
+)
+
+@attr.s
+class PriceCalculator(object):
+ """
+ :ivar int _shares_needed: The number of shares which are required to
+ reconstruct the original data.
+
+ :ivar int _shares_total: The total number of shares which will be
+ produced in the erasure encoding process.
+
+ :ivar int _pass_value: The bytes component of the bytes×time value of a
+ single pass.
+ """
+ _shares_needed = attr.ib()
+ _shares_total = attr.ib()
+ _pass_value = attr.ib()
+
+ def calculate(self, sizes):
+ """
+ Calculate the price to store data of the given sizes for one lease
+ period.
+
+ :param [int] sizes: The sizes of the individual data items in bytes.
+
+ :return int: The number of ZKAPs required.
+ """
+ share_sizes = (
+ share_size_for_data(self._shares_needed, size)
+ for size
+ in sizes
+ )
+ all_required_passes = (
+ required_passes(self._pass_value, [share_size] * self._shares_total)
+ for share_size
+ in share_sizes
+ )
+ price = sum(all_required_passes, 0)
+ return price
diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py
index 20f1519d0196268b1886b0b3e9938c807886dbac..fae91cd2aee22e7809e2472dce909658bfb17eb5 100644
--- a/src/_zkapauthorizer/resource.py
+++ b/src/_zkapauthorizer/resource.py
@@ -61,11 +61,16 @@ from ._base64 import (
)
from .storage_common import (
- required_passes,
+ get_configured_shares_needed,
+ get_configured_shares_total,
get_configured_pass_value,
get_configured_lease_duration,
)
+from .pricecalculator import (
+ PriceCalculator,
+)
+
from .controller import (
PaymentController,
get_redeemer,
@@ -117,13 +122,23 @@ def from_configuration(node_config, store, redeemer=None, default_token_count=No
if default_token_count is None:
default_token_count = NUM_TOKENS
controller = PaymentController(store, redeemer, default_token_count)
+
+ calculator = PriceCalculator(
+ get_configured_shares_needed(node_config),
+ get_configured_shares_total(node_config),
+ get_configured_pass_value(node_config),
+ )
+ calculate_price = _CalculatePrice(
+ calculator,
+ get_configured_lease_duration(node_config),
+ )
+
root = create_private_tree(
lambda: node_config.get_private_config(b"api_auth_token"),
authorizationless_resource_tree(
store,
controller,
- get_configured_pass_value(node_config),
- get_configured_lease_duration(node_config),
+ calculate_price,
),
)
root.store = store
@@ -131,15 +146,19 @@ def from_configuration(node_config, store, redeemer=None, default_token_count=No
return root
-def authorizationless_resource_tree(store, controller, pass_value, lease_duration):
+def authorizationless_resource_tree(
+ store,
+ controller,
+ calculate_price,
+):
"""
Create the full ZKAPAuthorizer client plugin resource hierarchy with no
authorization applied.
:param VoucherStore store: The store to use.
:param PaymentController controller: The payment controller to use.
- :param int pass_value: The bytes component of the bytes×time value of a single pass.
- :param int lease_duration: The number of seconds a lease will be valid.
+
+ :param IResource calculate_price: The resource for the price calculation endpoint.
:return IResource: The root of the resource hierarchy.
"""
@@ -164,7 +183,7 @@ def authorizationless_resource_tree(store, controller, pass_value, lease_duratio
)
root.putChild(
b"calculate-price",
- _CalculatePrice(pass_value, lease_duration),
+ calculate_price,
)
return root
@@ -177,12 +196,14 @@ class _CalculatePrice(Resource):
render_HEAD = render_GET = None
- def __init__(self, pass_value, lease_period):
+ def __init__(self, price_calculator, lease_period):
"""
- :param pass_value: See ``authorizationless_resource_tree``
+ :param _PriceCalculator price_calculator: The object which can actually
+ calculate storage prices.
+
:param lease_period: See ``authorizationless_resource_tree``
"""
- self._pass_value = pass_value
+ self._price_calculator = price_calculator
self._lease_period = lease_period
Resource.__init__(self)
@@ -219,15 +240,22 @@ class _CalculatePrice(Resource):
"error": "did not find required version number 1 in request",
})
- if not isinstance(sizes, list) or not all(isinstance(size, (int, long)) and size >= 0 for size in sizes):
+ if (not isinstance(sizes, list) or
+ not all(
+ isinstance(size, (int, long)) and size >= 0
+ for size
+ in sizes
+ )):
request.setResponseCode(BAD_REQUEST)
return dumps({
"error": "did not find required positive integer sizes list in request",
})
application_json(request)
+
+ price = self._price_calculator.calculate(sizes)
return dumps({
- u"price": required_passes(self._pass_value, sizes),
+ u"price": price,
u"period": self._lease_period,
})
diff --git a/src/_zkapauthorizer/storage_common.py b/src/_zkapauthorizer/storage_common.py
index 38a46d50b3afd253857c054abf38fe212ef37cd6..dd9a9f49e15bd542aa70cbbbdaadedfde5c64bbb 100644
--- a/src/_zkapauthorizer/storage_common.py
+++ b/src/_zkapauthorizer/storage_common.py
@@ -34,6 +34,10 @@ from .eliot import (
MUTABLE_PASSES_REQUIRED,
)
+from pyutil.mathutil import (
+ div_ceil,
+)
+
@attr.s(frozen=True)
class MorePassesRequired(Exception):
"""
@@ -73,6 +77,34 @@ slot_testv_and_readv_and_writev_message = _message_maker(u"slot_testv_and_readv_
# submitted.
BYTES_PER_PASS = 1024 * 1024
+def get_configured_shares_needed(node_config):
+ """
+ Determine the configured-specified value of "needed" shares (``k``).
+
+ If no value is explicitly configured, the Tahoe-LAFS default (as best as
+ we know it) is returned.
+ """
+ return int(node_config.get_config(
+ section=u"client",
+ option=u"shares.needed",
+ default=3,
+ ))
+
+
+def get_configured_shares_total(node_config):
+ """
+ Determine the configured-specified value of "total" shares (``N``).
+
+ If no value is explicitly configured, the Tahoe-LAFS default (as best as
+ we know it) is returned.
+ """
+ return int(node_config.get_config(
+ section=u"client",
+ option=u"shares.total",
+ default=10,
+ ))
+
+
def get_configured_pass_value(node_config):
"""
Determine the configuration-specified value of a single ZKAP.
@@ -126,6 +158,23 @@ def required_passes(bytes_per_pass, share_sizes):
return result
+def share_size_for_data(shares_needed, datasize):
+ """
+ Calculate the size of a single erasure encoding share for data of the
+ given size and with the given level of redundancy.
+
+ :param int shares_needed: The number of shares (``k``) from the erasure
+ encoding process which are required to reconstruct original data of
+ the indicated size.
+
+ :param int datasize: The size of the data to consider, in bytes.
+
+ :return int: The size of a single erasure encoding share for the given
+ inputs.
+ """
+ return div_ceil(datasize, shares_needed)
+
+
def has_writes(tw_vectors):
"""
:param tw_vectors: See
diff --git a/src/_zkapauthorizer/tests/strategies.py b/src/_zkapauthorizer/tests/strategies.py
index 025260d8be245dd8c703ed30429a12c20cb6ba24..4384cacb8d946d8408cd3e4f5e3bb8f9f8f835a9 100644
--- a/src/_zkapauthorizer/tests/strategies.py
+++ b/src/_zkapauthorizer/tests/strategies.py
@@ -122,10 +122,36 @@ def _config_string_from_sections(divided_sections):
))
-def tahoe_config_texts(storage_client_plugins):
+def tahoe_config_texts(storage_client_plugins, shares):
"""
Build the text of complete Tahoe-LAFS configurations for a node.
+
+ :param storage_client_plugins: A dictionary with storage client plugin
+ names as keys.
+
+ :param shares: A strategy to build erasure encoding parameters. These are
+ built as a three-tuple giving (needed, total, happy). Each element
+ may be an integer or None to leave it unconfigured (and rely on the
+ default).
"""
+ def merge_shares(shares, the_rest):
+ for (k, v) in zip(("needed", "happy", "total"), shares):
+ if v is not None:
+ the_rest["shares." + k] = u"{}".format(v)
+ return the_rest
+
+ client_section = builds(
+ merge_shares,
+ shares,
+ fixed_dictionaries(
+ {
+ "storage.plugins": just(
+ u",".join(storage_client_plugins.keys()),
+ ),
+ },
+ ),
+ )
+
return builds(
lambda *sections: _config_string_from_sections(
sections,
@@ -144,26 +170,23 @@ def tahoe_config_texts(storage_client_plugins):
"nickname": node_nicknames(),
},
),
- "client": fixed_dictionaries(
- {
- "storage.plugins": just(
- u",".join(storage_client_plugins.keys()),
- ),
- },
- ),
+ "client": client_section,
},
),
)
-def minimal_tahoe_configs(storage_client_plugins=None):
+def minimal_tahoe_configs(storage_client_plugins=None, shares=just((None, None, None))):
"""
Build complete Tahoe-LAFS configurations for a node.
+
+ :param shares: See ``tahoe_config_texts``.
"""
if storage_client_plugins is None:
storage_client_plugins = {}
return tahoe_config_texts(
storage_client_plugins,
+ shares,
).map(
lambda config_text: lambda basedir, portnumfile: config_from_string(
basedir,
@@ -287,14 +310,33 @@ def client_errorredeemer_configurations(details):
})
-def tahoe_configs(zkapauthz_v1_configuration=client_dummyredeemer_configurations()):
+def tahoe_configs(
+ zkapauthz_v1_configuration=client_dummyredeemer_configurations(),
+ shares=just((None, None, None)),
+):
"""
Build complete Tahoe-LAFS configurations including the zkapauthorizer
client plugin section.
+
+ :param shares: See ``tahoe_config_texts``.
"""
return minimal_tahoe_configs({
u"privatestorageio-zkapauthz-v1": zkapauthz_v1_configuration,
- })
+ }, shares)
+
+
+def share_parameters():
+ """
+ Build three-tuples of integers giving usable k, happy, N parameters to
+ Tahoe-LAFS' erasure encoding process.
+ """
+ return lists(
+ integers(min_value=1, max_value=255),
+ min_size=3,
+ max_size=3,
+ ).map(
+ sorted,
+ )
def vouchers():
diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py
index 0bbc9bb6418d335d1f93bb5712ace73301024b70..e13d4747e0c2c3246c4ce012b7da911df0970212 100644
--- a/src/_zkapauthorizer/tests/test_client_resource.py
+++ b/src/_zkapauthorizer/tests/test_client_resource.py
@@ -145,6 +145,10 @@ from ..resource import (
from_configuration,
)
+from ..pricecalculator import (
+ PriceCalculator,
+)
+
from ..storage_common import (
required_passes,
get_configured_pass_value,
@@ -163,6 +167,7 @@ from .strategies import (
requests,
request_paths,
api_auth_tokens,
+ share_parameters,
)
from .matchers import (
Provides,
@@ -1508,15 +1513,25 @@ class CalculatePriceTests(TestCase):
)
@given(
- tahoe_configs(),
+ # Make the share encoding parameters easily accessible without going
+ # through the Tahoe-LAFS configuration.
+ share_parameters().flatmap(
+ lambda params: tuples(
+ just(params),
+ tahoe_configs(shares=just(params)),
+ ),
+ ),
api_auth_tokens(),
lists(integers(min_value=0)),
)
- def test_calculated_price(self, get_config, api_auth_token, sizes):
+ def test_calculated_price(self, encoding_params_and_get_config, api_auth_token, sizes):
"""
A well-formed request returns the price in ZKAPs as an integer and the
storage period (the minimum allowed) that they pay for.
"""
+ encoding_params, get_config = encoding_params_and_get_config
+ shares_needed, shares_happy, shares_total = encoding_params
+
config = get_config_with_api_token(
self.useFixture(TempDir()),
get_config,
@@ -1525,10 +1540,11 @@ class CalculatePriceTests(TestCase):
root = root_from_config(config, datetime.now)
agent = RequestTraversalAgent(root)
- expected_price = required_passes(
- get_configured_pass_value(config),
- sizes,
- )
+ expected_price = PriceCalculator(
+ shares_needed=shares_needed,
+ shares_total=shares_total,
+ pass_value=get_configured_pass_value(config),
+ ).calculate(sizes)
self.assertThat(
authorized_request(
diff --git a/src/_zkapauthorizer/tests/test_pricecalculator.py b/src/_zkapauthorizer/tests/test_pricecalculator.py
new file mode 100644
index 0000000000000000000000000000000000000000..25eaa40d09102b66ffbb592d590497c33d1c8517
--- /dev/null
+++ b/src/_zkapauthorizer/tests/test_pricecalculator.py
@@ -0,0 +1,227 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 PrivateStorage.io, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Tests for ``_zkapauthorizer.pricecalculator``.
+"""
+
+from functools import (
+ partial,
+)
+
+from testtools import (
+ TestCase,
+)
+from testtools.matchers import (
+ Equals,
+ GreaterThan,
+ IsInstance,
+ MatchesAll,
+)
+
+from hypothesis import (
+ given,
+)
+
+from hypothesis.strategies import (
+ integers,
+ lists,
+ tuples,
+)
+
+from ..pricecalculator import (
+ PriceCalculator,
+)
+
+from .strategies import (
+ sizes,
+ share_parameters,
+)
+from .matchers import (
+ greater_or_equal,
+)
+
+file_sizes = lists(sizes(), min_size=1)
+
+class PriceCalculatorTests(TestCase):
+ """
+ Tests for ``PriceCalculator``.
+ """
+ @given(
+ integers(min_value=1),
+ integers(min_value=1),
+ file_sizes,
+ )
+ def test_pass_value(self, pass_value, more_value, file_sizes):
+ """
+ The result of ``PriceCalculator.calculate`` increases or remains the same
+ as pass value decreases.
+ """
+ calculator = partial(PriceCalculator, shares_needed=1, shares_total=1)
+ less_value = calculator(pass_value=pass_value)
+ more_value = calculator(pass_value=pass_value + more_value)
+
+ less_value_price = less_value.calculate(file_sizes)
+ more_value_price = more_value.calculate(file_sizes)
+
+ self.assertThat(
+ less_value_price,
+ greater_or_equal(more_value_price),
+ )
+
+ @given(
+ integers(min_value=1, max_value=127),
+ integers(min_value=1, max_value=127),
+ file_sizes,
+ )
+ def test_shares_needed(self, shares_needed, more_needed, file_sizes):
+ """
+ The result of ``PriceCalculator.calculate`` never increases as
+ ``shares_needed`` increases.
+ """
+ calculator = partial(PriceCalculator, pass_value=100, shares_total=255)
+ fewer_needed = calculator(shares_needed=shares_needed)
+ more_needed = calculator(shares_needed=shares_needed + more_needed)
+
+ fewer_needed_price = fewer_needed.calculate(file_sizes)
+ more_needed_price = more_needed.calculate(file_sizes)
+
+ self.assertThat(
+ fewer_needed_price,
+ greater_or_equal(more_needed_price),
+ )
+
+
+ @given(
+ integers(min_value=1, max_value=127),
+ integers(min_value=1, max_value=127),
+ file_sizes,
+ )
+ def test_shares_total(self, shares_total, more_total, file_sizes):
+ """
+ The result of ``PriceCalculator.calculate`` always increases as
+ ``shares_total`` increases.
+ """
+ calculator = partial(PriceCalculator, pass_value=100, shares_needed=1)
+ fewer_total = calculator(shares_total=shares_total)
+ more_total = calculator(shares_total=shares_total + more_total)
+
+ fewer_total_price = fewer_total.calculate(file_sizes)
+ more_total_price = more_total.calculate(file_sizes)
+
+ self.assertThat(
+ more_total_price,
+ greater_or_equal(fewer_total_price),
+ )
+
+ @given(
+ integers(min_value=1, max_value=100).flatmap(
+ lambda num_files: tuples(
+ lists(sizes(), min_size=num_files, max_size=num_files),
+ lists(sizes(), min_size=num_files, max_size=num_files),
+ ),
+ ),
+ integers(min_value=1),
+ share_parameters(),
+ )
+ def test_file_sizes(self, file_sizes, pass_value, parameters):
+ """
+ The result of ``PriceCalculator.calculate`` never decreases as the values
+ of ``file_sizes`` increase.
+ """
+ smaller_sizes, increases = file_sizes
+ larger_sizes = list(a + b for (a, b) in zip(smaller_sizes, increases))
+ k, happy, N = parameters
+
+ calculator = PriceCalculator(
+ pass_value=pass_value,
+ shares_needed=k,
+ shares_total=N,
+ )
+
+ smaller_sizes_price = calculator.calculate(smaller_sizes)
+ larger_sizes_price = calculator.calculate(larger_sizes)
+
+ self.assertThat(
+ larger_sizes_price,
+ greater_or_equal(smaller_sizes_price),
+ )
+
+ @given(
+ integers(min_value=1),
+ share_parameters(),
+ file_sizes,
+ )
+ def test_positive_integer_price(self, pass_value, parameters, file_sizes):
+ """
+ The result of ``PriceCalculator.calculate`` for a non-empty size list is
+ always a positive integer.
+ """
+ k, happy, N = parameters
+ calculator = PriceCalculator(
+ pass_value=pass_value,
+ shares_needed=k,
+ shares_total=N,
+ )
+ price = calculator.calculate(file_sizes)
+ self.assertThat(
+ price,
+ MatchesAll(
+ IsInstance((int, long)),
+ GreaterThan(0),
+ ),
+ )
+
+ @given(
+ integers(min_value=1),
+ share_parameters(),
+ file_sizes,
+ )
+ def test_linear_increase(self, pass_value, parameters, file_sizes):
+ """
+ The result of ``PriceCalculator.calculate`` doubles if the file size list
+ is doubled.
+ """
+ k, happy, N = parameters
+ calculator = PriceCalculator(
+ pass_value=pass_value,
+ shares_needed=k,
+ shares_total=N,
+ )
+ smaller_price = calculator.calculate(file_sizes)
+ larger_price = calculator.calculate(file_sizes + file_sizes)
+ self.assertThat(
+ larger_price,
+ Equals(smaller_price * 2),
+ )
+
+ @given(
+ integers(min_value=1),
+ )
+ def test_one_pass(self, pass_value):
+ """
+ The result of ``PriceCalculator.calculate`` is exactly ``1`` if the amount
+ of data to be stored equals the value of a pass.
+ """
+ calculator = PriceCalculator(
+ pass_value=pass_value,
+ shares_needed=1,
+ shares_total=1,
+ )
+ price = calculator.calculate([pass_value])
+ self.assertThat(
+ price,
+ Equals(1),
+ )
diff --git a/src/_zkapauthorizer/tests/test_strategies.py b/src/_zkapauthorizer/tests/test_strategies.py
index 6a0307ea4b799204dba2c4f297473a0d52619215..3f60fcd8a617e344e20b7c5e2bb496143e07d571 100644
--- a/src/_zkapauthorizer/tests/test_strategies.py
+++ b/src/_zkapauthorizer/tests/test_strategies.py
@@ -34,6 +34,8 @@ from hypothesis import (
)
from hypothesis.strategies import (
data,
+ one_of,
+ just,
)
from allmydata.client import (
@@ -42,6 +44,7 @@ from allmydata.client import (
from .strategies import (
tahoe_config_texts,
+ share_parameters,
)
class TahoeConfigsTests(TestCase):
@@ -54,7 +57,15 @@ class TahoeConfigsTests(TestCase):
Configurations built by the strategy can be parsed.
"""
tempdir = self.useFixture(TempDir())
- config_text = data.draw(tahoe_config_texts({}))
+ config_text = data.draw(
+ tahoe_config_texts(
+ storage_client_plugins={},
+ shares=one_of(
+ just((None, None, None)),
+ share_parameters(),
+ ),
+ ),
+ )
note(config_text)
config_from_string(
tempdir.join(b"tahoe.ini"),