From 8175f1725460a3d685af903de45a76475ca8a50d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone <exarkun@twistedmatrix.com> Date: Tue, 22 Feb 2022 15:10:14 -0500 Subject: [PATCH] add an endpoint for enabling replication --- src/_zkapauthorizer/_plugin.py | 5 +- src/_zkapauthorizer/recover.py | 13 ++ src/_zkapauthorizer/resource.py | 63 ++++++- src/_zkapauthorizer/tests/matchers.py | 19 +++ .../tests/test_client_resource.py | 158 +++++++++++++++++- src/_zkapauthorizer/tests/test_matchers.py | 52 +++++- 6 files changed, 294 insertions(+), 16 deletions(-) diff --git a/src/_zkapauthorizer/_plugin.py b/src/_zkapauthorizer/_plugin.py index 416dc11..a525108 100644 --- a/src/_zkapauthorizer/_plugin.py +++ b/src/_zkapauthorizer/_plugin.py @@ -46,7 +46,7 @@ from .controller import get_redeemer from .lease_maintenance import SERVICE_NAME as MAINTENANCE_SERVICE_NAME from .lease_maintenance import lease_maintenance_service, maintain_leases_from_root from .model import VoucherStore -from .recover import make_fail_downloader +from .recover import fail_setup_replication, make_fail_downloader from .resource import from_configuration as resource_from_configuration from .server.spending import get_spender from .spending import SpendingController @@ -189,10 +189,13 @@ class ZKAPAuthorizer(object): ) ) + setup_replication = fail_setup_replication + return resource_from_configuration( node_config, store=self._get_store(node_config), get_downloader=get_downloader, + setup_replication=setup_replication, redeemer=self._get_redeemer(node_config, None, reactor), clock=reactor, ) diff --git a/src/_zkapauthorizer/recover.py b/src/_zkapauthorizer/recover.py index ae6ab83..082077f 100644 --- a/src/_zkapauthorizer/recover.py +++ b/src/_zkapauthorizer/recover.py @@ -40,6 +40,12 @@ class AlreadyRecovering(Exception): """ +class ReplicationAlreadySetup(Exception): + """ + An attempt was made to setup of replication but it is already set up. + """ + + class RecoveryStages(Enum): """ Constants representing the different stages a recovery process may have @@ -232,3 +238,10 @@ def get_tahoe_lafs_downloader( return downloader return get_downloader + + +async def fail_setup_replication(): + """ + A replication setup function that always fails. + """ + raise Exception("Test not set up for replication") diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index fb36aec..a9e7038 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -21,13 +21,20 @@ vouchers for fresh tokens. In the future it should also allow users to read statistics about token usage. """ +from collections.abc import Awaitable from json import loads from typing import Callable from attr import Factory, define, field -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.logger import Logger -from twisted.web.http import ACCEPTED, BAD_REQUEST, CONFLICT, INTERNAL_SERVER_ERROR +from twisted.web.http import ( + ACCEPTED, + BAD_REQUEST, + CONFLICT, + CREATED, + INTERNAL_SERVER_ERROR, +) from twisted.web.resource import ErrorPage, IResource, NoResource, Resource from twisted.web.server import NOT_DONE_YET from zope.interface import Attribute @@ -41,7 +48,7 @@ from .controller import PaymentController, get_redeemer from .model import NotEmpty, VoucherStore from .pricecalculator import PriceCalculator from .private import create_private_tree -from .recover import Downloader, StatefulRecoverer +from .recover import Downloader, ReplicationAlreadySetup, StatefulRecoverer from .storage_common import ( get_configured_allowed_public_keys, get_configured_pass_value, @@ -91,6 +98,7 @@ def from_configuration( node_config, store, get_downloader, + setup_replication, redeemer=None, clock=None, ): @@ -151,6 +159,7 @@ def from_configuration( store, controller, get_downloader, + setup_replication, calculate_price, ), ) @@ -159,6 +168,48 @@ def from_configuration( return root +@define +class ReplicateResource(Resource): + """ + Integrate the replication configuration implementation with the HTTP + interface. + + :ivar _setup: The callable the resource will use to do the actual setup + work. + """ + + _setup: Callable[[], Awaitable] + + _log = Logger() + + def __attrs_post_init__(self): + Resource.__init__(self) + + def render_POST(self, request): + self._setup_replication(request) + return NOT_DONE_YET + + @inlineCallbacks + def _setup_replication(self, request): + """ + Call the replication setup function and asynchronously deliver its result + as a response to the given request. + """ + try: + cap_str = yield Deferred.fromCoroutine(self._setup()) + except ReplicationAlreadySetup: + request.setResponseCode(CONFLICT) + except: + self._log.error("replication setup failed") + request.setResponseCode(INTERNAL_SERVER_ERROR) + else: + application_json(request) + request.setResponseCode(CREATED) + request.write(dumps_utf8({"recovery-capability": cap_str})) + + request.finish() + + @define class RecoverResource(Resource): """ @@ -228,6 +279,7 @@ def authorizationless_resource_tree( store, controller, get_downloader: Callable[[str], Downloader], + setup_replication: Callable[[], str], calculate_price, ): """ @@ -250,7 +302,10 @@ def authorizationless_resource_tree( b"recover", RecoverResource(store, get_downloader), ) - + root.putChild( + b"replicate", + ReplicateResource(setup_replication), + ) root.putChild( b"voucher", _VoucherCollection( diff --git a/src/_zkapauthorizer/tests/matchers.py b/src/_zkapauthorizer/tests/matchers.py index 3a3399c..a5b05f0 100644 --- a/src/_zkapauthorizer/tests/matchers.py +++ b/src/_zkapauthorizer/tests/matchers.py @@ -26,6 +26,7 @@ __all__ = [ ] from datetime import datetime +from json import loads import attr from testtools.matchers import ( @@ -228,3 +229,21 @@ def matches_spent_passes(public_key_hash, spent_passes): } ), ) + + +def matches_json(matcher=Always()): + """ + Return a matcher for a JSON string which can be decoded to an object + matched by the given matcher. + """ + + class Matcher: + def match(self, s): + try: + value = loads(s) + except Exception as e: + return Mismatch(f"Failed to decode {str(s)[:80]!r}: {e}") + + return matcher.match(value) + + return Matcher() diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index 0dbec79..ff5d7ad 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -66,7 +66,16 @@ from treq.testing import RequestTraversalAgent from twisted.internet.task import Clock, Cooperator from twisted.python.filepath import FilePath from twisted.web.client import FileBodyProducer, readBody -from twisted.web.http import BAD_REQUEST, NOT_FOUND, NOT_IMPLEMENTED, OK, UNAUTHORIZED +from twisted.web.http import ( + BAD_REQUEST, + CONFLICT, + CREATED, + INTERNAL_SERVER_ERROR, + NOT_FOUND, + NOT_IMPLEMENTED, + OK, + UNAUTHORIZED, +) from twisted.web.http_headers import Headers from .. import NAME @@ -86,7 +95,12 @@ from ..model import ( memory_connect, ) from ..pricecalculator import PriceCalculator -from ..recover import make_fail_downloader, noop_downloader +from ..recover import ( + ReplicationAlreadySetup, + fail_setup_replication, + make_fail_downloader, + noop_downloader, +) from ..resource import NUM_TOKENS, from_configuration, get_token_count from ..storage_common import ( get_configured_allowed_public_keys, @@ -94,7 +108,7 @@ from ..storage_common import ( required_passes, ) from .json import loads -from .matchers import between, matches_response +from .matchers import between, matches_json, matches_response from .strategies import ( api_auth_tokens, client_doublespendredeemer_configurations, @@ -203,6 +217,7 @@ def root_from_config( config, now, get_downloader=get_fail_downloader, + setup_replication=fail_setup_replication, ): """ Create a client root resource from a Tahoe-LAFS configuration. @@ -222,6 +237,7 @@ def root_from_config( memory_connect, ), get_downloader=get_downloader, + setup_replication=setup_replication, clock=Clock(), ) @@ -481,8 +497,7 @@ class ResourceTests(TestCase): succeeded( matches_response( code_matcher=Equals(OK), - body_matcher=AfterPreprocessing( - loads, + body_matcher=matches_json( Equals({"version": zkapauthorizer_version}), ), ), @@ -490,6 +505,133 @@ class ResourceTests(TestCase): ) +class ReplicateTests(TestCase): + """ + Tests for the ``/replicate`` endpoint. + """ + + @given( + tahoe_configs(), + api_auth_tokens(), + ) + def test_already_configured(self, get_config, api_auth_token): + """ + If replication has already been configured then the endpoint returns a + response with a 409 status code. + """ + config = get_config_with_api_token( + self.useFixture(TempDir()), + get_config, + api_auth_token, + ) + + async def setup_replication(): + raise ReplicationAlreadySetup() + + root = root_from_config( + config, datetime.now, setup_replication=setup_replication + ) + agent = RequestTraversalAgent(root) + configuring = authorized_request( + api_auth_token, + agent, + b"POST", + b"http://127.0.0.1/replicate", + ) + self.assertThat( + configuring, + succeeded( + matches_response( + code_matcher=Equals(CONFLICT), + ), + ), + ) + + @given( + tahoe_configs(), + api_auth_tokens(), + ) + def test_internal_server_error(self, get_config, api_auth_token): + """ + If there is an unexpected exception setting up replication then the + endpoint returns a response with a 500 status code. + """ + config = get_config_with_api_token( + self.useFixture(TempDir()), + get_config, + api_auth_token, + ) + + async def setup_replication(): + raise Exception("surprise bug") + + root = root_from_config( + config, datetime.now, setup_replication=setup_replication + ) + agent = RequestTraversalAgent(root) + configuring = authorized_request( + api_auth_token, + agent, + b"POST", + b"http://127.0.0.1/replicate", + ) + self.assertThat( + configuring, + succeeded( + matches_response( + code_matcher=Equals(INTERNAL_SERVER_ERROR), + ), + ), + ) + + @given( + tahoe_configs(), + api_auth_tokens(), + ) + def test_created(self, get_config, api_auth_token): + """ + On successful replica configuration, the endpoint returns a response with + a 201 status code and an application/json-encoded body containing a + read-only directory capability. + """ + config = get_config_with_api_token( + self.useFixture(TempDir()), + get_config, + api_auth_token, + ) + cap_ro = "URI:DIR2-RO:aaaa:bbbb" + + async def setup_replication(): + return cap_ro + + root = root_from_config( + config, datetime.now, setup_replication=setup_replication + ) + agent = RequestTraversalAgent(root) + configuring = authorized_request( + api_auth_token, + agent, + b"POST", + b"http://127.0.0.1/replicate", + ) + self.assertThat( + configuring, + succeeded( + matches_response( + code_matcher=Equals(CREATED), + headers_matcher=application_json(), + body_matcher=matches_json( + Equals( + { + "recovery-capability": cap_ro, + } + ), + ), + ), + ), + ) + + class RecoverTests(TestCase): """ Tests for the ``/recover`` endpoint. @@ -746,8 +888,7 @@ class RecoverTests(TestCase): matches_response( code_matcher=Equals(OK), headers_matcher=application_json(), - body_matcher=AfterPreprocessing( - loads, + body_matcher=matches_json( Equals( { "stage": "download_failed", @@ -1605,8 +1746,7 @@ class CalculatePriceTests(TestCase): matches_response( code_matcher=Equals(OK), headers_matcher=application_json(), - body_matcher=AfterPreprocessing( - loads, + body_matcher=matches_json( Equals( { "price": expected_price, diff --git a/src/_zkapauthorizer/tests/test_matchers.py b/src/_zkapauthorizer/tests/test_matchers.py index d97ac1a..4d31d16 100644 --- a/src/_zkapauthorizer/tests/test_matchers.py +++ b/src/_zkapauthorizer/tests/test_matchers.py @@ -16,11 +16,13 @@ Tests for ``_zkapauthorizer.tests.matchers``. """ +from json import dumps + from testtools import TestCase -from testtools.matchers import Is, Not +from testtools.matchers import Always, Equals, Is, Not from zope.interface import Interface, implementer -from .matchers import Provides, returns +from .matchers import Provides, matches_json, returns class IX(Interface): @@ -94,3 +96,49 @@ class ReturnsTests(TestCase): returns(Is(result)).match(lambda: other), Not(Is(None)), ) + + +class MatchesJSONTests(TestCase): + """ + Tests for ``matches_json``. + """ + + def test_non_string(self): + """ + If the value given isn't a string then ``matches_json`` does not match. + """ + self.assertThat( + matches_json(Always()).match(object()), + Not(Is(None)), + ) + + def test_unparseable(self): + """ + If the value can't be parsed as JSON then ``matches_json`` does not match. + """ + self.assertThat( + matches_json(Always()).match("not json"), + Not(Is(None)), + ) + + def test_does_not_match(self): + """ + If the parsed value isn't matched by the given matcher then + ``matches_json`` does not match. + """ + expected = {"hello": "world"} + self.assertThat( + matches_json(Not(Equals(expected))).match(dumps(expected)), + Not(Is(None)), + ) + + def test_matches(self): + """ + If the parsed value is matched by the given matcher then ``matches_json`` + matches. + """ + expected = {"hello": "world"} + self.assertThat( + matches_json(Equals(expected)).match(dumps(expected)), + Is(None), + ) -- GitLab