diff --git a/default.nix b/default.nix
index bc6a0884559432f6b8c0c65473530de99154bbd5..8400053d9a5bb70d72475c3bef67806346585aa4 100644
--- a/default.nix
+++ b/default.nix
@@ -31,6 +31,7 @@ in
# This is kind of round-about but it seems to be the best way to
# convince mach-nix to use a specific package for a specific dependency.
tahoe-lafs = "nixpkgs";
+ zkap-spending-service = "nixpkgs";
# Make sure we use an sdist of zfec so that our patch to zfec's setup.py
# to remove its argparse dependency can be applied. If we get a wheel,
diff --git a/nix/sources.json b/nix/sources.json
index 658a129cece37115b20115201342083508e51732..593ceb1121bbd4c204ccd37e3601203ac5393167 100644
--- a/nix/sources.json
+++ b/nix/sources.json
@@ -76,5 +76,13 @@
"type": "tarball",
"url": "https://github.com/tahoe-lafs/tahoe-lafs/archive/5e8cc06e93e7b23dae0f1c3e0bc158b49a9e14e6.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+ },
+ "zkap-spending-service": {
+ "owner": "tomprince",
+ "rev": "c959529a7d416e5c943614dc47a97ad770a8cd46",
+ "sha256": "14311w257p2dcjc7dam8fqam2jr73ywfm7xzgc6bnbjqv0i0d3z3",
+ "type": "tarball",
+ "url": "https://whetstone.private.storage/tomprince/zkap-spending-service/-/archive/c959529a7d416e5c943614dc47a97ad770a8cd46/zkap-spending-service.zip",
+ "url_template": "https://whetstone.private.storage/<owner>/zkap-spending-service/-/archive/<rev>/zkap-spending-service.zip"
}
}
diff --git a/requirements/test.in b/requirements/test.in
index f8f848c4c8f9dbed99d1f42e98c79aa2e9f5a2e4..5fc456fbc0cd0c12509d5e6b5f3cf01b5af928a9 100644
--- a/requirements/test.in
+++ b/requirements/test.in
@@ -4,6 +4,7 @@ testtools
testresources
hypothesis
openapi_spec_validator
+zkap-spending-service@https://whetstone.private.storage/tomprince/zkap-spending-service/-/archive/c959529a7d416e5c943614dc47a97ad770a8cd46/zkap-spending-service.zip
# Lint requirements
# Pin these narrowly so that lint rules only change when we specifically
diff --git a/src/_zkapauthorizer/server/spending.py b/src/_zkapauthorizer/server/spending.py
index 66fe819a3b768ac48842b7b649aec52b4f987a62..aba1ad7a32073abf1b9c08733c18da93c52da343 100644
--- a/src/_zkapauthorizer/server/spending.py
+++ b/src/_zkapauthorizer/server/spending.py
@@ -1,18 +1,60 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 PrivateStorage.io, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+A client for the ZKAP Spending Service, which records spent ZKAPs.
+"""
+
+from __future__ import annotations
+
+import json
from typing import Any
-import attr
+import attrs
from challenge_bypass_ristretto import PublicKey
-from prometheus_client import CollectorRegistry
+from eliot import start_action
+from eliot.twisted import inline_callbacks
+from hyperlink import URL
+from prometheus_client import CollectorRegistry, Counter
+from treq.client import HTTPClient
+from treq.testing import StubTreq
from twisted.internet.interfaces import IReactorTime
+from twisted.web import http
+from twisted.web.client import Agent
from zope.interface import Interface, implementer
+from ..eliot import register_attr_exception
+
+
+@register_attr_exception
+@attrs.define
+class UnexpectedResponse(Exception):
+ """
+ The issuer responded in an unexpected and unhandled way.
+ """
+
+ code: int
+ body: bytes
+
class ISpender(Interface):
"""
An ``ISpender`` can records spent ZKAPs and reports double spends.
"""
- def mark_as_spent(public_key, passes):
+ def mark_as_spent(public_key, token_preimages):
# type: (PublicKey, list[bytes]) -> None
"""
Record the given ZKAPs (associated to the given public key as having
@@ -25,40 +67,124 @@ class ISpender(Interface):
"""
-@attr.s
-class _SpendingData(object):
- spent_tokens = attr.ib(init=False, factory=dict)
+def counter_attr(name, description, labels=()):
+ """
+ Return an attrs attribute that is a prometheus :py:`Counter` metric registered
+ with the ``_registry`` on the instance.
+ """
+ attrib = attrs.field(
+ init=False,
+ metadata={
+ "metric-labels": labels,
+ "metric-name": "{}".format(name),
+ },
+ )
+
+ @attrib.default
+ def make_counter(self):
+ return Counter(name, description, labelnames=labels, registry=self._registry)
- def reset(self):
- self.spent_tokens.clear()
+ return attrib
@implementer(ISpender)
-@attr.s
-class RecordingSpender(object):
+@attrs.define
+class Spender(object):
"""
- An in-memory :py:`ISpender` implementation that exposes the spent tokens
- for testing purposes.
+ An :py:`ISpender` that talks to a ZKAP Spending Service.
"""
- _recorder = attr.ib(validator=attr.validators.instance_of(_SpendingData))
+ _treq: HTTPClient = attrs.field(
+ validator=attrs.validators.instance_of((HTTPClient, StubTreq))
+ )
+ _api_root: URL = attrs.field(validator=attrs.validators.instance_of(URL))
+ _registry: CollectorRegistry = attrs.field()
@classmethod
- def make(cls):
- # type: () -> (_SpendingData, ISpender)
- recorder = _SpendingData()
- return recorder, cls(recorder)
-
- def mark_as_spent(self, public_key, passes):
- self._recorder.spent_tokens.setdefault(public_key.encode_base64(), []).extend(
- passes
+ def make(
+ cls, config: dict[str, Any], reactor: IReactorTime, registry: CollectorRegistry
+ ) -> ISpender:
+ spending_service_url = config.pop("spending-service-url")
+ return cls(
+ HTTPClient(Agent(reactor)),
+ URL.from_text(spending_service_url),
+ registry,
)
+ @inline_callbacks
+ def ping(self) -> None:
+ response = yield self._treq.get(self._api_root.child("v1", "_ping").to_text())
-def get_spender(config, reactor, registry):
- # type: (dict[str, Any], IReactorTime, CollectorRegistry) -> ISpender
- """
- Return an :py:`ISpender` to be used with the given storage server configuration.
- """
- recorder, spender = RecordingSpender.make()
- return spender
+ response_body = yield response.content()
+ if response.code != http.OK:
+ raise Exception("Not ok")
+
+ try:
+ result = json.loads(response_body)
+ except ValueError:
+ raise UnexpectedResponse(response.code, response_body)
+
+ if result.get("status") != "ok":
+ raise Exception("Didn't get ping from spending service.")
+
+ SPEND_PASSES = counter_attr("zkapauthorizer_spend_passes_total", "FIXME: DESC")
+ SPEND_PASSES_ERRORS = counter_attr(
+ "zkapauthorizer_spend_passes_error_total",
+ "FIXME: DESC",
+ labels=("reason", "code"),
+ )
+ SPEND_PASSES_FAILURES = counter_attr(
+ "zkapauthorizer_spend_passes_failures_total",
+ "FIXME: DESC",
+ )
+ SPEND_PASSES_SUCCESSES = counter_attr(
+ "zkapauthorizer_spend_passes_successes_total",
+ "FIXME: DESC",
+ )
+
+ @inline_callbacks
+ def mark_as_spent(self, public_key, token_preimages):
+ """
+ Takes a dictionary mapping public keys to lists of spend tokens,
+ and reports them as spent.
+ """
+ self.SPEND_PASSES.inc()
+ try:
+ with start_action(
+ action_type="zkapauthorizer:server:spend-passes"
+ ) as action:
+ response = yield self._treq.post(
+ self._api_root.child("v1", "spend").to_text(),
+ json.dumps(
+ {
+ "tokens": {
+ public_key.encode_base64().decode("ascii"): [
+ token_preimage.decode("ascii")
+ for token_preimage in token_preimages
+ ]
+ },
+ "force": True,
+ }
+ ),
+ headers={b"content-type": b"application/json"},
+ )
+ try:
+ result = yield response.json()
+ except:
+ raise UnexpectedResponse(response.code, "")
+
+ if response.code != http.OK:
+ self.SPEND_PASSES_ERRORS.labels(
+ reason=result["reason"], code=response.code
+ ).inc()
+ else:
+ action.add_success_fields(code=response.code, body=result)
+ self.SPEND_PASSES_SUCCESSES.inc()
+
+ except Exception:
+ self.SPEND_PASSES_FAILURES.inc()
+ # eliot will have logged any exception above. Since we want to
+ # fail open, we consume exceptions here.
+
+
+get_spender = Spender.make
diff --git a/src/_zkapauthorizer/tests/fixtures.py b/src/_zkapauthorizer/tests/fixtures.py
index b5fb82186f840c6106dfb247bc3e3822f5bd63e8..bb909441aa341823deb6ebd90d38ce030632f091 100644
--- a/src/_zkapauthorizer/tests/fixtures.py
+++ b/src/_zkapauthorizer/tests/fixtures.py
@@ -22,6 +22,8 @@ import attr
from allmydata.storage.server import StorageServer
from attrs import define, field
from fixtures import Fixture, TempDir
+from hyperlink import URL
+from prometheus_client import CollectorRegistry
from testtools import TestCase
from treq.client import HTTPClient
from twisted.internet.defer import Deferred, inlineCallbacks
@@ -29,9 +31,11 @@ from twisted.internet.interfaces import IReactorTime
from twisted.internet.task import Clock, deferLater
from twisted.python.filepath import FilePath
from twisted.web.client import Agent, HTTPConnectionPool
+from zss.testing import InMemoryBackend, make_in_memory_client
from ..controller import DummyRedeemer, PaymentController
from ..model import VoucherStore, memory_connect, open_and_initialize
+from ..server.spending import ISpender, Spender
@attr.s(auto_attribs=True)
@@ -186,3 +190,15 @@ class Treq(Fixture):
# reactor) seems to be enough. If it's not, sorry.
yield deferLater(self.reactor, 0, lambda: None)
yield deferLater(self.reactor, 0, lambda: None)
+
+
+def make_in_memory_spender(
+ registry: CollectorRegistry = None,
+) -> (InMemoryBackend, ISpender):
+
+ backend, treq = make_in_memory_client()
+ return backend, Spender(
+ treq,
+ URL.from_text("http://spender.invalid/"),
+ CollectorRegistry(),
+ )
diff --git a/src/_zkapauthorizer/tests/matchers.py b/src/_zkapauthorizer/tests/matchers.py
index 3a3399c8cefc409d62ab66ec8139bb09adb11c44..78e1a62d548239b92ce1b2f4b59e487907d98531 100644
--- a/src/_zkapauthorizer/tests/matchers.py
+++ b/src/_zkapauthorizer/tests/matchers.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import annotations
+
"""
Testtools matchers useful for the test suite.
"""
@@ -26,8 +28,11 @@ __all__ = [
]
from datetime import datetime
+from typing import Any
import attr
+import zss.store
+from prometheus_client import CollectorRegistry
from testtools.matchers import (
AfterPreprocessing,
AllMatch,
@@ -48,7 +53,6 @@ from testtools.twistedsupport import succeeded
from treq import content
from ..model import Pass
-from ..server.spending import _SpendingData
from ._exception import raises
@@ -212,14 +216,20 @@ def matches_response(
)
-def matches_spent_passes(public_key_hash, spent_passes):
- # type: (bytes, list[Pass]) -> Matcher[_SpendingData]
+def matches_spent_passes(
+ public_key_hash: bytes, spent_passes: list[Pass]
+) -> Matcher[zss.store.InMemoryBackend]:
"""
Returns a matcher for _SpendingData that checks whether the
spent pass match the given public key and passes.
"""
return AfterPreprocessing(
- lambda spending_recorder: spending_recorder.spent_tokens,
+ lambda spending_recorder: {
+ public_key.encode("ascii"): [
+ preimage.encode("ascii") for preimage in token_preimages
+ ]
+ for public_key, token_preimages in spending_recorder.spent_tokens.items()
+ },
MatchesDict(
{
public_key_hash: MatchesSetwise(
@@ -228,3 +238,28 @@ def matches_spent_passes(public_key_hash, spent_passes):
}
),
)
+
+
+def matches_metrics(
+ expected: dict[tuple[type, str], Any],
+) -> Matcher[CollectorRegistry]:
+ """
+ Returns a matcher for :py:`CollectorRegistry`, that checks whether the specified
+ metrics have the given values.
+
+ :param expected: Dictionary that maps tuples of attrs-class and :py:`_zkapauthorizer.server.spending.counter_attr` names to the expected value of the given metric.
+ """
+ expected = {
+ (attr.fields_dict(cls)[attrib].metadata["metric-name"], ()): value
+ for (cls, attrib), value in expected.items()
+ }
+
+ def summarize(registry: CollectorRegistry):
+ metrics = list(registry.collect())
+ return {
+ (sample.name, tuple(sample.labels.items())): sample.value
+ for metric in metrics
+ for sample in metric.samples
+ }
+
+ return AfterPreprocessing(summarize, ContainsDict(expected))
diff --git a/src/_zkapauthorizer/tests/test_controller.py b/src/_zkapauthorizer/tests/test_controller.py
index 86d933d1e00565bc5928fc2bffc2524ae39e8b09..93b694fc330c525daaebbfc0e95d672cbb5913e9 100644
--- a/src/_zkapauthorizer/tests/test_controller.py
+++ b/src/_zkapauthorizer/tests/test_controller.py
@@ -30,6 +30,7 @@ from challenge_bypass_ristretto import (
VerificationSignature,
random_signing_key,
)
+from hyperlink import URL
from hypothesis import assume, given
from hypothesis.strategies import datetimes, integers, lists, randoms, sampled_from
from testtools import TestCase
@@ -49,7 +50,6 @@ from testtools.twistedsupport import failed, has_no_result, succeeded
from treq.testing import StubTreq
from twisted.internet.defer import fail, succeed
from twisted.internet.task import Clock
-from twisted.python.url import URL
from twisted.web.http import BAD_REQUEST, INTERNAL_SERVER_ERROR, UNSUPPORTED_MEDIA_TYPE
from twisted.web.http_headers import Headers
from twisted.web.iweb import IAgent
diff --git a/src/_zkapauthorizer/tests/test_storage_protocol.py b/src/_zkapauthorizer/tests/test_storage_protocol.py
index 6ca4a6e6ae90fb458ff7144d7c95b96187a69656..836a4a77fdb244f3cdd450fca535f8bee5a47692 100644
--- a/src/_zkapauthorizer/tests/test_storage_protocol.py
+++ b/src/_zkapauthorizer/tests/test_storage_protocol.py
@@ -49,14 +49,13 @@ from ..api import (
ZKAPAuthorizerStorageServer,
)
from ..foolscap import ShareStat
-from ..server.spending import RecordingSpender
from ..storage_common import (
allocate_buckets_message,
get_implied_data_length,
required_passes,
)
from .common import skipIf
-from .fixtures import AnonymousStorageServer
+from .fixtures import AnonymousStorageServer, make_in_memory_spender
from .foolscap import LocalRemote
from .matchers import matches_spent_passes, matches_version_dictionary
from .storage_common import (
@@ -158,7 +157,7 @@ class ShareTests(TestCase):
AnonymousStorageServer(self.clock),
).storage_server
- self.spending_recorder, spender = RecordingSpender.make()
+ self.spending_recorder, spender = make_in_memory_spender()
self.server = ZKAPAuthorizerStorageServer(
self.anonymous_storage_server,
self.pass_value,
diff --git a/src/_zkapauthorizer/tests/test_storage_server.py b/src/_zkapauthorizer/tests/test_storage_server.py
index a0237bb7b82cf5c6069d7ae6f243bb37479f22d1..dd1f7427c66ea24d19377034db2259ce17bdda21 100644
--- a/src/_zkapauthorizer/tests/test_storage_server.py
+++ b/src/_zkapauthorizer/tests/test_storage_server.py
@@ -32,7 +32,6 @@ from twisted.python.runtime import platform
from .._storage_server import NewLengthRejected, _ValidationResult
from ..api import MorePassesRequired, ZKAPAuthorizerStorageServer
-from ..server.spending import RecordingSpender
from ..storage_common import (
add_lease_message,
allocate_buckets_message,
@@ -43,7 +42,7 @@ from ..storage_common import (
summarize,
)
from .common import skipIf
-from .fixtures import AnonymousStorageServer
+from .fixtures import AnonymousStorageServer, make_in_memory_spender
from .matchers import matches_spent_passes, raises
from .storage_common import get_passes, reset_storage_server, write_toy_shares
from .strategies import (
@@ -189,7 +188,7 @@ class PassValidationTests(TestCase):
def setUp(self):
super(PassValidationTests, self).setUp()
self.clock = Clock()
- self.spending_recorder, spender = RecordingSpender.make()
+ self.spending_recorder, spender = make_in_memory_spender()
# anonymous_storage_server uses time.time() so get our Clock close to
# the same time so we can do lease expiration calculations more
# easily.
diff --git a/tests.nix b/tests.nix
index a3bf6878d70864224deedd1c0290d6ec9200a575..1aaaadc0a43f082f0cc248cdf238d9d689c4426c 100644
--- a/tests.nix
+++ b/tests.nix
@@ -8,6 +8,7 @@ let
"testSuite"
"trialArgs"
];
+ sources = import nix/sources.nix;
in
{ privatestorage ? import ./. (fixArgs args)
, hypothesisProfile ? null
@@ -26,11 +27,22 @@ let
extraTrialArgs = builtins.concatStringsSep " " trialArgs';
testSuite' = if testSuite == null then "_zkapauthorizer" else testSuite;
+ zss = import sources.zkap-spending-service {
+ inherit pkgs mach-nix;
+ };
+
python = mach-nix.mkPython {
inherit (zkapauthorizer.meta.mach-nix) python providers;
requirements =
builtins.readFile ./requirements/test.in;
packagesExtra = [ zkapauthorizer ];
+ overridesPre = [
+ (
+ self: super: {
+ zkap-spending-service = zss;
+ }
+ )
+ ];
_.hypothesis.postUnpack = "";
};