Merge pull request #64 from PrivateStorageio/63.faster-fingerprinting

Speed up the blinded token fingerprinting function

misc/load-test.py

@@ -13,19 +13,33 @@
 #
 #   $ for n in $(seq 0 30); do sqlite3 vouchers.db "insert into vouchers (name) values ('aaa$n')"; done
 #
-# Then the test can be run as many times as necessary.  Repeated redemptions
-# are allowed since the same tokens are used on every run.
+# Then the test can be run as many times as necessary.
+#
+# The `redeemed` table must be cleared before each test run.  Random tokens
+# are generated for each test run and the server will reject tokens from a new
+# run if tokens from an old run are still present.
+#
+#  $ sqlite3 vouchers.db "delete from redeemed"
 #
 # Originally written for https://github.com/PrivateStorageio/PaymentServer/issues/60
 
 from __future__ import division
 
+from os import (
+    urandom,
+)
+
+from base64 import (
+    b64encode,
+)
+
 from time import (
     time,
 )
 
 from json import (
     dumps,
+    loads,
 )
 
 from treq.client import (
@@ -45,29 +59,79 @@ from twisted.internet.defer import (
     returnValue,
 )
 
-PARALLELISM = 30
+PARALLELISM = 50
+ITERATIONS = 16
+NUM_TOKENS = 5000
+
+
+def a_random_token():
+    return b64encode(urandom(32))
+
+
+def tokens_for_voucher(key, cache={}):
+    if key not in cache:
+        print("Generating tokens for {}".format(key))
+        cache[key] = list(
+            a_random_token()
+            for _
+            in range(NUM_TOKENS)
+        )
+    else:
+        print("Using cached tokens for {}".format(key))
+    return cache[key]
+
 
 @inlineCallbacks
-def redeem(client, index):
-    times = []
-    for i in range(16):
+def redeem_with_retry(client, data, headers):
+    """
+    Attempt a redemption.  Retry if it fails.
+
+    :return: A ``Deferred`` that fires with (duration of successful request,
+        number of failed requests).
+    """
+    errors = 0
+    while True:
         before = time()
         response = yield client.post(
             url="http://127.0.0.1:8080/v1/redeem",
+            data=data,
+            headers=headers,
+        )
+        after = time()
+        duration = int((after - before) * 1000)
+        body = yield readBody(response)
+        if response.code == 200:
+            print("Request complete in {}ms".format(duration))
+            returnValue((duration, errors))
+
+        errors += 1
+        try:
+            reason = loads(body)["reason"]
+        except ValueError:
+            reason = body
+
+        print("Request failed: {} {}".format(response.code, reason))
+
+
+@inlineCallbacks
+def redeem(client, index):
+    times = []
+    total_errors = 0
+    voucher = "aaa{}".format(index)
+    for i in range(ITERATIONS):
+        tokens = tokens_for_voucher((voucher, i))
+        duration, errors = yield redeem_with_retry(
+            client,
             data=dumps({
-                "redeemVoucher": "aaa{}".format(index),
-                "redeemTokens": ["foo-{}-{}".format(index, i)],
+                "redeemVoucher": voucher,
+                "redeemTokens": tokens,
                 "redeemCounter": i,
             }),
             headers={"content-type": "application/json"},
         )
-        after = time()
-        duration = int((after - before) * 1000)
-        print("Request complete in {}ms".format(duration))
-        body = yield readBody(response)
-        assert response.code == 200, (response.code, body)
         times.append(duration)
-    returnValue(times)
+        total_errors += errors
+    returnValue((times, total_errors))
 
 
 def mean(xs):
@@ -75,7 +139,7 @@ def mean(xs):
 
 
 def percentile(n, xs):
-    return sorted(xs)[int(len(xs) / 100 * 95)]
+    return sorted(xs)[int(len(xs) / 100 * n)]
 
 
 def median(xs):
@@ -94,11 +158,17 @@ def main(reactor):
     )
 
     times = []
-    for result in (yield gatherResults(ds)):
+    total_errors = 0
+    for (result, errors) in (yield gatherResults(ds)):
         times.extend(result)
+        total_errors += errors
 
     print("min: {}".format(min(times)))
     print("max: {}".format(max(times)))
     print("mean: {}".format(mean(times)))
     print("median: {}".format(median(times)))
     print("95th: {}".format(percentile(95, times)))
+    print("errors: {}".format(total_errors))
+    print("error rate: {}".format(
+        total_errors / (total_errors + PARALLELISM * ITERATIONS),
+    ))

src/PaymentServer/Redemption.hs

@@ -11,6 +11,10 @@ module PaymentServer.Redemption
   , redemptionServer
   ) where
 
+import Prelude hiding
+  ( concat
+  )
+
 import GHC.Generics
   ( Generic
   )
@@ -27,6 +31,7 @@ import Control.Monad.IO.Class
 import Data.Text
   ( Text
   , pack
+  , concat
   )
 import Data.Text.Encoding
   ( encodeUtf8
@@ -211,7 +216,4 @@ redeem issue database (Redeem voucher tokens counter) =
 -- be used as an identifier for this exact sequence of tokens.
 fingerprintFromTokens :: [BlindedToken] -> Fingerprint
 fingerprintFromTokens =
-  let
-    hash = pack . show . hashWith SHA3_512 . encodeUtf8
-  in
-    foldl (\b a -> hash $ a `mappend` b) "" . map hash
+  pack . show . hashWith SHA3_512 . encodeUtf8 . concat