Merge pull request #147 from PrivateStorageio/97.coverage

Fix test coverage collection and reporting from the Nix CI job

.circleci/config.yml

@@ -106,8 +106,8 @@ jobs:
             # If nixpkgs changes then potentially a lot of cached packages for
             # the base system will be invalidated so we may as well drop them
             # and make a new cache with the new packages.
-            - zkapauthorizer-nix-store-v3-{{ checksum "nixpkgs.rev" }}
-            - zkapauthorizer-nix-store-v3-
+            - zkapauthorizer-nix-store-v4-{{ checksum "nixpkgs.rev" }}
+            - zkapauthorizer-nix-store-v4-
 
       - run:
           name: "Run Test Suite"
@@ -139,15 +139,14 @@ jobs:
 
       - save_cache:
           name: "Cache Nix Store Paths"
-          key: zkapauthorizer-nix-store-v3-{{ checksum "nixpkgs.rev" }}
+          key: zkapauthorizer-nix-store-v4-{{ checksum "nixpkgs.rev" }}
           paths:
             - "/nix"
 
       - run:
           name: "Report Coverage"
           command: |
-            nix-shell -p 'python.withPackages (ps: [ ps.codecov ])' --run \
-              'codecov --file ./result-doc/share/doc/*/.coverage'
+            ./.circleci/report-coverage.sh
 
 workflows:
   version: 2

.circleci/report-coverage.sh

+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p "python.withPackages (ps: [ ps.codecov ])"
+set -x
+find ./result-doc/share/doc
+cp ./result-doc/share/doc/*/.coverage.* ./
+python -m coverage combine
+python -m coverage report
+python -m coverage xml
+codecov --file coverage.xml

.coveragerc

@@ -3,4 +3,33 @@ source =
     _zkapauthorizer
     twisted.plugins.zkapauthorizer
 
+# Measuring branch coverage is slower (so the conventional wisdom goes) but
+# too bad: it's an important part of the coverage information.
 branch = True
+
+# Whether or not we actually collect coverage information in parallel, we need
+# to have the coverage data files written according to the "parallel" naming
+# scheme so that we can use "coverage combine" later to rewrite paths in the
+# coverage report.
+parallel = True
+
+omit =
+# The Versioneer version file in the repository is generated by
+# Versioneer.  Let's call it Versioneer's responsibility to ensure it
+# works and not pay attention to our test suite's coverage of it.  Also,
+# the way Versioneer works is that the source file in the repository is
+# different from the source file in an installation - which is where we
+# measure coverage.  When the source files differ like this, it's very
+# difficult to produce a coherent coverage report (measurements against
+# one source file are meaningless when looking at a different source
+# file).
+    */_zkapauthorizer/_version.py
+
+[paths]
+source =
+# It looks like this in the checkout
+    src/
+# It looks like this in the Nix build environment
+    /nix/store/*/lib/python*/site-packages/
+# It looks like this in the Windows build environment
+    C:\hostedtoolcache\windows\Python\2.7.18\x64\Lib\site-packages\

.github/workflows/ci.yaml

@@ -63,11 +63,17 @@ jobs:
       env:
         MAGIC_FOLDER_HYPOTHESIS_PROFILE: "ci"
       run: |
-        python -m coverage run -m twisted.trial _zkapauthorizer
-
+        python -m coverage run --debug=config -m twisted.trial _zkapauthorizer
 
     - name: "Convert Coverage"
       run: |
+        echo "Files:"
+        dir
+        echo "Combining"
+        coverage combine
+        echo "Reporting"
+        coverage report
+        echo "Converting to XML"
         coverage xml
 
     - uses: codecov/codecov-action@v1

zkapauthorizer.nix

@@ -50,7 +50,7 @@ buildPythonPackage rec {
     runHook preCheck
     "${pyflakes}/bin/pyflakes" src/_zkapauthorizer
     ZKAPAUTHORIZER_HYPOTHESIS_PROFILE=${hypothesisProfile'} python -m ${if collectCoverage
-      then "coverage run --branch --source _zkapauthorizer,twisted.plugins.zkapauthorizer --module"
+      then "coverage run --debug=config --module"
       else ""
     } twisted.trial ${extraTrialArgs} ${testSuite'}
     runHook postCheck
@@ -58,10 +58,8 @@ buildPythonPackage rec {
 
   postCheck = if collectCoverage
     then ''
-    python -m coverage html
     mkdir -p "$doc/share/doc/${name}"
-    cp -vr .coverage htmlcov "$doc/share/doc/${name}"
-    python -m coverage report
+    cp -v .coverage.* "$doc/share/doc/${name}"
     ''
     else "";
 }