Adding code coverage (#2148)

* First stab at adding coverage tests

Signed-off-by: Spencer Wilson <[email protected]>

* Install dependencies for coveralls action

Signed-off-by: Spencer Wilson <[email protected]>

* Integrate code coverage into CI

Signed-off-by: Spencer Wilson <[email protected]>

* Properly parallelize

Signed-off-by: Spencer Wilson <[email protected]>

* Refine test coverage

Signed-off-by: Spencer Wilson <[email protected]>

* Ignore unused exclude path error

Signed-off-by: Spencer Wilson <[email protected]>

* Update exclude path

Signed-off-by: Spencer Wilson <[email protected]>

* Fix Markdown formatting

Signed-off-by: Spencer Wilson <[email protected]>

* Pin dependencies

Signed-off-by: Spencer Wilson <[email protected]>

* Update workflow badge path

Signed-off-by: Aiden Fox Ivey <[email protected]>

* Fix path in CI.md

Signed-off-by: Aiden Fox Ivey <[email protected]>

* Remove unnecessary call to install curl

Signed-off-by: Aiden Fox Ivey <[email protected]>

---------

Signed-off-by: Spencer Wilson <[email protected]>
Signed-off-by: Aiden Fox Ivey <[email protected]>
Co-authored-by: Spencer Wilson <[email protected]>

Author: aidenfoxivey

.CMake/compiler_opts.cmake

@@ -113,6 +113,10 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang")
     if(${OQS_DEBUG_BUILD})
         add_compile_options(-g3)
         add_compile_options(-fno-omit-frame-pointer)
+	if(${USE_COVERAGE})
+            add_compile_options(-coverage)
+            add_link_options(-coverage)
+	endif()
         if(USE_SANITIZER STREQUAL "Address")
             add_compile_options(-fno-optimize-sibling-calls)
             add_compile_options(-fsanitize-address-use-after-scope)
@@ -175,6 +179,10 @@ elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU")
     if(${OQS_DEBUG_BUILD})
         add_compile_options (-Wstrict-overflow)
         add_compile_options(-ggdb3)
+	if(${USE_COVERAGE})
+            add_compile_options(-coverage)
+            add_link_options(-coverage)
+	endif()
     else()
         add_compile_options(-O3)
         add_compile_options(-fomit-frame-pointer)

.github/workflows/code-coverage.yml

@@ -0,0 +1,60 @@
+name: Code coverage tests
+
+permissions:
+  contents: read
+
+on: [workflow_call, workflow_dispatch]
+
+jobs:
+  coverage:
+    name: Run code coverage testing
+    strategy:
+      matrix:
+        # The 'id' value for each job should be added to the 'carry-forward' string in the 'finish' job.
+        include:
+          - id: x64-generic
+            runner: ubuntu-latest
+            CMAKE_ARGS: -DOQS_DIST_BUILD=OFF -DOQS_OPT_TARGET=generic
+          - id: x64-distbuild
+            runner: ubuntu-latest
+            CMAKE_ARGS: -DOQS_DIST_BUILD=ON
+          - id: arm64-distbuild
+            runner: ubuntu-24.04-arm
+            CMAKE_ARGS: -DOQS_DIST_BUILD=ON
+    runs-on: ${{ matrix.runner }}
+    container: openquantumsafe/ci-ubuntu-latest:latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
+      - name: Configure
+        run: |
+          mkdir build && cd build && \
+          cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DUSE_COVERAGE=ON ${{ matrix.CMAKE_ARGS }} .. && \
+          cmake -LA -N ..
+      - name: Build
+        run: ninja
+        working-directory: build
+      - name: Run tests
+        run: |
+          python3 -m pytest --verbose --numprocesses=auto \
+          tests/test_acvp_vectors.py \
+          tests/test_cmdline.py \
+          tests/test_kat.py
+      - name: Run lcov
+        run: lcov -d . -c -o lcov.info --exclude /usr/lib,/usr/include --ignore-errors unused
+      - name: Upload to coveralls.io
+        uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # [email protected]
+        with:
+          flag-name: ${{ matrix.id }}
+          parallel: true
+
+  finish:
+    needs: coverage
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Finish coveralls.io
+        uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # [email protected]
+        with:
+          parallel-finished: true
+          carry-forward: "x64-generic,x64-distbuild,arm64-distbuild"

.github/workflows/commit-to-main.yml

@@ -12,6 +12,10 @@ jobs:
   platform-tests:
     uses: ./.github/workflows/platforms.yml
 
+  code-coverage:
+    uses: ./.github/workflows/code-coverage.yml
+    secrets: inherit
+
   scorecard:
     uses: ./.github/workflows/scorecard.yml
     secrets: inherit

.github/workflows/pr.yml

@@ -18,6 +18,11 @@ jobs:
     needs: basic-checks
     uses: ./.github/workflows/platforms.yml
 
+  code-coverage:
+    needs: basic-checks
+    uses: ./.github/workflows/code-coverage.yml
+    secrets: inherit
+
   scorecard:
     needs: basic-checks
     uses: ./.github/workflows/scorecard.yml

CI.md

@@ -26,12 +26,12 @@ For example, "[full tests] [trigger downstream]" will trigger both the platform
 #### <a name="pr.yml"></a> Pull request workflow (`pr.yml`)
 
 This workflow runs on pull requests.
-It calls [basic checks](#basic.yml), [platform tests](#platforms.yml) and [scorecard analysis](#scorecard.yml).
+It calls [basic checks](#basic.yml), [code coverage tests](#code-coverage.yml), [platform tests](#platforms.yml) and [scorecard analysis](#scorecard.yml).
 
 #### <a name="commit-to-main.yml"></a> Commit-to-main workflow (`commit-to-main.yml`)
 
 This workflow runs on pushes to the `main` branch (typically done automatically when a pull request is merged).
-It calls [platform tests](#platforms.yml), [scorecard analysis](#scorecard.yml), and [basic downstream tests](#downstream-basic.yml).
+It calls [platform tests](#platforms.yml), [code coverage tests](#code-coverage.yml), [scorecard analysis](#scorecard.yml), and [basic downstream tests](#downstream-basic.yml).
 
 #### <a name="weekly.yml"></a> Weekly workflow (`weekly.yml`)
 
@@ -53,6 +53,10 @@ Users with "write" permissions can also trigger them manually via the GitHub web
 
 This workflow runs a minimal set of tests that should pass before heavier tests are triggered.
 
+#### <a name="code-coverage.yml"></a> Code coverage tests (`code-coverage.yml`)
+
+This workflow runs code coverage tests and uploads the results to [Coveralls.io](https://coveralls.io/github/open-quantum-safe/liboqs).
+
 #### <a name="<platform>.yml"></a> Individual platform tests (`<platform>.yml`)
 
 These workflows contain tests for the individual [platforms supported by liboqs](PLATFORMS.md).

CONFIGURE.md

@@ -16,6 +16,7 @@ The following options can be passed to CMake before the build file generation pr
 - [OQS_USE_CUPQC](#OQS_USE_CUPQC)
 - [OQS_OPT_TARGET](#OQS_OPT_TARGET)
 - [OQS_SPEED_USE_ARM_PMU](#OQS_SPEED_USE_ARM_PMU)
+- [USE_COVERAGE](#USE_COVERAGE)
 - [USE_SANITIZER](#USE_SANITIZER)
 - [OQS_ENABLE_TEST_CONSTANT_TIME](#OQS_ENABLE_TEST_CONSTANT_TIME)
 - [OQS_STRICT_WARNINGS](#OQS_STRICT_WARNINGS)
@@ -36,7 +37,9 @@ This means liboqs is built as a static library by default.
 
 Can be set to the following values:
 
-- `Debug`: This turns off all compiler optimizations and produces debugging information. When the compiler is Clang, the [USE_SANITIZER](#USE_SANITIZER) option can also be specified to enable a Clang sanitizer. **This value only has effect when the compiler is GCC or Clang**
+- `Debug`: This turns off all compiler optimizations and produces debugging information. **This value only has effect when the compiler is GCC or Clang**
+  - The [USE_COVERAGE](#USE_COVERAGE) option can also be specified to enable code coverage testing.
+  - When the compiler is Clang, the [USE_SANITIZER](#USE_SANITIZER) option can also be specified to enable a Clang sanitizer.
 
 - `Release`: This compiles code at the `O3` optimization level, and sets other compiler flags that reduce the size of the binary.
 
@@ -173,6 +176,12 @@ Note that this option is not known to work on Apple M1 chips.
 
 **Default**: `OFF`.
 
+## USE_COVERAGE
+
+This has an effect when the compiler is GCC or Clang and when [CMAKE_BUILD_TYPE](#CMAKE_BUILD_TYPE) is `Debug`. Can be `ON` or `OFF`. When `ON`, code coverage testing will be enabled.
+
+**Default**: Unset.
+
 ## USE_SANITIZER
 
 This has an effect when the compiler is Clang and when [CMAKE_BUILD_TYPE](#CMAKE_BUILD_TYPE) is `Debug`. Then, it can be set to:

README.md

@@ -4,6 +4,7 @@ liboqs
 [![Main Branch Tests](https://github.com/open-quantum-safe/liboqs/actions/workflows/commit-to-main.yml/badge.svg)](https://github.com/open-quantum-safe/liboqs/actions/workflows/commit-to-main.yml)
 [![Weekly Tests](https://github.com/open-quantum-safe/liboqs/actions/workflows/weekly.yml/badge.svg)](https://github.com/open-quantum-safe/liboqs/actions/workflows/weekly.yml)
 ![Travis Build Status](https://img.shields.io/travis/com/open-quantum-safe/liboqs?logo=travis&label=Travis%20CI&labelColor=%23343B42&color=%232EBB4E)
+[![Coverage Status](https://coveralls.io/repos/github/open-quantum-safe/liboqs/badge.svg?branch=main)](https://coveralls.io/github/open-quantum-safe/liboqs?branch=main)
 
 liboqs is an open source C library for quantum-safe cryptographic algorithms.
 

src/oqsconfig.h.cmake

@@ -30,6 +30,7 @@
 #cmakedefine BUILD_SHARED_LIBS 1
 #cmakedefine OQS_BUILD_ONLY_LIB 1
 #cmakedefine OQS_OPT_TARGET "@OQS_OPT_TARGET@"
+#cmakedefine USE_COVERAGE 1
 #cmakedefine USE_SANITIZER "@USE_SANITIZER@"
 #cmakedefine CMAKE_BUILD_TYPE "@CMAKE_BUILD_TYPE@"
 

tests/system_info.c

@@ -216,6 +216,7 @@ static void print_oqs_configuration(void) {
 	 * OQS_OPT_TARGET: Visible by looking at compile options (-march or -mcpu):
 	 *                 'auto' -> "-march|cpu=native"
 	 * OQS_SPEED_USE_ARM_PMU: Output with Target platform
+	 * USE_COVERAGE: --coverage option present in compile options
 	 * USE_SANITIZER: -fsanitize= option present in compile options
 	 * OQS_ENABLE_TEST_CONSTANT_TIME: only shown below
 	 */
@@ -273,6 +274,9 @@ static void print_oqs_configuration(void) {
 #ifdef OQS_BUILD_ONLY_LIB
 	printf("OQS_BUILD_ONLY_LIB "); // pretty much impossible to appear but added for completeness
 #endif
+#ifdef USE_COVERAGE
+	printf("USE_COVERAGE " );
+#endif
 #ifdef USE_SANITIZER
 	printf("USE_SANITIZER=%s ", USE_SANITIZER);
 #endif