Tests refactoring: main test executable + CTest framework (issue 38)

.github/workflows/ci.yml

@@ -4,11 +4,10 @@
 
 name: Build and Test
 
-# Builds the OpenSimplex2 CMake project and runs its internal `test` target.
-# The `test` target (created by the QuasarApp CMake module, initTests/addTests)
-# first deploys the test executable with `cqtdeployer` (deployTest* target) and
-# then executes the generated runner, so cqtdeployer must be installed on PATH
-# before the test step (see the CQtDeployer install action below).
+# Builds the OpenSimplex2 CMake project and runs its tests via CTest.
+# `enable_testing()` creates the `test` build target which executes all
+# tests registered with add_test() in tests/CMakeLists.txt (see
+# "Testing With CMake and CTest", CMake documentation).
 on:
   pull_request:
     types: [opened, synchronize, reopened]
@@ -27,30 +26,11 @@ jobs:
         build_type: [Release]
 
     steps:
-      # The project is a plain C application (no Qt). On Windows the test source
-      # uses __attribute__((unused)) (GCC-only, MSVC cannot build it), so a MinGW
-      # toolchain is required: the aqt MinGW (tools_mingw1310) provides the gcc
-      # compiler and qmake.
-      - name: Install Qt (Windows)
-        uses: jurplel/install-qt-action@v4.3.1
-        if: runner.os == 'Windows'
-        with:
-          version: '6.10.3' # aqt has bug with latest qt versions in windows
-          target: 'desktop'
-          arch: 'win64_mingw'
-          cache: true
-          tools: 'tools_mingw1310'
-          aqtversion: ==3.3.*
-
-      - name: Install Qt (Linux)
-        uses: jurplel/install-qt-action@v4.3.1
-        if: runner.os == 'Linux'
-        with:
-          version: '6.12.0'
-          target: 'desktop'
-          arch: 'linux_gcc_64'
-          cache: true
-          aqtversion: ==3.3.*
+      # The project is a plain C application (no Qt), so no Qt installation
+      # is required: the CMake project (submodule/CMake/QuasarApp.cmake) only
+      # looks qmake up at configure time and continues if it is not found.
+      # Linux uses the pre-installed gcc, Windows the MinGW gcc installed
+      # from MSYS2 below.
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -62,33 +42,30 @@ jobs:
         with:
           key: ${{ matrix.os }}-${{ matrix.build_type }}
 
-      # The `test` target deploys the test binary with cqtdeployer, so install it.
-      # No version pin: the install action downloads the latest stable
-      # CQtDeployer release by default (latest: true).
-      - name: Install CQtDeployer
-        uses: QuasarApp/cqtdeployer-install-action@v1.1.0
-
-      # The test executable links against libpng (find_package(PNG REQUIRED)) and writes
-      # test2d/3d/4d.png at runtime.
+      # The test executable links against libpng (find_package(PNG REQUIRED)) and
+      # writes test2d/3d/4d.png at runtime.
       - name: Install libpng (Linux)
         if: runner.os == 'Linux'
         run: |
           sudo apt-get update
           sudo apt-get install -y libpng-dev
 
-      # The aqt MinGW does not ship libpng, so install the MinGW libpng from MSYS2 and
-      # point CMake at it (see the Windows configure step).
-      - name: Install libpng (Windows)
+      # The MinGW gcc compiler (mingw-w64-x86_64-gcc) and the MinGW libpng
+      # are installed from MSYS2 and CMake is pointed at the msys2 mingw64
+      # root (see the Windows configure step).
+      - name: Install MinGW gcc and libpng (Windows)
         id: msys2
         if: runner.os == 'Windows'
         uses: msys2/setup-msys2@v2
         with:
           msystem: MINGW64
-          install: mingw-w64-x86_64-libpng
+          install: mingw-w64-x86_64-gcc mingw-w64-x86_64-libpng
+          suppress-deprecation-warnings: msystem-mingw64
 
-      # setup-msys2 only adds the MSYS2 root to the Windows PATH, so the test
-      # executable would not find libpng16-16.dll (and libz-1.dll) at runtime and
-      # fails with 0xc0000135 (STATUS_DLL_NOT_FOUND). Add the mingw64 bin
+      # setup-msys2 only adds the MSYS2 root to the Windows PATH, so the gcc
+      # compiler (the configure step) and the test executable (libpng16-16.dll
+      # and libz-1.dll at runtime) would not be found and the tests would
+      # fail with 0xc0000135 (STATUS_DLL_NOT_FOUND). Add the mingw64 bin
       # directory to the PATH for the build/test steps.
       - name: Add MSYS2 mingw64 bin to PATH (Windows)
         if: runner.os == 'Windows'
@@ -113,7 +90,6 @@ jobs:
           -G Ninja
           -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
           -DCMAKE_C_COMPILER_LAUNCHER=ccache
-          -DCMAKE_PREFIX_PATH=${{ env.QT_ROOT_DIR }}
           -S ${{ github.workspace }}
 
       - name: Build
@@ -122,6 +98,7 @@ jobs:
       - name: ccache statistics
         run: ccache -s
 
-      # Runs the internal CMake `test` target (deploys + executes the test binary).
+      # Runs the `test` build target: it invokes CTest, which executes every
+      # test registered in tests/CMakeLists.txt (test_noise2/3/4).
       - name: Run Tests
         run: cmake --build ${{ github.workspace }}/build --target test

.github/workflows/cross-platform-compare.yml

@@ -40,39 +40,11 @@ jobs:
             platform: macos
 
     steps:
-      # The project is plain C (no Qt code), but the CMake project
-      # (submodule/CMake/QuasarApp.cmake) looks for qmake at configure time,
-      # so Qt must be installed on every platform (same setup as ci.yml).
-      - name: Install Qt (Windows)
-        uses: jurplel/install-qt-action@v4.3.1
-        if: runner.os == 'Windows'
-        with:
-          version: '6.10.3' # aqt has bug with latest qt versions in windows
-          target: 'desktop'
-          arch: 'win64_mingw'
-          cache: true
-          tools: 'tools_mingw1310'
-          aqtversion: ==3.3.*
-
-      - name: Install Qt (Linux)
-        uses: jurplel/install-qt-action@v4.3.1
-        if: runner.os == 'Linux'
-        with:
-          version: '6.12.0'
-          target: 'desktop'
-          arch: 'linux_gcc_64'
-          cache: true
-          aqtversion: ==3.3.*
-
-      - name: Install Qt (macOS)
-        uses: jurplel/install-qt-action@v4.3.1
-        if: runner.os == 'macOS'
-        with:
-          version: '6.10.3'
-          target: 'desktop'
-          arch: 'clang_64'
-          cache: true
-          aqtversion: ==3.3.*
+      # The project is plain C (no Qt code), so no Qt installation is
+      # required: the CMake project (submodule/CMake/QuasarApp.cmake) only
+      # looks qmake up at configure time and continues if it is not found.
+      # Linux and macOS use the pre-installed gcc/clang, Windows the MinGW
+      # gcc installed from MSYS2 below (same setup as ci.yml).
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -87,19 +59,22 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y libpng-dev
 
-      # The aqt MinGW does not ship libpng, so install the MinGW libpng from
-      # MSYS2 and point CMake at it (same setup as ci.yml).
-      - name: Install libpng (Windows)
+      # The MinGW gcc compiler (mingw-w64-x86_64-gcc) and the MinGW libpng
+      # are installed from MSYS2 and CMake is pointed at the msys2 mingw64
+      # root (see the Windows configure step, same setup as ci.yml).
+      - name: Install MinGW gcc and libpng (Windows)
         id: msys2
         if: runner.os == 'Windows'
         uses: msys2/setup-msys2@v2
         with:
           msystem: MINGW64
-          install: mingw-w64-x86_64-libpng
+          install: mingw-w64-x86_64-gcc mingw-w64-x86_64-libpng
+          suppress-deprecation-warnings: msystem-mingw64
 
-      # setup-msys2 only adds the MSYS2 root to the Windows PATH, so the
-      # compiled binaries would not find libpng16-16.dll (and libz-1.dll) at
-      # runtime and fail with 0xc0000135 (STATUS_DLL_NOT_FOUND).
+      # setup-msys2 only adds the MSYS2 root to the Windows PATH, so the gcc
+      # compiler (the configure step) and the compiled binaries (libpng16-16.dll
+      # and libz-1.dll at runtime) would not be found and fail with 0xc0000135
+      # (STATUS_DLL_NOT_FOUND).
       - name: Add MSYS2 mingw64 bin to PATH (Windows)
         if: runner.os == 'Windows'
         run: |
@@ -124,7 +99,6 @@ jobs:
           cmake -B ${{ github.workspace }}/build
           -G Ninja
           -DCMAKE_BUILD_TYPE=Release
-          -DCMAKE_PREFIX_PATH=${{ env.QT_ROOT_DIR }}
           -S ${{ github.workspace }}
 
       - name: Build

CMakeLists.txt

@@ -47,9 +47,11 @@ endif()
 
 option(OPENSIMPLEX2_TESTS "This option disables or enables tests of the ${PROJECT_NAME} project" ON)
 
-initAll()
-
 if (OPENSIMPLEX2_TESTS)
+    # CTest framework (see "Testing With CMake and CTest" chapter of the
+    # CMake documentation): enable_testing() creates the `test` build target
+    # that runs all tests registered in the tests/ subdirectory with ctest.
+    enable_testing()
     add_subdirectory(tests)
 else()
     message("The ${PROJECT_NAME} tests is disabled.")

README.md

@@ -133,17 +133,38 @@ The Makefile also builds the `test_OpenSimplex2F` binary (test source in
 
 ### Tests
 
+The tests use the [CTest](https://cmake.org/cmake/help/book/mastering-cmake/chapter/Testing%20With%20CMake%20and%20CTest.html)
+framework. The main test executable (`OpenSimplex2tests`, built from
+`tests/test_OpenSimplex2F.c`) implements every test in its own function
+(`test_noise2`, `test_noise3`, `test_noise4`); each of them is registered
+as a separate CTest test (see `tests/CMakeLists.txt`).
+
 To build the tests, keep the `OPENSIMPLEX2_TESTS` option enabled (default
-`ON`) and run the `test` target — it builds the `OpenSimplex2tests`
-executable from the `tests/` subdirectory and runs it (the targets are
-registered with the `addTests()` function of the QuasarApp CMake module):
+`ON`) and run the `test` target — it invokes CTest and executes all
+registered tests:
 
 ```
 cmake -B build
 cmake --build build
 cmake --build build --target test
 ```
 
+Individual tests can be run separately with CTest:
+
+```
+ctest --test-dir build -R test_noise3
+```
+
+or by passing the test name to the executable:
+
+```
+build/tests/OpenSimplex2tests test_noise3
+```
+
+The tests write 2D/3D/4D noise images (requires libpng) into the test
+directory of the build tree and check that all noise values stay within
+the expected `[-1, 1]` range.
+
 ## License
 
 This project is provided under the license defined in the

tests/CMakeLists.txt

@@ -19,8 +19,20 @@ target_include_directories(${CURRENT_PROJECT} PRIVATE "${CMAKE_CURRENT_SOURCE_DI
 target_compile_options(${CURRENT_PROJECT} PRIVATE -Wall -Wextra --pedantic)
 target_link_libraries(${CURRENT_PROJECT} PRIVATE OpenSimplex2Interface m PNG::PNG)
 
-initTests()
-addTests(${PROJECT_NAME} ${CURRENT_PROJECT})
+# CTest framework: every test function of the main test executable
+# (test_noise2, test_noise3, test_noise4) is registered as a separate
+# CTest test, so each of them can be run individually, e.g.:
+#     ctest -R test_noise2
+# or all together (the `test` build target / `ctest` from the build dir).
+# The generated PNG images are written to the build directory, not to the
+# source tree.
+set(TEST_NAMES test_noise2 test_noise3 test_noise4)
+
+foreach(test_name IN LISTS TEST_NAMES)
+    add_test(NAME ${test_name}
+             COMMAND ${CURRENT_PROJECT} ${test_name}
+             WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+endforeach()
 
 # Cross-platform noise benchmark (issue #31): evaluates every OpenSimplex2F
 # noise function with the same seed and the same input parameters on every
@@ -48,13 +60,13 @@ endif()
 
 # The cross-platform comparison workflow
 # (.github/workflows/cross-platform-compare.yml) builds and runs this target
-# on Linux, Windows and macOS. It is intentionally not part of the QuasarApp
-# `test` target, so no cqtdeployer deployment is required. The benchmark
-# writes noise_benchmark_results.txt into the build/cross_platform working
+# on Linux, Windows and macOS. It is intentionally not registered as a CTest
+# test, so it is not part of the `test` build target (the benchmark needs no
+# cqtdeployer deployment). The benchmark writes
+# noise_benchmark_results.txt into the build/cross_platform working
 # directory (the executable creates it).
 add_custom_target(${PROJECT_NAME}_cross_platform_benchmark
     COMMAND $<TARGET_FILE:${CROSS_PLATFORM_BENCHMARK}> "${CMAKE_BINARY_DIR}/cross_platform/noise_benchmark_results.txt"
     DEPENDS ${CROSS_PLATFORM_BENCHMARK}
     COMMENT "Running the OpenSimplex2F cross-platform noise benchmark"
     VERBATIM)
-