Provide support in Ginkgo for Coverage and Sanitizers build.

Terry Cojean · Terry Cojean · commit f78330be7f8f · 2019-02-19T14:16:11.000+01:00
+ Add a CMake script which simplifies adding new build types to the project
+ Use this script to provide three new build types:
  + COVERAGE: relies on gcov and the `--coverage` flag
  + ASAN: relies on the flag `-fsanitize=address`
  + TSAN: relies on the flag `-fsanitize=thread`
+ Add four new steps to the CI system, one for each of the new build types plus valgrind.
+ Update CI Benchmarking support as it was using the old format.
+ Add configuration files to simplify using CTest and support CDash.
+ Document these additions in the Readme.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -5,6 +5,10 @@ stages:
   - build
   - test
   - deploy
+  - coverage
+  - valgrind
+  - threadsanitizer
+  - addresssanitizer
   - benchmark-build
   - benchmark-cuda
   - benchmark-omp
@@ -367,6 +371,51 @@ gh-pages:
       - schedules
 
 
+coverage:
+  stage: coverage
+  image: localhost:5000/gko-cuda100-gnu7-llvm60
+  before_script: *default_before_script
+  script:
+    - ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=COVERAGE
+  dependencies: []
+#  only:
+#    - master
+#    - develop
+
+valgrind:
+  stage: valgrind
+  image: localhost:5000/gko-cuda100-gnu7-llvm60
+  before_script: *default_before_script
+  script:
+    - ctest -S cmake/CTestScript.cmake -DCTEST_MEMORYCHECK_TYPE=Valgrind
+  dependencies: []
+#  only:
+#    - master
+#    - develop
+
+threadsanitizer:
+  stage: threadsanitizer
+  image: localhost:5000/gko-cuda100-gnu7-llvm60
+  before_script: *default_before_script
+  script:
+    - ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=TSAN -DCTEST_MEMORYCHECK_TYPE=ThreadSanitizer
+  dependencies: []
+#  only:
+#    - master
+#    - develop
+
+addresssanitizer:
+  stage: addresssanitizer
+  image: localhost:5000/gko-cuda100-gnu7-llvm60
+  before_script: *default_before_script
+  script:
+    - ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=ASAN -DCTEST_MEMORYCHECK_TYPE=AddressSanitizer
+  dependencies: []
+#  only:
+#    - master
+#    - develop
+
+
 # Benchmark build
 .benchmark_before_script_template: &default_benchmark_before_script
   # set up identities
@@ -423,6 +472,8 @@ fineci-benchmark-build:
   dependencies: []
   only:
     - schedules
+#    - develop
+#    - master
 
 
 # Benchmark runs
@@ -433,9 +484,10 @@ fineci-benchmark-build:
     - |
       ${SSH_COMMAND} 'tee /dev/stderr | scl enable devtoolset-7 bash' \
             >results.json << EOT
+        module load cuda/cuda-10.0
         set -xe
         cd ginkgo/build/benchmark
-        SYSTEM_NAME=${SYSTEM_NAME} make ${EXECUTOR}_benchmark
+        make benchmark SYSTEM_NAME=${SYSTEM_NAME} EXECUTOR=${EXECUTOR}
         tar -czf data.tar.gz results
       EOT
     # publish them
@@ -455,6 +507,8 @@ fineci-benchmark-build:
   dependencies: []
   only:
     - schedules
+#    - develop
+#    - master
 
 fineci-benchmark-cuda:
   stage: benchmark-cuda
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -10,6 +10,8 @@ include(cmake/create_test.cmake)
 include(cmake/install_helpers.cmake)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
 
+include(cmake/build_type_helpers.cmake)
+
 # Ginkgo configuration options
 option(GINKGO_DEVEL_TOOLS "Add development tools to the build system" ON)
 option(GINKGO_BUILD_TESTS "Generate build files for unit tests" ON)
@@ -37,11 +39,13 @@ endif()
 
 
 if(GINKGO_BUILD_TESTS)
-    set(MEMORYCHECK_SUPPRESSIONS_FILE
-        "${Ginkgo_SOURCE_DIR}/dev_tools/valgrind/suppressions.supp"
-        CACHE FILEPATH "" FORCE)
-    include(CTest)
+    # Configure CTest
+    configure_file(
+        ${CMAKE_CURRENT_LIST_DIR}/cmake/CTestCustom.cmake.in
+        ${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake @ONLY)
+
     enable_testing()
+    include(CTest)
 endif()
 
 
diff --git a/CTestConfig.cmake b/CTestConfig.cmake
@@ -0,0 +1,7 @@
+set(CTEST_PROJECT_NAME "Ginkgo Project")
+set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC")
+
+set(CTEST_DROP_METHOD "http")
+set(CTEST_DROP_SITE "my.cdash.org")
+set(CTEST_DROP_LOCATION "/submit.php?project=Ginkgo+Project")
+set(CTEST_DROP_SITE_CDASH TRUE)
diff --git a/README.md b/README.md
@@ -89,11 +89,15 @@ Ginkgo adds the following additional switches to control what is being built:
     Ginkgo's documentation. The default is `OFF`.
 *   `-DGINKGO_EXPORT_BUILD_DIR={ON, OFF}` adds the Ginkgo build directory to the
     CMake package registry. The default is `OFF`.
-*   `-DCMAKE_INSTALL_PREFIX=path` sets the installation path for `make install`.
-    The default value is usually something like `/usr/local`
 *   `-DGINKGO_VERBOSE_LEVEL=integer` sets the verbosity of Ginkgo.
     * `0` disables all output in the main libraries,
     * `1` enables a few important messages related to unexpected behavior (default).
+*   `-DCMAKE_INSTALL_PREFIX=path` sets the installation path for `make install`.
+    The default value is usually something like `/usr/local`
+*   `-DCMAKE_BUILD_TYPE=type` specifies which configuration will be used for
+    this build of Ginkgo. There are no default. Supported values are CMake's
+    standard build types such as `DEBUG` and `RELEASE` and the Ginkgo specific 
+	`COVERAGE`, `ASAN` (AddressSanitizer) and `TSAN` (ThreadSanitizer) types.
 *   `-DBUILD_SHARED_LIBS={ON, OFF}` builds ginkgo as shared libraries (`OFF`)
     or as dynamic libraries (`ON`), default is `ON`
 *   `-DCMAKE_CUDA_HOST_COMPILER=path` instructs the build system to explicitly
@@ -156,9 +160,11 @@ Ginkgo to try to use an external version of a package. For this, set the CMake
 option `-DGINKGO_USE_EXTERNAL_<package>=ON`.
 
 ### Running the unit tests
-
 You need to compile ginkgo with `-DGINKGO_BUILD_TESTS=ON` option to be able to run the
-tests. Use the following command inside the build folder to run all tests:
+tests. 
+
+#### Using make test
+After configuring Ginkgo, use the following command inside the build folder to run all tests:
 
 ```sh
 make test
@@ -180,6 +186,34 @@ run the following from the build folder:
 
 where `path/to/test` is the path returned by `make test`.
 
+
+#### Using CTest 
+The tests can also be ran through CTest from the command line, for example when
+in a configured build directory:
+
+```sh 
+ctest -T start -T build -T test -T submit
+```
+
+Will start a new test campaign (usually in `Experimental` mode), build Ginkgo
+with the set configuration, run the tests and submit the results to our CDash
+dashboard.
+
+
+Another option is to use Ginkgo's CTest script which is configured to build
+Ginkgo with default settings, runs the tests and submits the test to our CDash
+dashboard automatically.
+
+To run the script, use the following command:
+
+```sh
+ctest -S cmake/CTestScript.cmake
+```
+
+The default settings are for our own CI system. Feel free to configure the
+script before launching it through variables or by directly changing its values.
+A documentation can be found in the script itself.
+
 ### Running the benchmarks
 
 In addition to the unit tests designed to verify correctness, Ginkgo also
diff --git a/cmake/CTestCustom.cmake.in b/cmake/CTestCustom.cmake.in
@@ -0,0 +1,21 @@
+list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE
+  # Exclude list set by cmake
+  @CUSTOM_COVERAGE_EXCLUDE@
+
+  # Exclude try_compile sources from coverage results:
+  "/CMakeFiles/CMakeTmp/"
+
+  "third_party"
+
+  "test"
+
+  "c\\+\\+"
+)
+
+set(CTEST_SOURCE_DIRECTORY "@Ginkgo_SOURCE_DIR@" CACHE STRING "" FORCE)
+set(CTEST_BINARY_DIRECTORY "@Ginkgo_BINARY_DIR@" CACHE STRING "" FORCE)
+
+set(CTEST_MEMORYCHECK_COMMAND_OPTIONS "--trace-children=yes --leak-check=full")
+set(CTEST_MEMORYCHECK_SUPPRESSIONS_FILE "@Ginkgo_SOURCE_DIR@/dev_tools/valgrind/suppressions.supp")
+
+set(CTEST_COVERAGE_EXTRA_FLAGS "-p")
diff --git a/cmake/CTestScript.cmake b/cmake/CTestScript.cmake
@@ -0,0 +1,143 @@
+#.rst:
+# Ginkgo CTestScript
+# -------
+#
+# Runs our tests through CTest, with support for Coverage or memory checking.
+#
+# This script provides a full CTest run whith result submission to Ginkgo's
+# CDash dashboard. The supported runs are:
+# + With or without coverage, requires the gcov tool.
+# + With or without address sanitizers.
+# + With or without thread sanitizers.
+# + With or without valgrind, requires the valgrind tool.
+#
+# Note that only one of these can be ran at once, as the build types
+# conflict. Ginkgo is always configured with CUDA, OpenMP and Reference
+# support. The results are always sent to the dashboard:
+# https://my.cdash.org/index.php?project=Ginkgo+Project
+#
+# Running the script
+# ^^^^^^^^^^^^^^^^^^
+#
+# To run the script, launch the command `ctest -S cmake/CTestScript.cmake`
+# from the Ginkgo's source directory. The default settings are for the CI
+# system's DEBUG tests run. To configure the script use standard CMake `-D`
+# parameters. For example, the following command runs coverage.
+#
+# `ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=COVERAGE`
+#
+# Instead, this runs the ThreadSanitizer:
+#
+# `ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=TSAN
+#     -DCTEST_MEMORYCHECK_TYPE=ThreadSanitizer`
+#
+# Input Variables
+# ^^^^^^^^^^^^^^^^
+#
+# This script can be configured with the following input variables:
+#
+# ``CTEST_SOURCE_DIRECTORY``
+# Where the sources are located. By default, the current directory.
+#
+# ``CTEST_BINARY_DIRECTORY``
+# In which directory should the sources be builts. Default, `./build`.
+#
+# ``CTEST_SITE``
+# A string to describe the machine this is ran on. Default FineCI.
+#
+# ``CTEST_CMAKE_GENERATOR``
+# Which generator should be used for the build. Default `Unix Makefiles`
+#
+# ``CTEST_BUILD_CONFIGURATION``
+# Which configuration should Ginkgo be built with. Default `DEBUG`.
+# The supported values are: COVERAGE, ASAN, TSAN, DEBUG and RELEASE.
+#
+# ``CTEST_TEST_MODEL``
+# Which CTest test model should be used. Default `Continuous`.
+# The supported values are the same as CTest's, namely:
+# Experimental, Nightly, Continuous.
+#
+# ``CTEST_BUILD_NAME``
+# The name of the build being ran. Default: `CTEST_BUILD_CONFIGURATION`
+#
+# ``CTEST_MEMORYCHECK_TYPE``
+# Whether memorycheck should be ran. Default: `None`. Supported values are:
+# Valgrind, ThreadSanitizer, AddressSanitizer and None.
+#
+
+if (NOT DEFINED CTEST_SOURCE_DIRECTORY)
+    set(CTEST_SOURCE_DIRECTORY ".")
+endif()
+
+if (NOT DEFINED CTEST_BINARY_DIRECTORY)
+    set(CTEST_BINARY_DIRECTORY "./build")
+endif()
+
+if (NOT DEFINED CTEST_SITE)
+    set(CTEST_SITE "Linux-FineCI")
+endif()
+
+if (NOT DEFINED CTEST_CMAKE_GENERATOR)
+    set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+endif()
+
+# Supported: COVERAGE, ASAN, TSAN, DEBUG and RELEASE
+if (NOT DEFINED CTEST_BUILD_CONFIGURATION)
+    set(CTEST_BUILD_CONFIGURATION "DEBUG")
+endif()
+
+if (NOT DEFINED CTEST_TEST_MODEL)
+    set(CTEST_TEST_MODEL "Continuous")
+endif()
+
+if (NOT DEFINED CTEST_BUILD_NAME)
+    set(CTEST_BUILD_NAME "${CTEST_BUILD_CONFIGURATION}")
+endif()
+
+#Supported: Valgrind, ThreadSanitizer, AddressSanitizer.
+if (NOT DEFINED CTEST_MEMORYCHECK_TYPE)
+    set(CTEST_MEMORYCHECK_TYPE "None")
+endif()
+
+# Find coverage and valgrind tools
+if(CTEST_MEMORYCHECK_TYPE STREQUAL "Valgrind")
+    find_program(CTEST_MEMORYCHECK_COMMAND valgrind)
+    set(CTEST_BUILD_NAME "Valgrind")
+endif()
+
+if(CTEST_BUILD_CONFIGURATION STREQUAL "COVERAGE")
+    find_program(CTEST_COVERAGE_COMMAND gcov)
+endif()
+
+include(ProcessorCount)
+ProcessorCount(PROC_COUNT)
+if(NOT PROC_COUNT EQUAL 0)
+    if(NOT WIN32)
+        set(CTEST_BUILD_FLAGS "-j${PROC_COUNT}")
+    endif(NOT WIN32)
+endif()
+
+ctest_start("${CTEST_TEST_MODEL}")
+ctest_submit(PARTS Start)
+
+ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}" OPTIONS "-DGINKGO_BUILD_REFERENCE=ON;-DGINKGO_BUILD_OMP=ON;-DGINKGO_BUILD_CUDA=ON;-DCMAKE_BUILD_TYPE=${CTEST_BUILD_CONFIGURATION}" APPEND)
+ctest_submit(PARTS Configure)
+
+ctest_read_custom_files( ${CTEST_BINARY_DIRECTORY} )
+
+ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}" APPEND)
+ctest_submit(PARTS Build)
+
+ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}" APPEND)
+ctest_submit(PARTS Test)
+
+if (CTEST_BUILD_CONFIGURATION STREQUAL "COVERAGE")
+    ctest_coverage(BUILD "${CTEST_BINARY_DIRECTORY}" APPEND)
+    ctest_submit(PARTS Coverage)
+endif()
+
+if(NOT CTEST_MEMORYCHECK_TYPE STREQUAL "None")
+    ctest_memcheck(BUILD "${CTEST_BINARY_DIRECTORY}" APPEND)
+    ctest_submit(PARTS MemCheck)
+endif()
+
diff --git a/cmake/build_type_helpers.cmake b/cmake/build_type_helpers.cmake
