[CMake] Running unit test as part of the build

[Michael Hertling]

On 09/13/2011 11:05 AM, Michael Wild wrote:
> On 09/12/2011 09:06 PM, Erik Johansson wrote:
>> On Mon, Sep 12, 2011 at 20:30, Michael Wild <themiwi at gmail.com> wrote:
>>> How about using a custom command that runs the unit test using a
>>> wrapper script that upon successful completion creates a stamp-file and
>>> depends upon the unit-test executable target itself?
>>
>> This seems to work:
>>
>> add_executable(unittest ${test_SRCS})
>>
>> set(unittest_stamp
>>   "${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/unittest.stamp")
>> add_custom_command(OUTPUT "${unittest_stamp}"
>>   COMMAND ${CMAKE_CTEST_COMMAND} $(ARGS)
>>   COMMAND ${CMAKE_COMMAND} -E touch "${unittest_stamp}"
>>   COMMENT "Running unit test"
>>   DEPENDS unittest)
>> add_custom_target(unittest_run ALL DEPENDS "${unittest_stamp}")
>>
>> // Erik
>>
> 
> I think the OP wanted to run the tests individually, so I don't think
> you should be using CMAKE_CTEST_COMMAND, but rather the unittest
> executable directly, such that only tests that failed or where the test
> executable changed are re-run.
> 
> 
> E.g.:
> 
> ###################################################################
> cmake_minimum_required(VERSION 2.8)
> project(tests CXX)
> 
> # usually one would put this in a proper file, not WRITE it like this
> file(WRITE ${CMAKE_BINARY_DIR}/run_test.cmake "
> if(NOT EXISTS \${TEST_EXE})
>   message(FATAL_ERROR \"'\${TEST_EXE}' does not exist\")
> endif()
> execute_process(COMMAND \${TEST_EXE}
>   RESULT_VARIABLE result)
> if(result)
>   message(\"\${TEST_NAME} failed\")
> else()
>   execute_process(COMMAND \${CMAKE_COMMAND} -E touch \${TEST_STAMP})
> endif()
> ")
> 
> # create some tests, number 2 fails
> file(WRITE ${CMAKE_BINARY_DIR}/test.cpp.in "
> // need to sleep a bit for the time stamps to work...
> #include <unistd.h>
> int main() {sleep(1); return @exit_status@;}
> ")
> foreach(i RANGE 1 3)
>   if(i EQUAL 2)
>     set(exit_status 1)
>   else()
>     set(exit_status 0)
>   endif()
>   configure_file(${CMAKE_BINARY_DIR}/test.cpp.in
>     ${CMAKE_BINARY_DIR}/test${i}.cpp @ONLY)
> endforeach()
> 
> add_custom_target(alltests ALL)
> foreach(t test1 test2 test3)
>   add_executable(unit${t} ${CMAKE_BINARY_DIR}/${t}.cpp)
>   set(stamp
>     "${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/unit${t}.stamp")
>   add_custom_command(OUTPUT "${stamp}"
>     COMMAND ${CMAKE_COMMAND} -DTEST_NAME=${t}
>       -DTEST_EXE=$<TARGET_FILE:unit${t}> -DTEST_STAMP=${stamp}
>       -P ${CMAKE_BINARY_DIR}/run_test.cmake
>     COMMENT "Running unit test ${t}"
>     DEPENDS unit${t})
>   add_custom_target(run_${t} ALL DEPENDS "${stamp}")
>   add_dependencies(alltests run_${t})
> endforeach()
> ###################################################################
> 
> HTH
> 
> Michael

Alternatively, you might run the test as a POST_BUILD custom command
which is attached to the unit test executable's target; look here:

# CMakeLists.txt:
CMAKE_MINIMUM_REQUIRED(VERSION 2.8 FATAL_ERROR)
PROJECT(UNITTEST C)
ENABLE_TESTING()
SET(CMAKE_VERBOSE_MAKEFILE ON)
FILE(WRITE ${CMAKE_BINARY_DIR}/pass.c "int main(void){return 0;}\n")
ADD_EXECUTABLE(pass EXCLUDE_FROM_ALL pass.c)
ADD_TEST(NAME pass COMMAND pass)
ADD_CUSTOM_COMMAND(TARGET pass POST_BUILD COMMAND ${CMAKE_COMMAND}
    -DTEST=pass
    -DTARGET=$<TARGET_FILE:pass>
    -P ${CMAKE_SOURCE_DIR}/unittest.cmake)
FILE(WRITE ${CMAKE_BINARY_DIR}/fail.c "int main(void){return 1;}\n")
ADD_EXECUTABLE(fail EXCLUDE_FROM_ALL fail.c)
ADD_TEST(NAME fail COMMAND fail)
ADD_CUSTOM_COMMAND(TARGET fail POST_BUILD COMMAND ${CMAKE_COMMAND}
    -DTEST=fail
    -DTARGET=$<TARGET_FILE:fail>
    -P ${CMAKE_SOURCE_DIR}/unittest.cmake)

# unittest.cmake:
EXECUTE_PROCESS(
    COMMAND ${CMAKE_CTEST_COMMAND} -R ${TEST}
    RESULT_VARIABLE RESULT)
IF(NOT RESULT EQUAL 0)
    FILE(REMOVE ${TARGET})
    MESSAGE(FATAL_ERROR "Test ${TEST} [${TARGET}] failed.")
ENDIF()

The unittest.cmake script runs the test on the unit test executable
target, and if the test fails, it removes the unit test executable
and bails out with a FATAL_ERROR. In this manner, the failing unit
test's executable is relinked the next time Make is run, and the
custom command performing the test is also run again. OTOH, if the
test passes, its executable will not be touched during the next Make
run, and the attached custom command will not be invoked, so the test
is dropped - as the OP desires - until the test executable is rebuilt.

Try "make pass" to see a successful unit test, and "make fail" for a
failing one. You will see "make pass" building the "pass" executable
and performing the succeeding test; a subsequent "make pass" will do
nothing unless "pass" is rebuilt. A "make fail", however, will always
build/relink and - unsuccessfully - perform its test; change fail.c to
return 0, and you will see "make fail" behave exactly like "make pass".

Effectively, this means using the unit test executable as a stamp file,
but in fact, you're already doing the same in your suggestions: If the
test has passed once, your stamp files are never removed, so the only
criterion to trigger the test again is a renewed unit test executable.
IMO, attaching the unit test to the unit test executable's target is
a more natural approach to the OP's concern. Moreover, one can drop
the custom target and the explicit stamp file.

Regards,

Michael