Skip to content

Commit 310e8f2

Browse files
Dario Berzanosawenzel
authored andcommitted
Improve CMake testing (O2-499)
Following discussions in WP3, tests are now wrapped by a script that handles timeouts and retries. Retries are not handled by native CTest. It is possible to ignore failures if some test has failed because of a timeout (not a CTest feature either). A separate logfile for each test is kept in the build directory. At the end of each test, the logfile is renamed to include the exit code in the file name. This makes it possible for the external process driving the build (alidist recipe, aliBuild, the C.I.) to keep logfiles and make them available even in case of success for further investigation. Timing out processes will have their stack trace collected before they are killed.
1 parent 23a254f commit 310e8f2

11 files changed

Lines changed: 252 additions & 47 deletions

File tree

CMakeLists.txt

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -230,15 +230,23 @@ if(CHECK_ROOTMACRO_COMPILE AND HAVESIMULATION)
230230
IF(NOT ${MACRO_FILE} IN_LIST IGNORE_MACROS)
231231
string(REPLACE ${CMAKE_SOURCE_DIR} "" MACRO_FILE_LABEL ${MACRO_FILE})
232232
MATH(EXPR TEST_NO "${TEST_NO} + 1")
233-
ADD_TEST(NAME ${MACRO_FILE_LABEL}
234-
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
235-
COMMAND root -n -b -l -q -e ".L ${MACRO_FILE}"
236-
)
233+
add_test_wrap(NAME ${MACRO_FILE_LABEL}
234+
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
235+
COMMAND root -n -b -l -q -e ".L ${MACRO_FILE}")
237236
SET_PROPERTY(TEST ${MACRO_FILE_LABEL} PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:${CMAKE_BINARY_DIR}/lib")
238237
SET_PROPERTY(TEST ${MACRO_FILE_LABEL} APPEND PROPERTY ENVIRONMENT "DYLD_LIBRARY_PATH=$ENV{DYLD_LIBRARY_PATH}:${CMAKE_BINARY_DIR}/lib")
239238
SET_PROPERTY(TEST ${MACRO_FILE_LABEL} APPEND PROPERTY ENVIRONMENT "ROOT_HIST=0")
240239
ELSE()
241-
MESSAGE(STATUS "EXCLUDING ROOT MACRO ${MACRO_FILE} FROM CHECK")
240+
MESSAGE(WARNING "ROOT macro ${MACRO_FILE} excluded from tests")
242241
ENDIF()
243242
ENDFOREACH()
244243
endif()
244+
245+
# Create tests wrapper (and make it executable)
246+
configure_file("${CMAKE_SOURCE_DIR}/cmake/tests-wrapper.sh.in"
247+
"${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/tests-wrapper.sh"
248+
@ONLY
249+
NEWLINE_STYLE UNIX)
250+
file(COPY "${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/tests-wrapper.sh"
251+
DESTINATION "${CMAKE_BINARY_DIR}"
252+
FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)

Detectors/MUON/MCH/Mapping/test/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ o2_generate_executable(EXE_NAME
1212
mch_mapping_test_bucket)
1313

1414
target_link_libraries(test_MCHMappingTest Boost::unit_test_framework)
15-
add_test(NAME test_MCHMappingTest COMMAND test_MCHMappingTest)
15+
add_test_wrap(NAME test_MCHMappingTest COMMAND test_MCHMappingTest)
1616

1717
if(benchmark_FOUND)
1818
o2_generate_executable(EXE_NAME

Detectors/MUON/MCH/Simulation/test/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@ O2_GENERATE_EXECUTABLE(
99
)
1010

1111
target_link_libraries(test_MCHSimulation Boost::unit_test_framework MCHSimulation)
12-
add_test(NAME test_MCHSimulation COMMAND test_MCHSimulation)
12+
add_test_wrap(NAME test_MCHSimulation COMMAND test_MCHSimulation)

Detectors/TPC/simulation/CMakeLists.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,9 @@ O2_GENERATE_TESTS(
8888

8989
# add the TPC run sim as a unit test (if simulation was enabled)
9090
if (HAVESIMULATION)
91-
add_test(NAME tpcsim_G4 COMMAND ${CMAKE_BINARY_DIR}/bin/runTPC -n 2 -e TGeant4)
91+
add_test_wrap(NAME tpcsim_G4 COMMAND ${CMAKE_BINARY_DIR}/bin/runTPC -n 2 -e TGeant4)
9292
set_tests_properties(tpcsim_G4 PROPERTIES PASS_REGULAR_EXPRESSION "Macro finished succesfully")
93-
add_test(NAME tpcsim_G3 COMMAND ${CMAKE_BINARY_DIR}/bin/runTPC -n 2 -e TGeant3)
93+
add_test_wrap(NAME tpcsim_G3 COMMAND ${CMAKE_BINARY_DIR}/bin/runTPC -n 2 -e TGeant3)
9494
set_tests_properties(tpcsim_G3 PROPERTIES PASS_REGULAR_EXPRESSION "Macro finished succesfully")
9595
# sets the necessary environment
9696
set_tests_properties(tpcsim_G3 tpcsim_G4 PROPERTIES ENVIRONMENT VMCWORKDIR=${CMAKE_SOURCE_DIR})

Examples/flp2epn-distributed/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ ForEach (_file RANGE 0 ${_length})
5656
)
5757
EndForEach (_file RANGE 0 ${_length})
5858

59-
#add_test(NAME run_flp2epn_distributed COMMAND ${CMAKE_BINARY_DIR}/Examples/flp2epn-distributed/test/testFLP2EPN-distributed.sh)
59+
#add_test_wrap(NAME run_flp2epn_distributed COMMAND ${CMAKE_BINARY_DIR}/Examples/flp2epn-distributed/test/testFLP2EPN-distributed.sh)
6060
#set_tests_properties(run_flp2epn_distributed PROPERTIES TIMEOUT "30")
6161
#set_tests_properties(run_flp2epn_distributed PROPERTIES PASS_REGULAR_EXPRESSION "acknowledged after")
6262

Framework/DebugGUI/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,6 @@ O2_GENERATE_EXECUTABLE(
7070
MODULE_LIBRARY_NAME ${LIBRARY_NAME}
7171
BUCKET_NAME ${MODULE_BUCKET_NAME}
7272
)
73-
add_test(NAME test_DebugGUI_test_ImGUIHeadless COMMAND test_DebugGUI_test_ImGUIHeadless)
73+
add_test_wrap(NAME test_DebugGUI_test_ImGUIHeadless COMMAND test_DebugGUI_test_ImGUIHeadless)
7474
target_link_libraries(test_DebugGUI_test_ImGUIHeadless Boost::unit_test_framework)
7575
set_tests_properties(test_DebugGUI_test_ImGUIHeadless PROPERTIES TIMEOUT 30)

Utilities/MCStepLogger/CMakeLists.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ O2_GENERATE_EXECUTABLE(
5151

5252
# check correct functioning of the logger and the MC analysis chain
5353
if (HAVESIMULATION)
54-
add_test(NAME mcloggertest COMMAND ${CMAKE_BINARY_DIR}/bin/runTPC -n 1 -e TGeant3)
54+
add_test_wrap(NAME mcloggertest COMMAND ${CMAKE_BINARY_DIR}/bin/runTPC -n 1 -e TGeant3)
5555
# tests if the logger was active
5656
set_tests_properties(mcloggertest PROPERTIES PASS_REGULAR_EXPRESSION "VolName.*COUNT")
5757
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
@@ -65,15 +65,15 @@ if (HAVESIMULATION)
6565
# check whether StepLogger output can be written to ROOT file
6666
# fix output file name for logged data
6767
set(STEPLOGGER_ROOTFILE "MCStepLoggerOutput_test.root")
68-
add_test(NAME mcloggertest_tofile COMMAND ${CMAKE_BINARY_DIR}/bin/runTPC -n 1 -e TGeant3)
68+
add_test_wrap(NAME mcloggertest_tofile COMMAND ${CMAKE_BINARY_DIR}/bin/runTPC -n 1 -e TGeant3)
6969
set_tests_properties(mcloggertest_tofile PROPERTIES ENVIRONMENT ${PRELOAD}=${CMAKE_BINARY_DIR}/lib/libMCStepLogger${CMAKE_SHARED_LIBRARY_SUFFIX})
7070
# set environment accordingly
7171
set_property(TEST mcloggertest_tofile APPEND PROPERTY ENVIRONMENT VMCWORKDIR=${CMAKE_SOURCE_DIR} MCSTEPLOG_TTREE=1 MCSTEPLOG_OUTFILE=${STEPLOGGER_ROOTFILE})
7272

7373
# check for working analysis
7474
# fix output file name for analysis
7575
set(MCANALYSIS_ROOTFILE "BasicMCAnalysis.root")
76-
add_test(NAME basicmcanalysis COMMAND ${CMAKE_BINARY_DIR}/bin/mcStepAnalysis analyze -f ${STEPLOGGER_ROOTFILE} -o ${MCANALYSIS_ROOTFILE} -l testLabel)
76+
add_test_wrap(NAME basicmcanalysis COMMAND ${CMAKE_BINARY_DIR}/bin/mcStepAnalysis analyze -f ${STEPLOGGER_ROOTFILE} -o ${MCANALYSIS_ROOTFILE} -l testLabel)
7777
set_tests_properties(basicmcanalysis PROPERTIES DEPENDS mcloggertest_tofile)
7878
# set environment accordingly
7979
set_property(TEST basicmcanalysis APPEND PROPERTY ENVIRONMENT VMCWORKDIR=${CMAKE_SOURCE_DIR})

cmake/O2Utils.cmake

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,55 @@ CHECK_VARIABLE(PARSED_ARGS_WORKFLOW_NAME "You must provide an executable name")
392392

393393
endfunction(O2_FRAMEWORK_WORKFLOW)
394394

395+
#------------------------------------------------------------------------------
396+
# add_test_wrap
397+
# Same as add_test() but optionally retry up to MAX_ATTEMPTS times upon failure.
398+
# This is achieved by using a shell script wrapper
399+
# arg NAME
400+
# arg COMMAND
401+
# arg WORKING_DIRECTORY
402+
# arg CONFIGURATIONS
403+
# arg DONT_FAIL_ON_TIMEOUT - if specified, it will not fail on timeouts
404+
# arg MAX_ATTEMPTS - the maximum number of attempts
405+
# arg TIMEOUT - the maximum number of attempts
406+
function(add_test_wrap)
407+
cmake_parse_arguments(PARSE_ARGV 0 "L"
408+
"DONT_FAIL_ON_TIMEOUT"
409+
"NAME;WORKING_DIRECTORY;MAX_ATTEMPTS;TIMEOUT"
410+
"COMMAND;CONFIGURATIONS")
411+
if("${L_MAX_ATTEMPTS}" GREATER 1)
412+
# Warn only for tests where retry has been requested
413+
message(WARNING "Test ${L_NAME} will be retried max ${L_MAX_ATTEMPTS} times")
414+
endif()
415+
416+
if(NOT L_TIMEOUT)
417+
set(L_TIMEOUT 100) # default timeout (seconds)
418+
endif()
419+
if(NOT L_MAX_ATTEMPTS)
420+
set(L_MAX_ATTEMPTS 1) # default number of attempts
421+
endif()
422+
if(L_DONT_FAIL_ON_TIMEOUT)
423+
set(L_DONT_FAIL_ON_TIMEOUT "--dont-fail-on-timeout")
424+
else()
425+
set(L_DONT_FAIL_ON_TIMEOUT "")
426+
endif()
427+
math(EXPR CTEST_TIMEOUT "(20 + ${L_TIMEOUT}) * ${L_MAX_ATTEMPTS}")
428+
429+
if(WIN32)
430+
# Shell script does not work on Windows. Use plain add_test() with no retry, use plain timeout
431+
add_test(NAME "${L_NAME}"
432+
COMMAND ${L_COMMAND}
433+
WORKING_DIRECTORY "${L_WORKING_DIRECTORY}"
434+
CONFIGURATIONS "${L_CONFIGURATIONS}")
435+
set_tests_properties(${L_NAME} PROPERTIES TIMEOUT ${L_TIMEOUT})
436+
else()
437+
add_test(NAME "${L_NAME}"
438+
COMMAND "${CMAKE_BINARY_DIR}/tests-wrapper.sh" "--name" "${L_NAME}" "--max-attempts" "${L_MAX_ATTEMPTS}" "--timeout" "${L_TIMEOUT}" ${L_DONT_FAIL_ON_TIMEOUT} "--" ${L_COMMAND}
439+
WORKING_DIRECTORY "${L_WORKING_DIRECTORY}"
440+
CONFIGURATIONS "${L_CONFIGURATIONS}")
441+
set_tests_properties(${L_NAME} PROPERTIES TIMEOUT ${CTEST_TIMEOUT})
442+
endif()
443+
endfunction()
395444

396445
#------------------------------------------------------------------------------
397446
# O2_GENERATE_TESTS
@@ -403,7 +452,7 @@ function(O2_GENERATE_TESTS)
403452
cmake_parse_arguments(
404453
PARSED_ARGS
405454
"" # bool args
406-
"BUCKET_NAME;MODULE_LIBRARY_NAME;TIMEOUT" # mono-valued arguments
455+
"BUCKET_NAME;MODULE_LIBRARY_NAME;TIMEOUT;MAX_ATTEMPTS" # mono-valued arguments
407456
"TEST_SRCS;COMMAND_LINE_ARGS" # multi-valued arguments
408457
${ARGN} # arguments
409458
)
@@ -424,10 +473,11 @@ function(O2_GENERATE_TESTS)
424473
NO_INSTALL FALSE
425474
)
426475
target_link_libraries(${test_name} Boost::unit_test_framework)
427-
add_test(NAME ${test_name} COMMAND ${test_name} ${PARSED_ARGS_COMMAND_LINE_ARGS})
428-
if (PARSED_ARGS_TIMEOUT)
429-
set_tests_properties(${test_name} PROPERTIES TIMEOUT ${PARSED_ARGS_TIMEOUT})
430-
endif()
476+
add_test_wrap(NAME ${test_name}
477+
DONT_FAIL_ON_TIMEOUT
478+
MAX_ATTEMPTS "${PARSED_ARGS_MAX_ATTEMPTS}"
479+
TIMEOUT "${PARSED_ARGS_TIMEOUT}"
480+
COMMAND ${test_name} ${PARSED_ARGS_COMMAND_LINE_ARGS})
431481
endforeach ()
432482
endfunction()
433483

cmake/tests-wrapper.sh.in

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
#!/bin/bash -e
2+
3+
# tests-wrapper.sh -- wrap execution of CTest scripts by adding retry
4+
#
5+
# Usage (through CMake):
6+
#
7+
# tests-wrapper.sh --name <name> [--max-attempts N] [--timeout T] \
8+
# --dont-fail-on-timeout] -- prog [arg1 [arg2...]]
9+
10+
MAX_ATTEMPTS=1
11+
TIMEOUT= # no timeout
12+
DONT_FAIL_ON_TIMEOUT= # if it times out, it fails
13+
ARGS=("$@")
14+
15+
while [[ $# -gt 0 ]]; do
16+
case "$1" in
17+
--)
18+
shift
19+
break
20+
;;
21+
--name)
22+
TEST_NAME="$2"
23+
shift 2
24+
;;
25+
--max-attempts)
26+
MAX_ATTEMPTS="$2"
27+
shift 2
28+
;;
29+
--timeout)
30+
TIMEOUT="$2"
31+
shift 2
32+
;;
33+
--dont-fail-on-timeout)
34+
DONT_FAIL_ON_TIMEOUT=1
35+
shift
36+
;;
37+
*)
38+
echo "Parameter unknown: $1" >&2
39+
exit 1
40+
;;
41+
esac
42+
done
43+
44+
# Check mandatory parmeters
45+
if [[ ! $TEST_NAME ]]; then
46+
echo "Test name is mandatory" >&2
47+
exit 1
48+
fi
49+
50+
LOG="@CMAKE_BINARY_DIR@/test_logs/${TEST_NAME//\//_}.log"
51+
mkdir -p "$(dirname "$LOG")"
52+
rm -f "$LOG"* &> /dev/null
53+
exec &> >(tee "$LOG")
54+
55+
function banner() {
56+
echo "=== $TEST_NAME - $1 ===" >&2
57+
}
58+
59+
banner "Starting test. Max attempts: $MAX_ATTEMPTS.${TIMEOUT:+" Timeout per attempt: $TIMEOUT."}${DONT_FAIL_ON_TIMEOUT:+" Timeouts are non-fatal."}"
60+
for A in "${ARGS[@]}"; do
61+
banner "Argument: $A"
62+
done
63+
banner "Current working directory: $PWD"
64+
banner "Environment"
65+
env
66+
banner "/Environment"
67+
68+
# Do we have timeout?
69+
TIMEOUT_EXEC=timeout
70+
TIMEOUT_CMD=
71+
TIMEOUT_PSTACK=
72+
type $TIMEOUT_EXEC &> /dev/null || TIMEOUT_EXEC=gtimeout
73+
type $TIMEOUT_EXEC &> /dev/null || TIMEOUT_EXEC=
74+
if [[ $TIMEOUT_EXEC && $TIMEOUT ]]; then
75+
# Kill with 15; if after 10 seconds it's still alive, send 9
76+
TIMEOUT_CMD="$TIMEOUT_EXEC --signal=SIGTERM --kill-after=10s ${TIMEOUT}s"
77+
# Get a stack trace, if possible, shortly before sending the first SIGTERM
78+
if type pstack &> /dev/null; then
79+
export GDB=$(which gdb 2> /dev/null)
80+
TIMEOUT_PSTACK=$(( TIMEOUT - 10 ))
81+
if [[ $TIMEOUT_PSTACK -lt 10 ]]; then
82+
TIMEOUT_PSTACK=10
83+
fi
84+
else
85+
banner "Will not get stack trace of processes timing out: pstack not found"
86+
fi
87+
fi
88+
89+
banner "Timeout prefix: $TIMEOUT_CMD"
90+
91+
CMD="$1"
92+
shift
93+
type "$CMD" &> /dev/null || CMD="@CMAKE_BINARY_DIR@/bin/$CMD"
94+
95+
for ((ATTEMPT=1; ATTEMPT<=MAX_ATTEMPTS; ATTEMPT++)); do
96+
banner "Running $CMD with args $* (attempt $ATTEMPT/$MAX_ATTEMPTS)"
97+
ERR=0
98+
rm -f "${LOG}.bt"
99+
$TIMEOUT_CMD "$CMD" "$@" &
100+
REAL_PID=$(pgrep -P$! 2> /dev/null || true)
101+
if [[ $REAL_PID && $TIMEOUT_PSTACK ]]; then
102+
banner "Process running as $REAL_PID for attempt $ATTEMPT"
103+
FINISHED=
104+
for ((I=0; I<TIMEOUT_PSTACK; I++)); do
105+
if ! kill -0 $REAL_PID &> /dev/null; then
106+
FINISHED=1
107+
break
108+
fi
109+
sleep 1
110+
done
111+
if [[ ! $FINISHED ]]; then
112+
pstack $REAL_PID &> "${LOG}.bt" || true
113+
fi
114+
fi
115+
wait $! || ERR=$? # wait timeout process, not real PID
116+
if [[ $ERR == 0 ]]; then
117+
banner "Test finished with success after $ATTEMPT attempts, exiting"
118+
mv "$LOG" "${LOG}.0"
119+
exit 0
120+
else
121+
banner "Test attempt $ATTEMPT/$MAX_ATTEMPTS failed with exit code $ERR"
122+
if [[ -e "${LOG}.bt" ]]; then
123+
banner "Stack trace follows for attempt $ATTEMPT"
124+
cat "${LOG}.bt"
125+
rm -f "${LOG}.bt"
126+
banner "End of stack trace for attempt $ATTEMPT"
127+
fi
128+
fi
129+
done
130+
131+
mv "$LOG" "${LOG}.${ERR}" # log file will contain exitcode in name
132+
banner "Test failed after $MAX_ATTEMPTS attempts with $ERR"
133+
if [[ $DONT_FAIL_ON_TIMEOUT && $ERR == 124 ]]; then
134+
# man timeout --> 124 is for "timed out"
135+
banner "Reason for failure: timeout, explicitly set as non-fatal. Exiting with 0"
136+
exit 0
137+
fi
138+
exit 1

0 commit comments

Comments
 (0)