[Cmake-commits] CMake branch, next, updated. v3.2.0-978-gecce429
Bill Hoffman
bill.hoffman at kitware.com
Tue Mar 10 18:20:27 EDT 2015
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "CMake".
The branch, next has been updated
via ecce4291e6ccf51902d16cff862d3a0c729f4802 (commit)
via 8c68af5e7628d7d18bd3e34e300044c459b9acfe (commit)
via 169c867f5671fa6cca2fde53b9d7732c348c7499 (commit)
via 5c3c29ffd3a36e710792d7b9954bf3ec95cee58a (commit)
from d88859dfcdc4e4dd1e00a854bb4115d7b230a72a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=ecce4291e6ccf51902d16cff862d3a0c729f4802
commit ecce4291e6ccf51902d16cff862d3a0c729f4802
Merge: d88859d 8c68af5
Author: Bill Hoffman <bill.hoffman at kitware.com>
AuthorDate: Tue Mar 10 18:20:24 2015 -0400
Commit: CMake Topic Stage <kwrobot at kitware.com>
CommitDate: Tue Mar 10 18:20:24 2015 -0400
Merge topic 'add_repeat_testing' into next
8c68af5e Add a test for the run until fail ctest option.
169c867f Fix some unitialized class variables.
5c3c29ff Add a new --repeat-until-fail option to ctest.
http://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=8c68af5e7628d7d18bd3e34e300044c459b9acfe
commit 8c68af5e7628d7d18bd3e34e300044c459b9acfe
Author: Bill Hoffman <bill.hoffman at kitware.com>
AuthorDate: Tue Mar 10 18:08:01 2015 -0400
Commit: Bill Hoffman <bill.hoffman at kitware.com>
CommitDate: Tue Mar 10 18:08:01 2015 -0400
Add a test for the run until fail ctest option.
diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt
index 08765de..4300c7e 100644
--- a/Tests/CMakeLists.txt
+++ b/Tests/CMakeLists.txt
@@ -2583,6 +2583,13 @@ ${CMake_BINARY_DIR}/bin/cmake -DDIR=dev -P ${CMake_SOURCE_DIR}/Utilities/Release
ADD_TEST_MACRO(CTestTestSerialOrder ${CMAKE_CTEST_COMMAND}
--output-on-failure -C "\${CTestTest_CONFIG}")
+ add_test_macro(CTestTestRunUntilFail ${CMAKE_CTEST_COMMAND}
+ --repeat-until-fail 4 -C "\${CTestTest_CONFIG}")
+ set_tests_properties(CTestTestRunUntilFail PROPERTIES
+ PASS_REGULAR_EXPRESSION
+ "1/2.*Test #1:.* initialization.*Passed.*Test #2.*test1.*Failed"
+ )
+
if(NOT BORLAND)
set(CTestLimitDashJ_CTEST_OPTIONS --force-new-ctest-process)
add_test_macro(CTestLimitDashJ ${CMAKE_CTEST_COMMAND} -j 4
diff --git a/Tests/CTestTestRunUntilFail/CMakeLists.txt b/Tests/CTestTestRunUntilFail/CMakeLists.txt
new file mode 100644
index 0000000..7788d73
--- /dev/null
+++ b/Tests/CTestTestRunUntilFail/CMakeLists.txt
@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 2.8.12)
+project(CTestTestSerialOrder)
+enable_testing()
+set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
+add_test(NAME initialization
+ COMMAND ${CMAKE_COMMAND}
+ "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+ -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
+add_test(NAME test1
+ COMMAND ${CMAKE_COMMAND}
+ "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+ -P "${CMAKE_CURRENT_SOURCE_DIR}/test1.cmake")
+set_tests_properties(test1 PROPERTIES DEPENDS "initialization")
diff --git a/Tests/CTestTestRunUntilFail/init.cmake b/Tests/CTestTestRunUntilFail/init.cmake
new file mode 100644
index 0000000..560fb5e
--- /dev/null
+++ b/Tests/CTestTestRunUntilFail/init.cmake
@@ -0,0 +1 @@
+file(WRITE "${TEST_OUTPUT_FILE}" "0")
diff --git a/Tests/CTestTestRunUntilFail/test1.cmake b/Tests/CTestTestRunUntilFail/test1.cmake
new file mode 100644
index 0000000..d7fafb8
--- /dev/null
+++ b/Tests/CTestTestRunUntilFail/test1.cmake
@@ -0,0 +1,8 @@
+message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
+file(READ "${TEST_OUTPUT_FILE}" COUNT)
+message("COUNT= ${COUNT}")
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
+if(${COUNT} EQUAL 2)
+ message(FATAL_ERROR "this test fails on the 2nd run")
+endif()
http://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=169c867f5671fa6cca2fde53b9d7732c348c7499
commit 169c867f5671fa6cca2fde53b9d7732c348c7499
Author: Bill Hoffman <bill.hoffman at kitware.com>
AuthorDate: Mon Mar 9 16:07:22 2015 -0400
Commit: Bill Hoffman <bill.hoffman at kitware.com>
CommitDate: Mon Mar 9 16:07:22 2015 -0400
Fix some unitialized class variables.
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 2f89963..55e2a44 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -328,6 +328,8 @@ cmCTest::cmCTest()
this->OutputTestOutputOnTestFailure = false;
this->ComputedCompressTestOutput = false;
this->ComputedCompressMemCheckOutput = false;
+ this->RepeatTests = 1; // default to run each test once
+ this->RepeatUntilFail = false;
if(cmSystemTools::GetEnv("CTEST_OUTPUT_ON_FAILURE"))
{
this->OutputTestOutputOnTestFailure = true;
http://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=5c3c29ffd3a36e710792d7b9954bf3ec95cee58a
commit 5c3c29ffd3a36e710792d7b9954bf3ec95cee58a
Author: Bill Hoffman <bill.hoffman at kitware.com>
AuthorDate: Thu Mar 5 16:51:10 2015 -0500
Commit: Bill Hoffman <bill.hoffman at kitware.com>
CommitDate: Thu Mar 5 16:51:10 2015 -0500
Add a new --repeat-until-fail option to ctest.
This option tells ctest to run each test N times until the test fails or
the N times have run. This is useful for finding random failing tests.
diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index eb33d8e..bd090db 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -121,6 +121,11 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
this->RunningCount += GetProcessorsUsed(test);
cmCTestRunTest* testRun = new cmCTestRunTest(this->TestHandler);
+ if(this->CTest->GetRepeatUntilFail())
+ {
+ testRun->SetRunUntilFailOn();
+ testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
+ }
testRun->SetIndex(test);
testRun->SetTestProperties(this->Properties[test]);
@@ -289,7 +294,13 @@ bool cmCTestMultiProcessHandler::CheckOutput()
cmCTestRunTest* p = *i;
int test = p->GetIndex();
- if(p->EndTest(this->Completed, this->Total, true))
+ bool testResult = p->EndTest(this->Completed, this->Total, true);
+ if(p->StartAgain())
+ {
+ this->Completed--; // remove the completed test because run again
+ continue;
+ }
+ if(testResult)
{
this->Passed->push_back(p->GetTestProperties()->Name);
}
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index 03131fd..6f72684 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -33,6 +33,9 @@ cmCTestRunTest::cmCTestRunTest(cmCTestTestHandler* handler)
this->CompressedOutput = "";
this->CompressionRatio = 2;
this->StopTimePassed = false;
+ this->NumberOfRunsLeft = 1; // default to 1 run of the test
+ this->RunUntilFail = false; // default to run the test once
+ this->RunAgain = false; // default to not having to run again
}
cmCTestRunTest::~cmCTestRunTest()
@@ -357,13 +360,50 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
this->MemCheckPostProcess();
this->ComputeWeightedCost();
}
- // Always push the current TestResult onto the
+ // If the test does not need to rerun push the current TestResult onto the
// TestHandler vector
- this->TestHandler->TestResults.push_back(this->TestResult);
+ if(!this->NeedsToRerun())
+ {
+ this->TestHandler->TestResults.push_back(this->TestResult);
+ }
delete this->TestProcess;
return passed;
}
+bool cmCTestRunTest::StartAgain()
+{
+ if(!this->RunAgain)
+ {
+ return false;
+ }
+ this->RunAgain = false; // reset
+ // change to tests directory
+ std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
+ cmSystemTools::ChangeDirectory(this->TestProperties->Directory);
+ this->StartTest(this->TotalNumberOfTests);
+ // change back
+ cmSystemTools::ChangeDirectory(current_dir);
+ return true;
+}
+
+bool cmCTestRunTest::NeedsToRerun()
+{
+ this->NumberOfRunsLeft--;
+ if(this->NumberOfRunsLeft == 0)
+ {
+ return false;
+ }
+ // if number of runs left is not 0, and we are running until
+ // we find a failed test, then return true so the test can be
+ // restarted
+ if(this->RunUntilFail
+ && this->TestResult.Status == cmCTestTestHandler::COMPLETED)
+ {
+ this->RunAgain = true;
+ return true;
+ }
+ return false;
+}
//----------------------------------------------------------------------
void cmCTestRunTest::ComputeWeightedCost()
{
@@ -400,6 +440,7 @@ void cmCTestRunTest::MemCheckPostProcess()
// Starts the execution of a test. Returns once it has started
bool cmCTestRunTest::StartTest(size_t total)
{
+ this->TotalNumberOfTests = total; // save for rerun case
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(2*getNumWidth(total) + 8)
<< "Start "
<< std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
@@ -494,10 +535,10 @@ bool cmCTestRunTest::StartTest(size_t total)
//----------------------------------------------------------------------
void cmCTestRunTest::ComputeArguments()
{
+ this->Arguments.clear(); // reset becaue this might be a rerun
std::vector<std::string>::const_iterator j =
this->TestProperties->Args.begin();
++j; // skip test name
-
// find the test executable
if(this->TestHandler->MemCheck)
{
@@ -682,10 +723,28 @@ bool cmCTestRunTest::ForkProcess(double testTimeOut, bool explicitTimeout,
void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
{
- cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
- << completed << "/");
- cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
- << total << " ");
+ // if this is the last or only run of this test
+ // then print out completed / total
+ // Only issue is if a test fails and we are running until fail
+ // then it will never print out the completed / total, same would
+ // got for run until pass. Trick is when this is called we don't
+ // yet know if we are passing or failing.
+ if(this->NumberOfRunsLeft == 1)
+ {
+ cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+ << completed << "/");
+ cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+ << total << " ");
+ }
+ // if this is one of several runs of a test just print blank space
+ // to keep things neat
+ else
+ {
+ cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+ << " " << " ");
+ cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
+ << " " << " ");
+ }
if ( this->TestHandler->MemCheck )
{
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 476f3e1..3b5c831 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -27,6 +27,8 @@ public:
cmCTestRunTest(cmCTestTestHandler* handler);
~cmCTestRunTest();
+ void SetNumberOfRuns(int n) {this->NumberOfRunsLeft = n;}
+ void SetRunUntilFailOn() { this->RunUntilFail = true;}
void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties * prop)
{ this->TestProperties = prop; }
@@ -58,7 +60,10 @@ public:
void ComputeArguments();
void ComputeWeightedCost();
+
+ bool StartAgain();
private:
+ bool NeedsToRerun();
void DartProcessing();
void ExeNotFound(std::string exe);
// Figures out a final timeout which is min(STOP_TIME, NOW+TIMEOUT)
@@ -92,6 +97,10 @@ private:
std::string ActualCommand;
std::vector<std::string> Arguments;
bool StopTimePassed;
+ bool RunUntilFail;
+ int NumberOfRunsLeft;
+ bool RunAgain;
+ size_t TotalNumberOfTests;
};
inline int getNumWidth(size_t n)
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index e6d3960..2f89963 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -2005,6 +2005,12 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
this->SetParallelLevel(plevel);
this->ParallelLevelSetInCli = true;
}
+ if(this->CheckArgument(arg, "--repeat-until-fail") && i < args.size() - 1)
+ {
+ i++;
+ this->RepeatTests = atoi(args[i].c_str());
+ this->RepeatUntilFail = true;
+ }
if(this->CheckArgument(arg, "--no-compress-output"))
{
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index 88191c4..7fd84d8 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -429,8 +429,13 @@ public:
{
return this->Definitions;
}
-
+ // return the number of times a test should be run
+ int GetTestRepeat() { return this->RepeatTests;}
+ // return true if test should run until fail
+ bool GetRepeatUntilFail() { return this->RepeatUntilFail;}
private:
+ int RepeatTests;
+ bool RepeatUntilFail;
std::string ConfigType;
std::string ScheduleType;
std::string StopTime;
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index c0eb8ac..351b644 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -49,6 +49,9 @@ static const char * cmDocumentationOptions[][2] =
{"-F", "Enable failover."},
{"-j <jobs>, --parallel <jobs>", "Run the tests in parallel using the "
"given number of jobs."},
+ {"--repeat-until-fail <tries>", "Run each test <tries> times "
+ "unless the test fails at that point stop repeating. Useful for finding"
+ " random failuers."},
{"-Q,--quiet", "Make ctest quiet."},
{"-O <file>, --output-log <file>", "Output to log file"},
{"-N,--show-only", "Disable actual execution of tests."},
-----------------------------------------------------------------------
Summary of changes:
Source/CTest/cmCTestMultiProcessHandler.cxx | 13 ++++-
Source/CTest/cmCTestRunTest.cxx | 73 ++++++++++++++++++++++++---
Source/CTest/cmCTestRunTest.h | 9 ++++
Source/cmCTest.cxx | 8 +++
Source/cmCTest.h | 7 ++-
Source/ctest.cxx | 3 ++
Tests/CMakeLists.txt | 7 +++
Tests/CTestTestRunUntilFail/CMakeLists.txt | 13 +++++
Tests/CTestTestRunUntilFail/init.cmake | 1 +
Tests/CTestTestRunUntilFail/test1.cmake | 8 +++
10 files changed, 133 insertions(+), 9 deletions(-)
create mode 100644 Tests/CTestTestRunUntilFail/CMakeLists.txt
create mode 100644 Tests/CTestTestRunUntilFail/init.cmake
create mode 100644 Tests/CTestTestRunUntilFail/test1.cmake
hooks/post-receive
--
CMake
More information about the Cmake-commits
mailing list