Merge pull request #1317 from rmcgibbo/run-ctest

Rewrite run-ctest.py

.travis.yml

@@ -93,6 +93,7 @@ matrix:
            CMAKE_FLAGS=""
 
 before_install:
+  - START_TIME=$(date +%s)
   - wget https://anaconda.org/omnia/ccache/3.2.4/download/${TRAVIS_OS_NAME}-64/ccache-3.2.4-0.tar.bz2
   - mkdir -p $HOME/ccache && tar xf ccache-3.2.4-0.tar.bz2 -C $HOME/ccache
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
@@ -130,7 +131,6 @@ before_install:
     fi
 
 script:
-  - CTEST_STOP_TIME=$(python -c "from datetime import datetime, timedelta; import sys; sys.stdout.write((datetime.now() + timedelta(minutes=25)).strftime('%H:%M:%S'))")
   - cmake . $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX=$HOME/OpenMM
   - make -j2 install
   - if [[ "$OPENCL" == "true" ]]; then ./TestOpenCLDeviceQuery; fi
@@ -144,8 +144,9 @@ script:
       (cd python/tests && py.test -v);
     fi
 
-  - # run all of the tests, making sure failures at this stage don't cause travis failures
-  - python devtools/run-ctest.py -j2 --schedule-random --stop-time $CTEST_STOP_TIME
+  # Run the tests, and rerun any failing tests.
+  - python devtools/run-ctest.py --start-time $START_TIME
+
   - if [[ ! -z "${DOCS_DEPLOY}" && "${DOCS_DEPLOY}" = "true" ]]; then
       pip install sphinx sphinxcontrib-lunrsearch sphinxcontrib-autodoc_doxygen;
       make C++ApiDocs PythonApiDocs;

devtools/run-ctest.py

+"""
+Run test suite through CTest, with some options set for the CI environment.
+
+- Runs with a per-test and overall timeout which can be governed by the
+  avialable time on the CI system.
+- Reruns tests which fail (does not rerun tests which merely timeout).
+
+"""
 from __future__ import print_function
 import sys
+import os.path
+import shutil
+import time
+from glob import glob
 from os.path import join, exists
 from subprocess import call
+from argparse import ArgumentParser
+from datetime import datetime, timedelta
+from xml.etree import ElementTree
+
 
 def main():
-    # pass any extra arguments to the first ctest invocation
-    if call(['ctest', '--output-on-failure'] + sys.argv[1:], shell=True) == 0:
-        return 0
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--start-time",
+        help="Time at which the overall CI job started (unix timestamp)",
+        type=int,
+        default=int(time.time()))
+    parser.add_argument(
+        "--job-duration",
+        help="Overall time budget for the CI job (minutes). Default=30",
+        default=30.,
+        type=float)
+    parser.add_argument(
+        "--run-percent",
+        help="Allocate this percent test execution time executing the main "
+        "test suite. The remaining fraction will be used for re-running "
+        "failing tests. Default=90",
+        type=float,
+        default=90.)
+    parser.add_argument(
+        '--timeout',
+        help="Timeout for individual tests (seconds). Default=180",
+        type=str,
+        default='180')
+
+    args = parser.parse_args()
+
+    status = execute_tests(args)
+    if status != 0:
+        status = execute_failed_tests(args)
+    return status
+
 
-    # load the log file containing the failed tests and reformat
-    # it in a way so that ctest can take it as input to rerun
-    # the failing tests
-    log = join('Testing', 'Temporary', 'LastTestsFailed.log')
-    assert exists(log)
+def execute_tests(options):
+    start_time = datetime.fromtimestamp(options.start_time)
+    stop_time = start_time + timedelta(minutes=options.job_duration)
 
-    failed = []
-    for line in open(log):
-        failed.append(line.split(':')[0])
+    # timedelta for the amount of time from now until the CI job runs out
+    remaining = stop_time - datetime.now()
 
-    with open('FailedTests.log', 'w') as f:
-        print(','.join(x + ',' + x for x in failed), file=f)
+    # tell CTest only to use some fraction of the remaining time for this
+    # invocation
+    stop_time = start_time + timedelta(
+        seconds=(options.run_percent / 100.0) * remaining.seconds)
 
-    return call(['ctest', '--output-on-failure', '-I', 'FailedTests.log'])
+    if os.path.isdir('Testing'):
+        shutil.rmtree('Testing')
+    return call(['ctest',
+                 '--output-on-failure',
+                 '--schedule-random',
+                 '-T', 'Test',
+                 '--timeout', options.timeout,
+                 '--stop-time', stop_time.strftime('%H:%M:%S')])
+
+
+def execute_failed_tests(options):
+    matches = glob('Testing/*/Test.xml')
+    assert len(matches) == 1
+    root = ElementTree.parse(matches[0])
+    tests = root.findall('.//Testing/Test')
+
+    def failed_without_timeout(test_node):
+        if test_node.get('Status') == 'failed':
+            return test_node.find(
+                'Results/NamedMeasurement[@name="Exit Code"]/Value').text != 'Timeout'
+
+    failed_tests = [t.find('Name').text
+                    for t in tests if failed_without_timeout(t)]
+    print('*'*30)
+    print('Rerunning failing tests...')
+    print('*'*30)
+
+    start_time = datetime.fromtimestamp(options.start_time)
+    stop_time = start_time + timedelta(minutes=options.job_duration)
+    return call(['ctest',
+                 '--output-on-failure',
+                 '--schedule-random',
+                 '-R', '|'.join(failed_tests),
+                 '--timeout', options.timeout,
+                 '--stop-time', stop_time.strftime('%H:%M:%S')])
 
 
 if __name__ == '__main__':
     sys.exit(main())
-