Use Jenkinsfile to drive the pipeline job (#2198)

* Add a Jenkinsfile to drive a pipeline

* Also tweaks the installation script for Jenkins to be more amenable to
running other variants in parallel
* Restrict packaging script to only accept Python 2.7, since I think
that's long been the minimum supported Python version
* Allow run-ctest.py to run tests in parallel (and take advantage of
that in the Jenkins tests).

* CUDA -> CUDA_VERSION

* Jenkins will automatically post result back to GitHub.

Not need to have a separate step do that explicitly.

* Run as a login shell.

* Force looking for CUDA node.

* Make sure we load conda module.

* Start building in a docker slave -- much nicer!

* Try to fix up the installation a little.

* Fix more build issues.

* More build debugging.

* Oops -- recursive :-o

* Put opencl lib in library path

* Try 9.2

* Fix library path

* More updates to Jenkins build recipes.

* Try a new (simpler?) approach with building

* Oops...

devtools/Jenkinsfile

+pipeline {
+    agent any
+
+    stages {
+        stage("Build and test") {
+            parallel {
+                stage("Build and test CUDA platform") {
+                    agent { label "cuda" }
+                    steps {
+                        sh '''#!/bin/bash -lex
+                            git clean -fxd && git checkout .
+                            env
+                            module load cuda conda
+                            export OPENMM_CUDA_COMPILER=`which nvcc`
+                            bash -e devtools/ci/jenkins/install.sh
+                            bash -e devtools/ci/jenkins/test.sh -R 'TestCuda' --parallel 2
+                        '''
+                    }
+                }
+
+                stage("Build and test OpenCL platform") {
+                    agent { label "cuda" }
+                    steps {
+                        sh '''#!/bin/bash -lex
+                            git clean -fxd && git checkout .
+                            env
+                            module load cuda conda
+                            export OPENMM_CUDA_COMPILER=`which nvcc`
+                            bash -e devtools/ci/jenkins/install.sh
+                            bash -e devtools/ci/jenkins/test.sh -R 'TestOpenCL' --parallel 2
+                        '''
+                    }
+                }
+
+                stage("Build/test CPU platforms") {
+                    agent {
+                        dockerfile {
+                            dir "devtools/ci/jenkins"
+                            label "docker"
+                        }
+                    }
+
+                    steps {
+                        sh '''#!/bin/bash -lex
+                        git clean -fxd && git checkout .
+                        bash -e devtools/ci/jenkins/install_and_test_cpu.sh
+                        '''
+                    }
+                }
+            } // parallel
+        }
+    }
+}

devtools/ci/jenkins/Dockerfile

+FROM ubuntu:xenial
+
+ENV PATH="/opt/miniconda/bin:${PATH}"
+
+RUN apt-get update && \
+    apt-get install -y wget git gromacs doxygen gfortran libfftw3-dev gcc g++ bzip2 automake make lsb-core && \
+    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O miniconda.sh && \
+    bash miniconda.sh -b -p "/opt/miniconda" && \
+    rm -f miniconda.sh && \
+    mkdir /.conda &&  chmod -R 777 /.conda &&  chmod -R 777 /opt/miniconda && \
+    conda install -y cmake numpy scipy pytest swig cython

devtools/ci/jenkins/install.sh

@@ -11,34 +11,12 @@ cmake --version
 echo "Using g++ (`which g++`) version:"
 g++ --version
 
-module load cuda/9.0
-
-# Constants
-CONDAENV=openmm-test-3.5
-INSTALL_DIRECTORY="`pwd`/install"
-export OPENMM_CUDA_COMPILER=`which nvcc`
-
-# Create a conda environment, but clean up after one first. If it doesn't exist, don't complain.
-# But since we are invoking this shell with -e (exit on all errors), we need || true to prevent this
-# command from crashing the whole shell
-create_conda_env() {
-  conda create -yn ${CONDAENV} python=3.5 --no-default-packages --quiet
-}
-conda remove -yn ${CONDAENV} --all --quiet || true
-create_conda_env || create_conda_env # Crappy way to work around conda concurrency restrictions
-conda install -yn ${CONDAENV} numpy scipy pytest cython --quiet
-source activate ${CONDAENV} # enter our new environment
-
-# Build OpenMM
-cmake -DCMAKE_INSTALL_PREFIX="${INSTALL_DIRECTORY}" -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc .
+if [ ! -z "$OPENMM_CUDA_COMPILER" ]; then
+    echo "Using nvcc ($OPENMM_CUDA_COMPILER) version:"
+    $OPENMM_CUDA_COMPILER --version
+    CUDA_ARGS="-DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME}"
+fi
+
+cmake -DCMAKE_INSTALL_PREFIX="`pwd`/install" -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc \
+      -DSWIG_EXECUTABLE=`which swig` $CUDA_ARGS $EXTRA_CMAKE_ARGS .
 make -j6 install
-make PythonInstall
-
-# Now run the tests
-python -m simtk.testInstallation
-cd python/tests && py.test -v && cd ../..
-python devtools/run-ctest.py --job-duration=120 --timeout 300
-
-# Now remove the conda environment
-source deactivate
-conda remove -yn ${CONDAENV} --all --quiet

devtools/ci/jenkins/install_and_test_cpu.sh

+#!/bin/bash
+
+EXTRA_CMAKE_ARGS="-DOPENMM_BUILD_CUDA_LIB=false -DOPENMM_BUILD_OPENCL_LIB=false"
+. devtools/ci/jenkins/install.sh
+python devtools/run-ctest.py --job-duration=120 --timeout 300 --in-order -R 'Test(Cpu|Reference)' --parallel 4
+
+# Build & test Python
+make PythonInstall
+python -m simtk.testInstallation
+cd python/tests && py.test -v

devtools/ci/jenkins/test.sh

+#!/bin/sh
+
+python devtools/run-ctest.py --job-duration=120 --timeout 300 --in-order $*
\ No newline at end of file

devtools/packaging/install.sh

@@ -26,10 +26,10 @@ fi
 
 # Make sure it's a supported Python version.
 
-pythonOk=$(${pythonBin} -c "import sys; v=sys.version_info; print((v[0]==2 and v[1]>5) or v[0]>2)")
+pythonOk=$(${pythonBin} -c "import sys; v=sys.version_info; print((v[0]==2 and v[1]>6) or v[0]>2)")
 if [ ${pythonOk} != "True" ]
 then
-  echo "Unsupported Python version.  Only versions 2.6 and higher are supported."
+  echo "Unsupported Python version.  Only versions 2.7 and higher are supported."
   exit
 fi
 

devtools/run-ctest.py

@@ -43,16 +43,26 @@ def main():
         help="Timeout for individual tests (seconds). Default=180",
         type=str,
         default='180')
+    parser.add_argument(
+        '--in-order',
+        help='Run the tests in order',
+        default=False,
+        action='store_true')
+    parser.add_argument(
+        '--parallel',
+        help='Number of processors to use',
+        type=int,
+        default=1)
 
-    args = parser.parse_args()
+    args, raw_args = parser.parse_known_args()
 
-    status = execute_tests(args)
+    status = execute_tests(args, raw_args)
     if status != 0:
-        status = execute_failed_tests(args)
+        status = execute_failed_tests(args, raw_args)
     return status
 
 
-def execute_tests(options):
+def execute_tests(options, raw_options):
     start_time = datetime.fromtimestamp(options.start_time)
     stop_time = start_time + timedelta(minutes=options.job_duration)
 
@@ -68,13 +78,14 @@ def execute_tests(options):
         shutil.rmtree('Testing')
     return call(['ctest',
                  '--output-on-failure',
-                 '--schedule-random',
+                 '--parallel', str(options.parallel),
                  '-T', 'Test',
                  '--timeout', options.timeout,
-                 '--stop-time', stop_time.strftime('%H:%M:%S')])
+                 '--stop-time', stop_time.strftime('%H:%M:%S')] + raw_options +
+                 (['--schedule-random'] if options.in_order else []))
 
 
-def execute_failed_tests(options):
+def execute_failed_tests(options, raw_options):
     matches = glob('Testing/*/Test.xml')
     assert len(matches) == 1
     root = ElementTree.parse(matches[0])
@@ -93,12 +104,13 @@ def execute_failed_tests(options):
 
     start_time = datetime.fromtimestamp(options.start_time)
     stop_time = start_time + timedelta(minutes=options.job_duration)
-    return call(['ctest',
+    return call(['ctest'] + raw_options + [
                  '--output-on-failure',
-                 '--schedule-random',
+                 '--parallel', str(options.parallel),
                  '-R', '|'.join(failed_tests),
                  '--timeout', options.timeout,
-                 '--stop-time', stop_time.strftime('%H:%M:%S')])
+                 '--stop-time', stop_time.strftime('%H:%M:%S')] +
+                 (['--schedule-random'] if options.in_order else []))
 
 
 if __name__ == '__main__':

wrappers/python/simtk/openmm/app/internal/pdbx/reader/PdbxParser.py

@@ -82,6 +82,9 @@ class PdbxReader(object):
             self.__parser(self.__tokenizer(self.__ifh), containerList)
         except StopIteration:
             pass
+        except RuntimeError as err:
+            if 'StopIteration' not in str(err):
+                raise
         else:
             raise PdbxError()
 

wrappers/python/simtk/openmm/app/internal/pdbx/reader/PdbxReader.py

@@ -71,6 +71,9 @@ class PdbxReader(object):
             self.__parser(self.__tokenizer(self.__ifh), containerList)
         except StopIteration:
             pass
+        except RuntimeError as err:
+            if 'StopIteration' not in str(err):
+                raise
         else:
             raise PdbxError()