Big refactor of multibuild to ease configuration

Make configuration more general.

Move responsibility for source futzing back to the travis.yml file,
because we are now doing only one build per job.

README.rst

+######################################################
+Utilities for building on travis-ci with OSX and Linux
+######################################################
+
+A set of scripts to automate some of the work of building wheels on OSX and the
+manylinux1 docker system.

common_utils.sh

 #!/bin/bash
 # Utilities for both OSX and Docker
+# Python should be on the PATH
 set -e
 
-# Get our own location on this filesystem
 MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
 
 function abspath {
@@ -18,10 +18,6 @@ function realpath {
     python -c "import os; print(os.path.realpath('$1'))"
 }
 
-function get_root {
-    abspath $MULTIBUILD_DIR/..
-}
-
 function lex_ver {
     # Echoes dot-separated version string padded with zeros
     # Thus:
@@ -36,26 +32,53 @@ function is_function {
     set -e
 }
 
-function clean_fix_source {
-    git checkout $1
-    git clean -fxd
-    git reset --hard
-    git submodule update --init --recursive
-    if [ -n $(is_function "patch_source") ]; then patch_source; fi
+function gh-clone {
+    git clone https://github.com/$1
+}
+
+function rm_mkdir {
+    # Remove directory if present, then make directory
+    local path=$1
+    [ -z "$path" ] && echo "Need not-empty path" && exit 1
+    [ -d "$path" ] && rm -rf $path
+    mkdir $path
+}
+
+function build_wheel {
+    # Builds wheel, puts into $WHEEL_SDIR
+    #
+    # Depends on
+    #     REPO_DIR  (or via input argument)
+    #     WHEEL_SDIR  (optional, default "wheelhouse")
+    #     BUILD_DEPENDS (optional, default "")
+    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
+    local repo_dir=${1:-$REPO_DIR}
+    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
+    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
+    if [ -n $(is_function "pre_build") ]; then pre_build; fi
+    if [ -n "$BUILD_DEPENDS" ]; then pip install $(pip_opts) $BUILD_DEPENDS; fi
+    (cd $repo_dir  && pip wheel $(pip_opts) -w $wheelhouse --no-deps .)
+    repair_wheelhouse $wheelhouse
+}
+
+function pip_opts {
+    [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL"
 }
 
 function install_wheel {
     # Install test dependencies and built wheel
+    #
     # Pass any input flags to pip install steps
+    #
     # Depends on:
-    #     MANYLINUX_URL
-    #     WHEEL_SDIR
-    #     TEST_DEPENDS  (optional)
-    local wheelhouse=$(get_root)/$WHEEL_SDIR
+    #     WHEEL_SDIR  (optional, default "wheelhouse")
+    #     TEST_DEPENDS  (optional, default "")
+    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
+    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
     if [ -n "$TEST_DEPENDS" ]; then
-        pip install --find-links $MANYLINUX_URL $@ $TEST_DEPENDS
+        pip install $(pip_opts) $@ $TEST_DEPENDS
     fi
     # Install compatible wheel
-    pip install --find-links $MANYLINUX_URL $@ \
+    pip install $(pip_opts) $@ \
         $(python $MULTIBUILD_DIR/supported_wheels.py $wheelhouse/*.whl)
 }

docker_build_package.sh

-#!/bin/bash
-# Depends on:
-#   REPO_DIR | PKG_SPEC
-#       (REPO_DIR for in source build; PKG_SPEC for pip build)
-#   PYTHON_VERSION
-#   BUILD_COMMIT
-#   UNICODE_WIDTH  (can be empty)
-#   BUILD_DEPENDS  (can be empty)
-set -e
-
-# Manylinux, openblas version, lex_ver, Python versions
-MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
-source $MULTIBUILD_DIR/manylinux_utils.sh
-source $MULTIBUILD_DIR/common_utils.sh
-
-# Configuration for this package
-source /io/config_funcs.sh
-
-# Unicode widths
-UNICODE_WIDTH=${UNICODE_WIDTH:-32}
-WHEEL_SDIR=${WHEEL_SDIR:-wheelhouse}
-
-# Do any building prior to package building
-if [ -n $(is_function "pre_build") ]; then
-    # Library building tools
-    source $MULTIBUILD_DIR/docker_lib_builders.sh
-    pre_build
-fi
-
-# Directory to store wheels
-rm_mkdir /unfixed_wheels
-
-if [ -n "$REPO_DIR" ]; then
-    # Enter source tree
-    cd /io/$REPO_DIR
-    build_source="."
-elif [ -n "$PKG_SPEC" ]; then
-    build_source=$PKG_SPEC
-else:
-    echo "Must specify REPO_DIR or PKG_SPEC"
-    exit 1
-fi
-
-WHEELHOUSE=/io/$WHEEL_SDIR
-
-# Compile wheel
-PIP="$(cpython_path $PYTHON_VERSION $UNICODE_WIDTH)/bin/pip"
-if [ -n "$BUILD_DEPENDS" ]; then
-    $PIP install -f $MANYLINUX_URL $BUILD_DEPENDS
-fi
-clean_fix_source $BUILD_COMMIT
-if [ -n "$REPO_DIR" ]; then clean_fix_source $BUILD_COMMIT; fi
-$PIP wheel -f $MANYLINUX_URL -w /unfixed_wheels --no-deps $build_source
-
-# Bundle external shared libraries into the wheels
-repair_wheelhouse /unfixed_wheels $WHEELHOUSE

docker_build_wrap.sh

+#!/bin/bash
+# Depends on:
+#   REPO_DIR
+#   PYTHON_VERSION
+#   BUILD_COMMIT
+#   UNICODE_WIDTH  (can be empty)
+#   BUILD_DEPENDS  (can be empty)
+set -e
+
+# Unicode width, default 32
+UNICODE_WIDTH=${UNICODE_WIDTH:-32}
+
+# Location of wheels, default "wheelhouse"
+WHEEL_SDIR=${WHEEL_SDIR:-wheelhouse}
+
+# Manylinux, openblas version, lex_ver, Python versions
+MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
+# This next also sources common utils.
+source $MULTIBUILD_DIR/manylinux_utils.sh
+source $MULTIBUILD_DIR/docker_lib_builders.sh
+
+# Set PATH for chosen Python, Unicode width
+export PATH="$(cpython_path $PYTHON_VERSION $UNICODE_WIDTH)/bin:$PATH"
+
+# Change into root directory of repo
+cd /io
+
+# Configuration for this package, possibly overriding `build_wheel` defined in
+# `common_utils.sh` via `manylinux_utils.sh`.
+source config.sh
+
+build_wheel $REPO_DIR

docker_lib_builders.sh

@@ -13,9 +13,6 @@ XZ_VERSION="${XZ_VERSION:-5.2.2}"
 LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}"
 OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.2.18}"
 
-# Get needed utilities
-MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
-source ${MULTIBUILD_DIR}/manylinux_utils.sh
 
 function build_simple {
     local name=$1

docker_install_run.sh → docker_test_wrap.sh

@@ -6,10 +6,13 @@ set -e
 MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
 source $MULTIBUILD_DIR/common_utils.sh
 
+# Change into root directory of repo
+cd /io
+
 # Configuration for this package
-# This can ovverride `install_wheel`, otherwise defined in common_utils.sh.
+# This can override `install_wheel`, otherwise defined in common_utils.sh.
 # It must define `run_tests`.
-source $(get_root)/config_funcs.sh
+source config.sh
 
 install_wheel
 run_tests

manylinux_utils.sh

-# Useful defines common across manylinux1 builds
+# Useful utilities common across manylinux1 builds
+
+MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
+source $MULTIBUILD_DIR/common_utils.sh
 
 # UNICODE_WIDTH selects "32"=wide (UCS4) or "16"=narrow (UCS2/UTF16) builds
 UNICODE_WIDTH="${UNICODE_WIDTH:-32}"
 
-function gh-clone {
-    git clone https://github.com/$1
-}
-
-function rm_mkdir {
-    # Remove directory if present, then make directory
-    local path=$1
-    if [ -d "$path" ]; then
-        rm -rf $path
-    fi
-    mkdir $path
-}
-
-function lex_ver {
-    # Echoes dot-separated version string padded with zeros
-    # Thus:
-    # 3.2.1 -> 003002001
-    # 3     -> 003000000
-    echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}'
-}
-
-function strip_dots {
-    # Strip "." characters from string
-    echo $1 | sed "s/\.//g"
-}
-
 function cpython_path {
     # Return path to cpython given
     # * version (of form "2.7")
@@ -48,30 +25,16 @@ function cpython_path {
         echo "Incorrect u_width value $u_width"
         exit 1
     fi
-    local no_dots=$(strip_dots $py_ver)
+    local no_dots=$(echo $py_ver | tr -d .)
     echo "/opt/python/cp${no_dots}-cp${no_dots}m${u_suff}"
 }
 
-
-function gh-clone {
-    git clone https://github.com/$1
-}
-
-function rm_mkdir {
-    # Remove directory if present, then make directory
-    local path=$1
-    if [ -d "$path" ]; then
-        rm -rf $path
-    fi
-    mkdir $path
-}
-
 function repair_wheelhouse {
     local in_dir=$1
-    local out_dir=$2
+    local out_dir=${2:-$in_dir}
     for whl in $in_dir/*.whl; do
         if [[ $whl == *none-any.whl ]]; then
-            cp $whl $out_dir
+            [ "$in_dir" == "$out_dir" ] || cp $whl $out_dir
         else
             auditwheel repair $whl -w $out_dir/
         fi

osx_utils.sh

@@ -219,3 +219,11 @@ function get_macpython_environment {
     fi
     set_py_vars
 }
+
+function repair_wheelhouse {
+    local wheelhouse=$1
+    pip install delocate
+    delocate-listdeps $wheelhouse/*.whl # lists library dependencies
+    delocate-wheel $wheelhouse/*.whl # copies library dependencies into wheel
+    delocate-addplat --rm-orig -x 10_9 -x 10_10 $wheelhouse/*.whl
+}

travis_linux_steps.sh

@@ -3,7 +3,7 @@
 #
 # In fact the main work is to wrap up the real functions in docker commands.
 # The real work is in the BUILD_SCRIPT (which builds the wheel) and
-# `docker_install_run.sh`, which can be configured with `config_funcs.sh`.
+# `docker_install_run.sh`, which can be configured with `config.sh`.
 #
 # Must define
 #  before_install
@@ -11,14 +11,11 @@
 #  install_run
 set -e
 
+MANYLINUX_URL=${MANYLINUX_URL:-https://nipy.bic.berkeley.edu/manylinux}
+
 # Get our own location on this filesystem
 MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
 
-# Docker build script
-BUILD_SCRIPT=${BUILD_SCRIPT:-${MULTIBUILD_DIR}/docker_build_package.sh}
-
-UNICODE_WIDTH=${UNICODE_WIDTH:-32}
-
 function before_install {
     # Install a virtualenv to work in.
     virtualenv --python=python venv
@@ -30,15 +27,19 @@ function before_install {
 function build_wheel {
     # Builds wheel, puts into $WHEEL_SDIR
     #
+    # In fact wraps the actual work which happens in the container.
+    #
     # Depends on
-    #  PLAT
-    #  BUILD_DEPENDS
-    #  BUILD_COMMIT
-    #  BUILD_PRE_SCRIPT
-    #  BUILD_SCRIPT
-    #  REPO_DIR | PKG_SPEC
+    #     REPO_DIR  (or via input argument)
+    #     PLAT (can be passed in as argument)
     #     TRAVIS_PYTHON_VERSION
-    local plat=${1:-$PLAT}
+    #     UNICODE_WIDTH (optional)
+    #     BUILD_DEPENDS (optional)
+    #     MANYLINUX_URL (optional)
+    #     WHEEL_SDIR (optional)
+    local repo_dir=${1:-$REPO_DIR}
+    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
+    local plat=${2:-$PLAT}
     local docker_image=quay.io/pypa/manylinux1_$plat
     docker pull $docker_image
     docker run --rm \
@@ -47,18 +48,26 @@ function build_wheel {
         -e WHEEL_SDIR="$WHEEL_SDIR" \
         -e MANYLINUX_URL="$MANYLINUX_URL" \
         -e BUILD_DEPENDS="$BUILD_DEPENDS" \
-        -e BUILD_COMMIT="$BUILD_COMMIT" \
-        -e PKG_SPEC="$PKG_SPEC" \
-        -e REPO_DIR="$REPO_DIR" \
+        -e REPO_DIR="$repo_dir" \
         -v $PWD:/io \
-        $docker_image /io/$BUILD_SCRIPT
+        $docker_image /io/$MULTIBUILD_DIR/docker_build_wrap.sh
 }
 
 function install_run {
+    # Install wheel, run tests
+    #
+    # In fact wraps the actual work which happens in the container.
+    #
+    # Depends on
+    #  PLAT (can be passed in as argument)
+    #  TRAVIS_PYTHON_VERSION
+    #  UNICODE_WIDTH (optional)
+    #  WHEEL_SDIR (optional)
+    #  MANYLINUX_URL (optional)
+    #  TEST_DEPENDS  (optional)
     local plat=${1:-$PLAT}
     bitness=$([ "$plat" == i686 ] && echo 32 || echo 64)
     local docker_image="matthewbrett/trusty:$bitness"
-    local multibuild_sdir=$(relpath $MULTIBUILD_DIR)
     docker pull $docker_image
     docker run --rm \
         -e PYTHON_VERSION="$TRAVIS_PYTHON_VERSION" \
@@ -67,5 +76,5 @@ function install_run {
         -e MANYLINUX_URL="$MANYLINUX_URL" \
         -e TEST_DEPENDS="$TEST_DEPENDS" \
         -v $PWD:/io \
-        $docker_image /io/$multibuild_sdir/docker_install_run.sh
+        $docker_image /io/$MULTIBUILD_DIR/docker_test_wrap.sh
 }

travis_osx_steps.sh

@@ -6,7 +6,8 @@ set -e
 MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
 source $MULTIBUILD_DIR/osx_utils.sh
 
-# NB - config_funcs.sh sourced at end of this function
+# NB - config.sh sourced at end of this function.
+# config.sh can override any function defined here.
 
 function before_install {
     export CC=clang
@@ -16,42 +17,15 @@ function before_install {
     pip install --upgrade pip wheel
 }
 
-function delocate_wheel {
-    local wheelhouse=$PWD/$WHEEL_SDIR
-    pip install delocate
-    delocate-listdeps $wheelhouse/*.whl # lists library dependencies
-    delocate-wheel $wheelhouse/*.whl # copies library dependencies into wheel
-    delocate-addplat --rm-orig -x 10_9 -x 10_10 $wheelhouse/*.whl
-}
-
-function build_wheel {
-    # Builds wheel, puts into $WHEEL_SDIR
-    #
-    # Depends on
-    #  WHEEL_SDIR
-    #  BUILD_DEPENDS
-    #  REPO_DIR | PKG_SPEC
-    #  BUILD_COMMIT
-    local wheelhouse=$PWD/$WHEEL_SDIR
-    if [ -n $(is_function "pre_build") ]; then pre_build; fi
-    if [ -n "$BUILD_DEPENDS" ]; then pip install $BUILD_DEPENDS; fi
-    if [ -n "$REPO_DIR" ]; then
-        (cd $REPO_DIR \
-            && clean_fix_source $BUILD_COMMIT \
-            && pip wheel -w $wheelhouse --no-deps .)
-    else
-        pip wheel -w $wheelhouse --no-deps $PKG_SPEC
-    fi
-    delocate_wheel
-}
+# build_wheel function defined in common_utils (via osx_utils)
 
 function install_run {
-    # Depend on function `run_tests` defined in `config_funcs.sh`
+    # Depends on function `run_tests` defined in `config.sh`
     install_wheel
     mkdir tmp_for_test
     (cd tmp_for_test && run_tests)
 }
 
 # Local configuration may define custom pre-build, source patching.
-# It can also overwrite the functions above
-source $PWD/config_funcs.sh
+# It can also overwrite the functions above.
+source config.sh

travis_steps.sh

 #!/bin/bash
 
 WHEEL_SDIR=${WHEEL_SDIR:-wheelhouse}
-MANYLINUX_URL=${MANYLINUX_URL:-https://nipy.bic.berkeley.edu/manylinux}
 
 MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
-# Get utilities common to OSX and Linux
-source $MULTIBUILD_DIR/common_utils.sh
-
-# Specify REPO_DIR to build from directory in this repository.
-# Specify PKG_SPEC to build from pip requirement (e.g numpy==1.7.1)
-# PKG_SPEC is hardly tested, please let us know of bugs.
-if [ -z "$REPO_DIR$PKG_SPEC" ]; then
-    echo "Must specify REPO_DIR or PKG_SPEC"
-    exit 1
-fi
 
 if [ ! -d "$PWD/$WHEEL_SDIR" ]; then mkdir $PWD/WHEEL_SDIR; fi
 if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then