Refactor away manylinux-build submodule

.gitmodules

 [submodule "terryfy"]
 	path = terryfy
 	url = https://github.com/MacPython/terryfy.git
-[submodule "manylinux"]
-	path = manylinux
-	url = https://github.com/matthew-brett/manylinux-builds

common_utils.sh

@@ -2,13 +2,43 @@
 # Utilities for both OSX and Docker
 set -e
 
+# Get our own location on this filesystem
+MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
+
 function abspath {
     python -c "import os.path; print(os.path.abspath('$1'))"
 }
 
-# Get our own location on this filesystem
-MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
-ROOT_DIR=$(abspath $MULTIBUILD_DIR/..)
+function relpath {
+    # Path of first input relative to second (or $PWD if not specified)
+    python -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))"
+}
+
+function get_root {
+    abspath $MULTIBUILD_DIR/..
+}
+
+function lex_ver {
+    # Echoes dot-separated version string padded with zeros
+    # Thus:
+    # 3.2.1 -> 003002001
+    # 3     -> 003000000
+    echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}'
+}
+
+function is_function {
+    set +e
+    $(declare -Ff "$1") > /dev/null && echo true
+    set -e
+}
+
+function clean_fix_source {
+    git checkout $1
+    git clean -fxd
+    git reset --hard
+    git submodule update --init --recursive
+    if [ -n $(is_function "patch_source") ]; then patch_source; fi
+}
 
 function install_wheel {
     # Install test dependencies and built wheel
@@ -17,19 +47,20 @@ function install_wheel {
     #     MANYLINUX_URL
     #     WHEEL_SDIR
     #     TEST_DEPENDS  (optional)
+    local wheelhouse=$(get_root)/$WHEEL_SDIR
     if [ -n "$TEST_DEPENDS" ]; then
         pip install --find-links $MANYLINUX_URL $@ $TEST_DEPENDS
     fi
     # Install compatible wheel
     pip install --find-links $MANYLINUX_URL $@ \
-        $(python $MULTIBUILD_DIR/supported_wheels.py $ROOT_DIR/$WHEEL_SDIR/*.whl)
+        $(python $MULTIBUILD_DIR/supported_wheels.py $wheelhouse/*.whl)
 }
 
 function install_run {
-    local run_tests_script=${1:-$RUN_TESTS_SCRIPT}
+    # Depend on function `run_tests` defined in `config_funcs.sh`
     install_wheel
+    # Configuration for this package
+    source $(get_root)/config_funcs.sh
     mkdir tmp_for_test
-    cd tmp_for_test
-    source $ROOT_DIR/$run_tests_script
-    cd ..
+    (cd tmp_for_test && run_tests)
 }

docker_build_package.sh

+#!/bin/bash
+# Depends on:
+#   REPO_DIR | PKG_SPEC
+#       (REPO_DIR for in source build; PKG_SPEC for pip build)
+#   PYTHON_VERSION
+#   BUILD_COMMIT
+#   UNICODE_WIDTH  (can be empty)
+#   BUILD_DEPENDS  (can be empty)
+set -e
+
+# Manylinux, openblas version, lex_ver, Python versions
+MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
+source $MULTIBUILD_DIR/manylinux_utils.sh
+source $MULTIBUILD_DIR/common_utils.sh
+
+# Configuration for this package
+source /io/config_funcs.sh
+
+# Unicode widths
+UNICODE_WIDTH=${UNICODE_WIDTH:-32}
+WHEEL_SDIR=${WHEEL_SDIR:-wheelhouse}
+
+# Do any building prior to package building
+if [ -n $(is_function "pre_build") ]; then
+    # Library building tools
+    source $MULTIBUILD_DIR/docker_lib_builders.sh
+    pre_build
+fi
+
+# Directory to store wheels
+rm_mkdir /unfixed_wheels
+
+if [ -n "$REPO_DIR" ]; then
+    # Enter source tree
+    cd /io/$REPO_DIR
+    build_source="."
+elif [ -n "$PKG_SPEC" ]; then
+    build_source=$PKG_SPEC
+else:
+    echo "Must specify REPO_DIR or PKG_SPEC"
+    exit 1
+fi
+
+WHEELHOUSE=/io/$WHEEL_SDIR
+
+# Compile wheel
+PIP="$(cpython_path $PYTHON_VERSION $UNICODE_WIDTH)/bin/pip"
+if [ -n "$BUILD_DEPENDS" ]; then
+    $PIP install -f $MANYLINUX_URL $BUILD_DEPENDS
+fi
+clean_fix_source $BUILD_COMMIT
+if [ -n "$REPO_DIR" ]; then clean_fix_source $BUILD_COMMIT; fi
+$PIP wheel -f $MANYLINUX_URL -w /unfixed_wheels --no-deps $build_source
+
+# Bundle external shared libraries into the wheels
+repair_wheelhouse /unfixed_wheels $WHEELHOUSE

docker_install_run.sh

@@ -6,4 +6,4 @@ set -e
 MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
 source $MULTIBUILD_DIR/common_utils.sh
 
-install_run $RUN_TESTS_SCRIPT
+install_run

docker_lib_builders.sh

+# Recipes for building some libaries
+# We use system zlib by default - see build_new_zlib
+ZLIB_VERSION="${ZLIB_VERSION:-1.2.8}"
+LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.21}"
+BZIP2_VERSION="${BZIP2_VERSION:-1.0.6}"
+FREETYPE_VERSION="${FREETYPE_VERSION:-2.6.3}"
+TIFF_VERSION="${FREETYPE_VERSION:-4.0.6}"
+OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}"
+LCMS2_VERSION="${LCMS2_VERSION:-2.7}"
+GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}"
+LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}"
+XZ_VERSION="${XZ_VERSION:-5.2.2}"
+LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}"
+OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.2.18}"
+
+# Get needed utilities
+MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
+source ${MULTIBUILD_DIR}/manylinux_utils.sh
+
+function build_simple {
+    local name=$1
+    local version=$2
+    local url=$3
+    if [ -e "${name}-stamp" ]; then
+        return
+    fi
+    local name_version="${name}-${version}"
+    local targz=${name_version}.tar.gz
+    curl -LO $url/$targz
+    tar zxf $targz
+    (cd $name_version && ./configure && make && make install)
+    touch "${name}-stamp"
+}
+
+function build_openblas {
+    if [ -e openblas-stamp ]; then return; fi
+    git clone https://github.com/xianyi/OpenBLAS
+    (cd OpenBLAS \
+        && git checkout "v${OPENBLAS_VERSION}" \
+        && make DYNAMIC_ARCH=1 USE_OPENMP=0 NUM_THREADS=64 > /dev/null \
+        && make PREFIX=/usr/local/ install)
+    touch openblas-stamp
+}
+
+function build_zlib {
+    # Gives an old but safe version
+    if [ -e zlib-stamp ]; then return; fi
+    yum install -y zlib-devel
+    touch zlib-stamp
+}
+
+function build_new_zlib {
+    # Careful, this one may cause yum to segfault
+    build_simple zlib $ZLIB_VERSION http://zlib.net
+}
+
+function build_jpeg {
+    if [ -e jpeg-stamp ]; then return; fi
+    curl -LO http://ijg.org/files/jpegsrc.v9b.tar.gz
+    tar zxf jpegsrc.v9b.tar.gz
+    (cd jpeg-9b && ./configure && make && make install)
+    touch jpeg-stamp
+}
+
+function build_libpng {
+    build_zlib
+    build_simple libpng $LIBPNG_VERSION http://download.sourceforge.net/libpng
+}
+
+function build_bzip2 {
+    if [ -e bzip2-stamp ]; then return; fi
+    curl -LO http://bzip.org/${BZIP2_VERSION}/bzip2-${BZIP2_VERSION}.tar.gz
+    tar zxf bzip2-${BZIP2_VERSION}.tar.gz
+    (cd bzip2-${BZIP2_VERSION} && make -f Makefile-libbz2_so && make install)
+    touch bzip2-stamp
+}
+
+function build_tiff {
+    build_zlib
+    build_jpeg
+    build_openjpeg
+    build_xz
+    build_simple tiff $TIFF_VERSION ftp://ftp.remotesensing.org/pub/libtiff
+}
+
+function build_openjpeg {
+    if [ -e openjpeg-stamp ]; then return; fi
+    yum install -y cmake28
+    curl -LO https://github.com/uclouvain/openjpeg/archive/version.${OPENJPEG_VERSION}.tar.gz
+    tar zxf version.${OPENJPEG_VERSION}.tar.gz
+    (cd openjpeg-version.${OPENJPEG_VERSION} && cmake28 . && make install)
+    touch openjpeg-stamp
+}
+
+function build_lcms2 {
+    build_tiff
+    build_simple lcms2 $LCMS2_VERSION http://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION
+}
+
+function build_giflib {
+    build_simple giflib $GIFLIB_VERSION http://downloads.sourceforge.net/project/giflib
+}
+
+function build_xz {
+    build_simple xz $XZ_VERSION http://tukaani.org/xz
+}
+
+function build_libwebp {
+    if [ -e libwebp-stamp ]; then return; fi
+    build_libpng
+    build_tiff
+    build_giflib
+    curl -LO https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${LIBWEBP_VERSION}.tar.gz
+    tar zxf libwebp-${LIBWEBP_VERSION}.tar.gz
+    (cd libwebp-${LIBWEBP_VERSION} && \
+        ./configure --enable-libwebpmux --enable-libwebpdemux && \
+         make && make install)
+    touch libwebp-stamp
+}
+
+function build_freetype {
+    build_libpng
+    build_bzip2
+    build_simple freetype $FREETYPE_VERSION http://download.savannah.gnu.org/releases/freetype
+}
+
+function build_libyaml {
+    build_simple yaml $LIBYAML_VERSION http://pyyaml.org/download/libyaml
+}

manylinux @ e02c166b

-Subproject commit e02c166ba4ac82968cf629c8eaef55b74dd4a0c0

manylinux_utils.sh

+# Useful defines common across manylinux1 builds
+
+# Width of Python unicode digits in bits
+UNICODE_WIDTH=${UNICODE_WIDTH:-32}
+
+function gh-clone {
+    git clone https://github.com/$1
+}
+
+function rm_mkdir {
+    # Remove directory if present, then make directory
+    local path=$1
+    if [ -d "$path" ]; then
+        rm -rf $path
+    fi
+    mkdir $path
+}
+
+function lex_ver {
+    # Echoes dot-separated version string padded with zeros
+    # Thus:
+    # 3.2.1 -> 003002001
+    # 3     -> 003000000
+    echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}'
+}
+
+function strip_dots {
+    # Strip "." characters from string
+    echo $1 | sed "s/\.//g"
+}
+
+function cpython_path {
+    # Return path to cpython given
+    # * version (of form "2.7")
+    # * u_width ("16" or "32" default "32")
+    #
+    # For back-compatibility "u" as u_width also means "32"
+    local py_ver="${1:-2.7}"
+    local u_width="${2:-${UNICODE_WIDTH}}"
+    local u_suff=u
+    # Back-compatibility
+    if [ "$u_width" == "u" ]; then u_width=32; fi
+    # For Python >= 3.3, "u" suffix not meaningful
+    if [ $(lex_ver $py_ver) -ge $(lex_ver 3.3) ] ||
+        [ "$u_width" == "16" ]; then
+        u_suff=""
+    elif [ "$u_width" != "32" ]; then
+        echo "Incorrect u_width value $u_width"
+        exit 1
+    fi
+    local no_dots=$(strip_dots $py_ver)
+    echo "/opt/python/cp${no_dots}-cp${no_dots}m${u_suff}"
+}
+
+
+function gh-clone {
+    git clone https://github.com/$1
+}
+
+function rm_mkdir {
+    # Remove directory if present, then make directory
+    local path=$1
+    if [ -d "$path" ]; then
+        rm -rf $path
+    fi
+    mkdir $path
+}
+
+function repair_wheelhouse {
+    local in_dir=$1
+    local out_dir=$2
+    for whl in $in_dir/*.whl; do
+        if [[ $whl == *none-any.whl ]]; then
+            cp $whl $out_dir
+        else
+            auditwheel repair $whl -w $out_dir/
+        fi
+    done
+    chmod -R a+rwX $out_dir
+}

travis_linux_steps.sh

@@ -2,10 +2,12 @@
 # Wheel build, install, run test steps on Linux
 set -e
 
-# Get needed utilities
+# Get our own location on this filesystem
 MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
-UTIL_DIR=${UTIL_DIR:-${MULTIBUILD_DIR}/manylinux}
-BUILD_SCRIPT=${BUILD_SCRIPT:-/io/$UTIL_DIR/build_package.sh}
+
+# Docker build script
+BUILD_SCRIPT=${BUILD_SCRIPT:-${MULTIBUILD_DIR}/docker_build_package.sh}
+
 UNICODE_WIDTH=${UNICODE_WIDTH:-32}
 
 function before_install {
@@ -26,22 +28,20 @@ function build_wheel {
     #  BUILD_SCRIPT
     #  REPO_DIR | PKG_SPEC
     #  TRAVIS_PYTHON_VERSION
-    #
     local plat=${1:-$PLAT}
     local docker_image=quay.io/pypa/manylinux1_$plat
     docker pull $docker_image
-    if [ "$plat" == "i686" ]; then local intro_cmd=linux32; fi
     docker run --rm \
         -e PYTHON_VERSION="$TRAVIS_PYTHON_VERSION" \
         -e UNICODE_WIDTH="$UNICODE_WIDTH" \
         -e WHEEL_SDIR="$WHEEL_SDIR" \
+        -e MANYLINUX_URL="$MANYLINUX_URL" \
         -e BUILD_DEPENDS="$BUILD_DEPENDS" \
         -e BUILD_COMMIT="$BUILD_COMMIT" \
-        -e BUILD_PRE_SCRIPT="$BUILD_PRE_SCRIPT" \
         -e PKG_SPEC="$PKG_SPEC" \
         -e REPO_DIR="$REPO_DIR" \
         -v $PWD:/io \
-        $docker_image $intro_cmd $BUILD_SCRIPT
+        $docker_image /io/$BUILD_SCRIPT
 }
 
 function relpath {
@@ -49,13 +49,8 @@ function relpath {
 }
 
 function install_run {
-    local run_tests_script=${1:-$RUN_TESTS_SCRIPT}
     local plat=${2:-$PLAT}
-    if [ "$plat" == "i686" ]; then
-        local bitness=32
-    else
-        local bitness=64
-    fi
+    bitness=$([ "$plat" == i686 ] && echo 32 || echo 64)
     local docker_image="matthewbrett/trusty:$bitness"
     local multibuild_sdir=$(relpath $MULTIBUILD_DIR)
     docker pull $docker_image
@@ -63,7 +58,6 @@ function install_run {
         -e PYTHON_VERSION="$TRAVIS_PYTHON_VERSION" \
         -e UNICODE_WIDTH="$UNICODE_WIDTH" \
         -e WHEEL_SDIR="$WHEEL_SDIR" \
-        -e RUN_TESTS_SCRIPT="$run_tests_script" \
         -e MANYLINUX_URL="$MANYLINUX_URL" \
         -e TEST_DEPENDS="$TEST_DEPENDS" \
         -v $PWD:/io \

travis_osx_steps.sh

@@ -5,7 +5,9 @@ set -e
 # Get needed utilities
 MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
 source $MULTIBUILD_DIR/terryfy/travis_tools.sh
-source $MULTIBUILD_DIR/common_utils.sh
+
+# Local configuration may define custom pre-build, source patching
+source $PWD/config_funcs.sh
 
 function before_install {
     export CC=clang
@@ -24,15 +26,12 @@ function build_wheel {
     #  REPO_DIR | PKG_SPEC
     #  BUILD_COMMIT
     local wheelhouse=$PWD/$WHEEL_SDIR
-    if [ -n "$BUILD_PRE_SCRIPT" ]; then source $BUILD_PRE_SCRIPT; fi
+    if [ -n $(is_function "pre_build") ]; then pre_build; fi
     if [ -n "$BUILD_DEPENDS" ]; then pip install $BUILD_DEPENDS; fi
     if [ -n "$REPO_DIR" ]; then
-        cd $REPO_DIR
-        git fetch origin
-        git checkout $BUILD_COMMIT
-        git clean -fxd
-        pip wheel -w $wheelhouse --no-deps .
-        cd ..
+        (cd $REPO_DIR \
+            && clean_fix_source $BUILD_COMMIT \
+            && pip wheel -w $wheelhouse --no-deps .)
     else
         pip wheel -w $wheelhouse --no-deps $PKG_SPEC
     fi

travis_steps.sh

@@ -3,9 +3,10 @@
 git submodule update --init --recursive
 
 WHEEL_SDIR=${WHEEL_SDIR:-wheelhouse}
-MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
 MANYLINUX_URL=${MANYLINUX_URL:-https://nipy.bic.berkeley.edu/manylinux}
-RUN_TESTS_SCRIPT=${RUN_TESTS_SCRIPT:-install_test.sh}
+MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
+# Get utilities common to OSX and Linux
+source $MULTIBUILD_DIR/common_utils.sh
 
 # Specify REPO_DIR to build from directory in this repository.
 # Specify PKG_SPEC to build from pip requirement (e.g numpy==1.7.1)