Skip to content

GitLab

Commit 75aa56d9 authored by Matthew Brett's avatar Matthew Brett
Browse files

Refactor to test on linux docker containers

parent cd784590
Hide whitespace changes
Inline Side-by-side
Showing with 104 additions and 81 deletions
common_utils.sh 0 → 100644
View file @ 75aa56d9
#!/bin/bash
# Utilities for both OSX and Docker
set -e
function abspath {
python -c "import os.path; print(os.path.abspath('$1'))"
}
# Get our own location on this filesystem
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
ROOT_DIR=$(abspath $MULTIBUILD_DIR/..)
function install_wheel {
# Install test dependencies and built wheel
# Pass any input flags to pip install steps
# Depends on:
# MANYLINUX_URL
# WHEEL_SDIR
# TEST_DEPENDS (optional)
if [ -n "$TEST_DEPENDS" ]; then
pip install --find-links $MANYLINUX_URL $@ $TEST_DEPENDS
fi
# Install compatible wheel
pip install --find-links $MANYLINUX_URL $@ \
$(python $MULTIBUILD_DIR/supported_wheels.py $ROOT_DIR/$WHEEL_SDIR/*.whl)
}
function install_run {
local run_tests_script=${1:-$RUN_TESTS_SCRIPT}
install_wheel
mkdir tmp_for_test
cd tmp_for_test
source $ROOT_DIR/$run_tests_script
cd ..
}
docker_install_run.sh 0 → 100755
View file @ 75aa56d9
#!/bin/bash
# Install and test steps on Linux
set -e
# Get needed utilities
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
source $MULTIBUILD_DIR/common_utils.sh
install_run $RUN_TESTS_SCRIPT
manylinux @ 20df2921
Compare 8786096e...20df2921
Subproject commit 8786096ec20bf5a11976b8489873602776303cc3 Subproject commit 20df29211c3b92626e4f6600c3938d3e7f3ea5ed
travis_linux_steps.sh
View file @ 75aa56d9
#!/bin/bash #!/bin/bash
# Travis steps for Linux # Wheel build, install, run test steps on Linux
set -e set -e
ROOT_DIR=$(dirname "${BASH_SOURCE[0]}") # Get needed utilities
UTIL_DIR=${UTIL_DIR:-${ROOT_DIR}/manylinux} MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
UTIL_DIR=${UTIL_DIR:-${MULTIBUILD_DIR}/manylinux}
BUILD_SCRIPT=${BUILD_SCRIPT:-/io/$UTIL_DIR/build_package.sh} BUILD_SCRIPT=${BUILD_SCRIPT:-/io/$UTIL_DIR/build_package.sh}
UNICODE_WIDTHS=${UNICODE_WIDTHS:-32} UNICODE_WIDTH=${UNICODE_WIDTH:-32}
function before_install { function before_install {
virtualenv --python=python venv virtualenv --python=python venv
...@@ -14,10 +15,11 @@ function before_install { ...@@ -14,10 +15,11 @@ function before_install {
pip install --upgrade pip wheel pip install --upgrade pip wheel
} }
function build_wheels { function build_wheel {
# Builds wheel, puts into $WHEELHOUSE # Builds wheel, puts into $WHEEL_SDIR
# #
# Depends on # Depends on
# PLAT
# BUILD_DEPENDS # BUILD_DEPENDS
# BUILD_COMMIT # BUILD_COMMIT
# BUILD_PRE_SCRIPT # BUILD_PRE_SCRIPT
...@@ -25,50 +27,14 @@ function build_wheels { ...@@ -25,50 +27,14 @@ function build_wheels {
# REPO_DIR | PKG_SPEC # REPO_DIR | PKG_SPEC
# TRAVIS_PYTHON_VERSION # TRAVIS_PYTHON_VERSION
# #
# Build both 32- and 64-bit local plat=${1:-$PLAT}
build_plat_wheels i686
build_plat_wheels x86_64
}
function valid_unicode_widths {
local py_ver=${1:-$TRAVIS_PYTHON_VERSION}
local ok_widths=""
if [ "${py_ver:0:1}" == 2 ]; then local py2=1; fi
for width in ${@:2}; do
if [ "$width" == 32 ]; then
ok_widths="$ok_widths 32"
elif [ "$width" == 16 ]; then
if [ -n "$py2" ]; then
ok_widths="$ok_widths 16"
fi
else
echo "Invalid unicode width $width"
exit 1
fi
done
echo $ok_widths
}
function build_plat_wheels {
# Builds wheels
#
# Depends on
#
# BUILD_DEPENDS (can be empty)
# BUILD_COMMIT
# BUILD_PRE_SCRIPT (can be empty)
# BUILD_SCRIPT
# REPO_DIR | PKG_SPEC
# TRAVIS_PYTHON_VERSION
local plat=${1:-x86_64}
local docker_image=quay.io/pypa/manylinux1_$plat local docker_image=quay.io/pypa/manylinux1_$plat
docker pull $docker_image docker pull $docker_image
if [ "$plat" == "i686" ]; then local intro_cmd=linux32; fi if [ "$plat" == "i686" ]; then local intro_cmd=linux32; fi
local widths=$(valid_unicode_widths $TRAVIS_PYTHON_VERSION $UNICODE_WIDTHS)
docker run --rm \ docker run --rm \
-e UTIL_DIR="$UTIL_DIR" \
-e PYTHON_VERSION="$TRAVIS_PYTHON_VERSION" \ -e PYTHON_VERSION="$TRAVIS_PYTHON_VERSION" \
-e UNICODE_WIDTHS="$widths" \ -e UNICODE_WIDTH="$UNICODE_WIDTH" \
-e WHEEL_SDIR="$WHEEL_SDIR" \
-e BUILD_DEPENDS="$BUILD_DEPENDS" \ -e BUILD_DEPENDS="$BUILD_DEPENDS" \
-e BUILD_COMMIT="$BUILD_COMMIT" \ -e BUILD_COMMIT="$BUILD_COMMIT" \
-e BUILD_PRE_SCRIPT="$BUILD_PRE_SCRIPT" \ -e BUILD_PRE_SCRIPT="$BUILD_PRE_SCRIPT" \
...@@ -77,3 +43,29 @@ function build_plat_wheels { ...@@ -77,3 +43,29 @@ function build_plat_wheels {
-v $PWD:/io \ -v $PWD:/io \
$docker_image $intro_cmd $BUILD_SCRIPT $docker_image $intro_cmd $BUILD_SCRIPT
} }
function relpath {
python -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))"
}
function install_run {
local run_tests_script=${1:-$RUN_TESTS_SCRIPT}
local plat=${2:-$PLAT}
if [ "$plat" == "i686" ]; then
local bitness=32
else
local bitness=64
fi
local docker_image="matthewbrett/trusty:$bitness"
local multibuild_sdir=$(relpath $MULTIBUILD_DIR)
docker pull $docker_image
docker run --rm \
-e PYTHON_VERSION="$TRAVIS_PYTHON_VERSION" \
-e UNICODE_WIDTH="$UNICODE_WIDTH" \
-e WHEEL_SDIR="$WHEEL_SDIR" \
-e RUN_TESTS_SCRIPT="$run_tests_script" \
-e MANYLINUX_URL="$MANYLINUX_URL" \
-e TEST_DEPENDS="$TEST_DEPENDS" \
-v $PWD:/io \
$docker_image /io/$multibuild_sdir/docker_install_run.sh
}
travis_osx_steps.sh
View file @ 75aa56d9
#!/bin/bash #!/bin/bash
# Install and wheel building steps on OSX # Wheel build, install, run test steps on OSX
set -e set -e
# Get needed utilities # Get needed utilities
ROOT_DIR=$(dirname "${BASH_SOURCE[0]}") MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
TERRYFY_DIR=$ROOT_DIR/terryfy source $MULTIBUILD_DIR/terryfy/travis_tools.sh
source $TERRYFY_DIR/travis_tools.sh source $MULTIBUILD_DIR/common_utils.sh
function before_install { function before_install {
export CC=clang export CC=clang
...@@ -15,27 +15,28 @@ function before_install { ...@@ -15,27 +15,28 @@ function before_install {
pip install --upgrade pip wheel pip install --upgrade pip wheel
} }
function build_wheels { function build_wheel {
# Builds wheel, puts into $WHEELHOUSE # Builds wheel, puts into $WHEEL_SDIR
# #
# Depends on # Depends on
# REPO_DIR | PKG_SPEC # WHEEL_SDIR
# BUILD_DEPENDS # BUILD_DEPENDS
# REPO_DIR | PKG_SPEC
# BUILD_COMMIT # BUILD_COMMIT
# WHEELHOUSE local wheelhouse=$PWD/$WHEEL_SDIR
if [ -n "$BUILD_DEPENDS" ]; then pip install $BUILD_DEPENDS; fi if [ -n "$BUILD_DEPENDS" ]; then pip install $BUILD_DEPENDS; fi
if [ -n "$REPO_DIR" ]; then if [ -n "$REPO_DIR" ]; then
cd $REPO_DIR cd $REPO_DIR
git fetch origin git fetch origin
git checkout $BUILD_COMMIT git checkout $BUILD_COMMIT
git clean -fxd git clean -fxd
pip wheel -w $WHEELHOUSE --no-deps . pip wheel -w $wheelhouse --no-deps .
cd .. cd ..
else else
pip wheel -w $WHEELHOUSE --no-deps $PKG_SPEC pip wheel -w $wheelhouse --no-deps $PKG_SPEC
fi fi
pip install delocate pip install delocate
delocate-listdeps $WHEELHOUSE/*.whl # lists library dependencies delocate-listdeps $wheelhouse/*.whl # lists library dependencies
delocate-wheel $WHEELHOUSE/*.whl # copies library dependencies into wheel delocate-wheel $wheelhouse/*.whl # copies library dependencies into wheel
delocate-addplat --rm-orig -x 10_9 -x 10_10 $WHEELHOUSE/*.whl delocate-addplat --rm-orig -x 10_9 -x 10_10 $wheelhouse/*.whl
} }
travis_steps.sh
View file @ 75aa56d9
...@@ -2,16 +2,10 @@ ...@@ -2,16 +2,10 @@
# Update submodules # Update submodules
git submodule update --init --recursive git submodule update --init --recursive
WHEELHOUSE=$PWD/wheelhouse WHEEL_SDIR=${WHEEL_SDIR:-wheelhouse}
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
MANYLINUX_URL=${MANYLINUX_URL:-https://nipy.bic.berkeley.edu/manylinux} MANYLINUX_URL=${MANYLINUX_URL:-https://nipy.bic.berkeley.edu/manylinux}
RUN_TESTS_SCRIPT=${RUN_TESTS_SCRIPT:-run_tests.sh}
if [ ! -d "$WHEELHOUSE" ]; then mkdir $WHEELHOUSE; fi
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
source $MULTIBUILD_DIR/travis_osx_steps.sh
else
source $MULTIBUILD_DIR/travis_linux_steps.sh
fi
# Specify REPO_DIR to build from directory in this repository. # Specify REPO_DIR to build from directory in this repository.
# Specify PKG_SPEC to build from pip requirement (e.g numpy==1.7.1) # Specify PKG_SPEC to build from pip requirement (e.g numpy==1.7.1)
...@@ -21,17 +15,9 @@ if [ -z "$REPO_DIR$PKG_SPEC" ]; then ...@@ -21,17 +15,9 @@ if [ -z "$REPO_DIR$PKG_SPEC" ]; then
exit 1 exit 1
fi fi
function install_wheel { if [ ! -d "$PWD/$WHEEL_SDIR" ]; then mkdir $PWD/WHEEL_SDIR; fi
# Install test dependencies and built wheel if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
# Pass any input flags to pip install steps source $MULTIBUILD_DIR/travis_osx_steps.sh
# Depends on: else
# MANYLINUX_URL source $MULTIBUILD_DIR/travis_linux_steps.sh
# WHEELHOUSE fi
# TEST_DEPENDS (optional)
if [ -n "$TEST_DEPENDS" ]; then
pip install --find-links $MANYLINUX_URL $@ $TEST_DEPENDS
fi
# Install compatible wheel
pip install --find-links $MANYLINUX_URL $@ \
$(python $MULTIBUILD_DIR/supported_wheels.py $WHEELHOUSE/*.whl)
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment