Merge pull request #281 from mattip/manylinux2014

add aarch64, s390x, ppc64le manylinux2014 support

.travis.yml

@@ -20,7 +20,7 @@ matrix:
       dist: xenial
       env:
         - TEST_BUILDS=1
-    # 64-bit builds, manylinux2014
+    # 64-bit builds, manylinux2010
     - os: linux
       dist: xenial
       env:
@@ -32,6 +32,12 @@ matrix:
       env:
         - TEST_BUILDS=1
         - PLAT=i686
+    - os: linux
+      dist: xenial
+      env:
+        - TEST_BUILDS=1
+        - PLAT=i686
+        - MB_ML_VER=2010
     # Builds with caching
     - os: linux
       dist: xenial
@@ -172,6 +178,30 @@ matrix:
       osx_image: xcode7.3
       env:
         - MB_PYTHON_VERSION=3.5
+    # ARM aarch 64 builds
+    - os: linux
+      arch: arm64
+      env:
+        - MB_PYTHON_VERSION=3.7
+        - MB_ML_VER=2014 
+        - TEST_BUILDS=1
+        - PLAT=aarch64
+    #  s390x builds
+    - os: linux
+      arch: s390x
+      env:
+        - MB_PYTHON_VERSION=3.7
+        - MB_ML_VER=2014 
+        - TEST_BUILDS=1
+        - PLAT=s390x
+    #  ppc64le builds
+    - os: linux
+      arch: ppc64le
+      env:
+        - MB_PYTHON_VERSION=3.7
+        - MB_ML_VER=2014 
+        - TEST_BUILDS=1
+        - PLAT=ppc64le
 
 script:
   - export ENV_VARS_PATH="tests/env_vars.sh"

README.rst

@@ -84,23 +84,21 @@ a clean Ubuntu 14.04 container.
 Build phase
 -----------
 
-Specify the Manylinux version to build for with the `MB_ML_VER` environment variable. The default version is `1`.  Versions that are currently valid are:
+Specify the Manylinux version to build for with the ``MB_ML_VER`` environment
+variable. The default version is ``1``.  Versions that are currently valid are:
 
-* `1` (see [PEP 513](https://www.python.org/dev/peps/pep-0513);
-* `2010` (see [PEP
+* ``1`` corresponding to manylinux1 (see [PEP 513](https://www.python.org/dev/peps/pep-0513);
+* ``2010``  corresponding to manylinux2010 (see [PEP
   571](https://www.python.org/dev/peps/pep-0571).
-
-At some point `2014` will be a valid version - see [PEP
-599](https://www.python.org/dev/peps/pep-0599).
+* ``2014`` corresponding to manylinux2014 and adds more architectures to ``PLAT``
+  (see [PEP 599](https://www.python.org/dev/peps/pep-0599).
 
 The environment variable specified which Manylinux docker container you are building in.
 
-The `PLAT` environment variable can be one of `x86_64` or `i686`, specifying 64-bit and 32-bit builds, respectively.  The default is 64-bit.
-
-At the time of writing, Manylinux2010 only supports 64-bit
-builds, so `MB_ML_VER=2010` and `PLAT=i686` is an invalid
-combination, and will generate an error when trying to find the
-matching Docker image.
+The ``PLAT`` environment variable can be one of ``x86_64``, ``i686`` ``s390x``,
+``ppc64le``, or ``aarch64``, specifying 64-bit x86, 32-bit x86, 64-bit s390x,
+PowerPC, and ARM builds, respectively.  The default is ``x86_64``. Only ``x86_64``
+and ``i686`` are valid on manylinux1 and manylinux2010.
 
 ``multibuild/travis_linux_steps.sh`` defines the ``build_wheel`` function,
 which starts up the Manylinux1 Docker container to run a wrapper script

common_utils.sh

@@ -102,7 +102,7 @@ alias gh-clone=gh_clone
 function set_opts {
     # Set options from input options string (in $- format).
     local opts=$1
-    local chars="exhimBH"
+    local chars="exhmBH"
     for (( i=0; i<${#chars}; i++ )); do
         char=${chars:$i:1}
         [ -n "${opts//[^${char}]/}" ] && set -$char || set +$char
@@ -310,6 +310,9 @@ function get_platform {
     python -c 'import platform; print(platform.uname()[4])'
 }
 
+if [ "$(get_platform)" == x86_64 ] || \
+    [ "$(get_platform)" == i686 ]; then IS_X86=1; fi
+
 function get_distutils_platform {
     # Report platform as given by distutils get_platform.
     # This is the platform tag that pip will use.

configure_build.sh

@@ -41,6 +41,7 @@ else
     export CFLAGS="${CFLAGS:-$STRIP_FLAGS}"
     export CXXFLAGS="${CXXFLAGS:-$STRIP_FLAGS}"
     export FFLAGS="${FFLAGS:-$STRIP_FLAGS}"
+    yum install -y libtool
 fi
 
 # Promote BUILD_PREFIX on search path to any newly built libs

library_builders.sh

@@ -15,17 +15,17 @@ ZLIB_VERSION="${ZLIB_VERSION:-1.2.10}"
 LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.21}"
 BZIP2_VERSION="${BZIP2_VERSION:-1.0.6}"
 FREETYPE_VERSION="${FREETYPE_VERSION:-2.6.3}"
-TIFF_VERSION="${TIFF_VERSION:-4.0.6}"
+TIFF_VERSION="${TIFF_VERSION:-4.1.0}"
 JPEG_VERSION="${JPEG_VERSION:-9b}"
 OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}"
-LCMS2_VERSION="${LCMS2_VERSION:-2.7}"
+LCMS2_VERSION="${LCMS2_VERSION:-2.9}"
 GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}"
 LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}"
 XZ_VERSION="${XZ_VERSION:-5.2.2}"
-LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}"
+LIBYAML_VERSION="${LIBYAML_VERSION:-0.2.2}"
 SZIP_VERSION="${SZIP_VERSION:-2.1.1}"
 HDF5_VERSION="${HDF5_VERSION:-1.10.4}"
-LIBAEC_VERSION="${LIBAEC_VERSION:-0.3.3}"
+LIBAEC_VERSION="${LIBAEC_VERSION:-1.0.4}"
 LZO_VERSION=${LZO_VERSION:-2.10}
 LZF_VERSION="${LZF_VERSION:-3.6}"
 BLOSC_VERSION=${BLOSC_VERSION:-1.10.2}
@@ -40,7 +40,7 @@ RAGEL_VERSION=${RAGEL_VERSION:-6.10}
 FLEX_VERSION=${FLEX_VERSION:-2.6.4}
 BISON_VERSION=${BISON_VERSION:-3.0.4}
 FFTW_VERSION=${FFTW_VERSION:-3.3.7}
-CFITSIO_VERSION=${CFITSIO_VERSION:-3370}
+CFITSIO_VERSION=${CFITSIO_VERSION:-3450}
 OPENSSL_ROOT=openssl-1.0.2t
 # Hash from https://www.openssl.org/source/openssl-1.0.2?.tar.gz.sha256
 OPENSSL_HASH=14cb464efe7ac6b54799b34456bd69558a749a4931ecfd9cf9f71d7881cac7bc
@@ -94,6 +94,9 @@ function build_openblas {
     if [ -n "$IS_OSX" ]; then
         brew install openblas
         brew link --force openblas
+    elif [ ! -v IS_X86 ]; then
+		# Skip this for now until we can build a suitable tar.gz
+        return;
     else
         mkdir -p $ARCHIVE_SDIR
         local plat=${1:-${PLAT:-x86_64}}
@@ -236,10 +239,10 @@ function build_hdf5 {
 
 function build_libaec {
     if [ -e libaec-stamp ]; then return; fi
-    local root_name=libaec-0.3.3
+    local root_name=libaec-1.0.4
     local tar_name=${root_name}.tar.gz
     # Note URL will change for each version
-    fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/48398bd5b7bc05a3edb3325abfeac864/${tar_name}
+    fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/ea0b7d197a950b0c110da8dfdecbb71f/${tar_name}
     (cd $root_name \
         && ./configure --prefix=$BUILD_PREFIX \
         && make \

tests/test_common_utils.sh

@@ -86,8 +86,14 @@ actual="$(set -e; suppress bad_mid_cmd)"
 # Reset options
 set_opts $ORIG_OPTS
 
-# On Linux docker containers in travis, can only be x86_64 or i686
-[ "$(get_platform)" == x86_64 ] || [ "$(get_platform)" == i686 ] || exit 1
+# On Linux docker containers in travis, can be x86_64, i686, s390x, ppc64le, or
+# aarch64
+[ "$(get_platform)" == x86_64 ] || \
+    [ "$(get_platform)" == i686 ] || \
+    [ "$(get_platform)" == aarch64 ] || \
+    [ "$(get_platform)" == ppc64le ] || \
+    [ "$(get_platform)" == s390x ] || \
+    exit 1
 
 # Crudest possible check for get_distutils_platform
 expected=$(python -c "import distutils.util as du; print(du.get_platform())")