Merge pull request #297 from mattip/fix-pypy

MRG: fixes for platform, pypy wheel builds

.gitignore

 downloads/
+archives/
 *.orig
 *.swp
+
+# lib_check downloads these
+*-stamp
+arb*/
+bzip2*/
+cfitsio/
+flex*/
+freetype*/
+giflib*/
+hdf5*/
+jpeg*/
+lcms2*/
+libaec*/
+libpng*/
+libwebp*/
+lzo*/
+mpir*/
+openssl*/
+pcre*/
+ragel*/
+szip*/
+swig*/
+tiff*/
+xz*/
+yaml*/

.travis.yml

@@ -44,6 +44,12 @@ matrix:
       env:
         - TEST_BUILDS=1
         - USE_CCACHE=1
+    # Build with pypy3
+    - os: linux
+      env:
+        - MB_PYTHON_VERSION=pypy3.6-7.3
+        - MB_ML_VER=2014
+        - TEST_BUILDS=1
     # OSX builds
     - os: osx
       env:
@@ -108,6 +114,7 @@ matrix:
     - os: osx
       env:
         - MB_PYTHON_VERSION=pypy-7.2
+        - TEST_BUILDS=1
         - VENV=venv
     # Default OSX (xcode image) is 10.13 (xcode 9.4.1) as of 2018-08-03
     # See: https://docs.travis-ci.com/user/reference/osx/
@@ -183,7 +190,7 @@ matrix:
       arch: arm64
       env:
         - MB_PYTHON_VERSION=3.7
-        - MB_ML_VER=2014 
+        - MB_ML_VER=2014
         - TEST_BUILDS=1
         - PLAT=aarch64
     #  s390x builds
@@ -191,7 +198,7 @@ matrix:
       arch: s390x
       env:
         - MB_PYTHON_VERSION=3.7
-        - MB_ML_VER=2014 
+        - MB_ML_VER=2014
         - TEST_BUILDS=1
         - PLAT=s390x
     #  ppc64le builds
@@ -199,7 +206,7 @@ matrix:
       arch: ppc64le
       env:
         - MB_PYTHON_VERSION=3.7
-        - MB_ML_VER=2014 
+        - MB_ML_VER=2014
         - TEST_BUILDS=1
         - PLAT=ppc64le
 

common_utils.sh

@@ -12,6 +12,10 @@ COMMON_UTILS_SOURCED=1
 set -e
 
 MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
+DOWNLOADS_SDIR=downloads
+PYPY_URL=https://bitbucket.org/pypy/pypy/downloads
+GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
+
 if [ $(uname) == "Darwin" ]; then IS_OSX=1; fi
 
 # Work round bug in travis xcode image described at
@@ -372,12 +376,7 @@ function fill_submodule {
     (cd "$repo_dir" && git remote set-url origin $origin_url)
 }
 
-PYPY_URL=https://bitbucket.org/pypy/pypy/downloads
-
-# As of 2019-10-15, the latest verions of PyPy.
-LATEST_PP_4p0=4.0.1
-LATEST_PP_4=$LATEST_PP_4p0
-
+# As of 2020-01-15, the latest verions of PyPy.
 LATEST_PP_5p0=5.0.1
 LATEST_PP_5p1=5.1.1
 LATEST_PP_5p3=5.3.1
@@ -394,7 +393,8 @@ LATEST_PP_6=$LATEST_PP_6p0
 LATEST_PP_7p0=7.0.0
 LATEST_PP_7p1=7.1.1
 LATEST_PP_7p2=7.2.0
-LATEST_PP_7=$LATEST_PP_7p2
+LATEST_PP_7p3=7.3.0
+LATEST_PP_7=$LATEST_PP_7p3
 
 function unroll_version {
     # Convert major or major.minor format to major.minor.micro using the above
@@ -415,6 +415,55 @@ function unroll_version {
     fi
 }
 
+function install_pypy {
+    # Installs pypy.org PyPy
+    # Parameter $version
+    # Version given in major or major.minor or major.minor.micro e.g
+    # "3" or "3.7" or "3.7.1".
+    # Uses $PLAT
+    # sets $PYTHON_EXE variable to python executable
+
+    local version=$1
+    suffix=linux64
+    case "$PLAT" in
+    "x86_64")  suffix="linux64";;
+    "i686")    suffix="linux32";;
+    "darwin")  suffix="osx64";;
+    "ppc64le") suffix="ppc64le";;
+    "s30x")    suffix="s390x";;
+    "aarch64")  suffix="aarch64";;
+    *) if [ -n "$IS_OSX" ]; then
+            suffix="osx64";
+       else
+            echo unknown platform "$PLAT"; exit 1
+       fi;;
+    esac
+
+    # Need to convert pypy-7.2 to pypy2.7-v7.2.0 and pypy3.6-7.3 to pypy3.6-v7.3.0
+    local prefix=$(get_pypy_build_prefix $version)
+    # since prefix is pypy3.6v7.2 or pypy2.7v7.2, grab the 4th (0-index) letter
+    local major=${prefix:4:1}
+    # get the pypy version 7.2.0
+    local py_version=$(fill_pypy_ver $(echo $version | cut -f2 -d-))
+
+    local py_build=$prefix$py_version-$suffix
+    local py_zip=$py_build.tar.bz2
+    local zip_path=$DOWNLOADS_SDIR/$py_zip
+    mkdir -p $DOWNLOADS_SDIR
+    wget -nv $PYPY_URL/${py_zip} -P $DOWNLOADS_SDIR
+    untar $zip_path
+    # bug/feature: pypy package for pypy3 only has bin/pypy3 :(
+    if [ "$major" == "3" ] && [ ! -x "$py_build/bin/pypy" ]; then
+        ln $py_build/bin/pypy3 $py_build/bin/pypy
+    fi
+    PYTHON_EXE=$(realpath $py_build/bin/pypy)
+    $PYTHON_EXE -mensurepip
+    if [ "$major" == "3" ] && [ ! -x "$py_build/bin/pip" ]; then
+        ln $py_build/bin/pip3 $py_build/bin/pip
+    fi
+    PIP_CMD=pip
+}
+
 function fill_pypy_ver {
     # Convert major or major.minor format to major.minor.micro
     # Parameters:
@@ -427,9 +476,13 @@ function fill_pypy_ver {
 function get_pypy_build_prefix {
     # Return the file prefix of a PyPy file
     # Parameters:
-    #   $version : pypy2 version number
+    #   $version : pypy version number, for example pypy-7.2 or pypy3.6-7.2
     local version=$1
-    if [[ $version =~ ([0-9]+)\.([0-9]+) ]]; then
+    if [[ $version =~ pypy([0-9]+)\.([0-9]+)-([0-9]+)\.([0-9]+) ]]; then
+        local py_major=${BASH_REMATCH[1]}
+        local py_minor=${BASH_REMATCH[2]}
+        echo "pypy$py_major.$py_minor-v"
+    elif [[ $version =~ ([0-9]+)\.([0-9]+) ]]; then
         local major=${BASH_REMATCH[1]}
         local minor=${BASH_REMATCH[2]}
         if (( $major > 6 )); then
@@ -440,7 +493,7 @@ function get_pypy_build_prefix {
             echo "pypy-"
         fi
     else
-        echo "error: expected version number, got $1" 1>&2
+        echo "error: expected version like pypy-7.2 or pypy3.6-7.2, got $1" 1>&2
         exit 1
     fi
 }
@@ -462,3 +515,49 @@ retry () {
     }
     return 0
 }
+
+function install_pip {
+    # Generic install pip
+    # Gets needed version from version implied by $PYTHON_EXE
+    # Installs pip into python given by $PYTHON_EXE
+    # Assumes pip will be installed into same directory as $PYTHON_EXE
+    check_python
+    mkdir -p $DOWNLOADS_SDIR
+    local py_mm=`get_py_mm`
+    local get_pip_path=$DOWNLOADS_SDIR/get-pip.py
+    curl $GET_PIP_URL > $get_pip_path
+    # Travis VMS now install pip for system python by default - force install
+    # even if installed already.
+    $PYTHON_EXE $get_pip_path --ignore-installed $pip_args
+    PIP_CMD=$(dirname $PYTHON_EXE)/pip$py_mm
+    if [ "$USER" != "root" ]; then
+        # inside a docker, there is no sudo but the user is already root
+        PIP_CMD="sudo $PIP_CMD"
+    fi
+    # Append pip_args if present (avoiding trailing space cf using variable
+    # above).
+    if [ -n "$pip_args" ]; then
+        PIP_CMD="$PIP_CMD $pip_args"
+    fi
+}
+
+function check_python {
+    if [ -z "$PYTHON_EXE" ]; then
+        echo "PYTHON_EXE variable not defined"
+        exit 1
+    fi
+}
+
+function check_pip {
+    if [ -z "$PIP_CMD" ]; then
+        echo "PIP_CMD variable not defined"
+        exit 1
+    fi
+}
+
+function get_py_mm {
+    check_python
+    $PYTHON_EXE -c "import sys; print('{0}.{1}'.format(*sys.version_info[0:2]))"
+}
+
+

configure_build.sh

@@ -10,6 +10,7 @@ fi
 CONFIGURE_BUILD_SOURCED=1
 
 BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}"
+MB_ML_VER=${MB_ML_VER:-1}
 
 # IS_OSX is defined in common_utils.sh
 if [ -n "$IS_OSX" ]; then
@@ -41,7 +42,15 @@ else
     export CFLAGS="${CFLAGS:-$STRIP_FLAGS}"
     export CXXFLAGS="${CXXFLAGS:-$STRIP_FLAGS}"
     export FFLAGS="${FFLAGS:-$STRIP_FLAGS}"
-    yum install -y libtool
+    if [[ $MB_ML_VER == "1" ]]; then
+        if [ "${MB_PYTHON_VERSION:0:4}" == "pypy" ]; then
+            yum install -y libtool wget
+        else
+            yum install -y libtool
+        fi
+    elif [ "${MB_PYTHON_VERSION:0:4}" == "pypy" ]; then
+        yum install -y wget
+    fi
 fi
 
 # Promote BUILD_PREFIX on search path to any newly built libs

docker_build_wrap.sh

@@ -35,8 +35,17 @@ if [ "$USE_CCACHE" == "1" ]; then
     activate_ccache
 fi
 
-# Set PATH for chosen Python, Unicode width
-export PATH="$(cpython_path $PYTHON_VERSION $UNICODE_WIDTH)/bin:$PATH"
+# The following also sets PYTHON_EXE and PIP_CMD
+if [ "${PYTHON_VERSION:0:4}" == "pypy" ]; then
+  install_pypy $PYTHON_VERSION
+  export PATH=$(dirname $PYTHON_EXE):$PATH
+else
+  # Set PATH for chosen Python, Unicode width
+  PYTHON_EXE=$(cpython_path $PYTHON_VERSION $UNICODE_WIDTH)/bin/python
+  ls $(dirname $PYTHON_EXE)
+  export PATH="$(dirname $PYTHON_EXE):$PATH"
+  install_pip
+fi
 
 # Configuration for this package, possibly overriding `build_wheel` defined in
 # `common_utils.sh` via `manylinux_utils.sh`.

osx_utils.sh

@@ -9,7 +9,6 @@ source $MULTIBUILD_DIR/common_utils.sh
 MACPYTHON_URL=https://www.python.org/ftp/python
 MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
 GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
-DOWNLOADS_SDIR=downloads
 WORKING_SDIR=working
 
 # As of 19 Dec 2019 - latest Python of each version with binary download
@@ -226,7 +225,7 @@ function macpython_impl_for_version {
     #         "pypy-5.4" for PyPy
     local version=$1
     check_var $1
-    if [[ "$version" =~ pypy-([0-9\.]+) ]]; then
+    if [[ "$version" =~ ^pypy ]]; then
         echo pp
     elif [[ "$version" =~ ([0-9\.]+) ]]; then
         echo cp
@@ -254,7 +253,7 @@ function install_macpython {
     # Install Python and set $PYTHON_EXE to the installed executable
     # Parameters:
     #     $version : [implementation-]major[.minor[.patch]]
-    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
+    #         The Python implementation to install, e.g. "3.6", "pypy-5.4" or "pypy3.6-7.2"
     #     $py_osx_ver: {major.minor | not defined}
     #       if defined, the macOS version that CPython is built for, e.g.
     #       "10.6" or "10.9". Ignored for PyPy
@@ -263,7 +262,7 @@ function install_macpython {
     local impl=$(macpython_impl_for_version $version)
     local stripped_ver=$(strip_macpython_ver_prefix $version)
     if [[ "$impl" == "pp" ]]; then
-        install_mac_pypy $stripped_ver
+        install_pypy $version
     elif [[ "$impl" == "cp" ]]; then
         install_mac_cpython $stripped_ver $py_osx_ver
     else
@@ -304,43 +303,6 @@ function install_mac_cpython {
     fi
 }
 
-function install_mac_pypy {
-    # Installs pypy.org PyPy
-    # Parameter $version
-    # Version given in major or major.minor or major.minor.micro e.g
-    # "3" or "3.7" or "3.7.1".
-    # sets $PYTHON_EXE variable to python executable
-    local py_version=$(fill_pypy_ver $1)
-    local py_build=$(get_pypy_build_prefix $py_version)$py_version-osx64
-    local py_zip=$py_build.tar.bz2
-    local zip_path=$DOWNLOADS_SDIR/$py_zip
-    mkdir -p $DOWNLOADS_SDIR
-    wget -nv $PYPY_URL/${py_zip} -P $DOWNLOADS_SDIR
-    untar $zip_path
-    PYTHON_EXE=$(realpath $py_build/bin/pypy)
-}
-
-function install_pip {
-    # Generic install pip
-    # Gets needed version from version implied by $PYTHON_EXE
-    # Installs pip into python given by $PYTHON_EXE
-    # Assumes pip will be installed into same directory as $PYTHON_EXE
-    check_python
-    mkdir -p $DOWNLOADS_SDIR
-    local py_mm=`get_py_mm`
-    local get_pip_path=$DOWNLOADS_SDIR/get-pip.py
-    curl $GET_PIP_URL > $get_pip_path
-    # Travis VMS now install pip for system python by default - force install
-    # even if installed already.
-    sudo $PYTHON_EXE $get_pip_path --ignore-installed $pip_args
-    PIP_CMD="sudo $(dirname $PYTHON_EXE)/pip$py_mm"
-    # Append pip_args if present (avoiding trailing space cf using variable
-    # above).
-    if [ -n "$pip_args" ]; then
-        PIP_CMD="$PIP_CMD $pip_args"
-    fi
-}
-
 function install_virtualenv {
     # Generic install of virtualenv
     # Installs virtualenv into python given by $PYTHON_EXE

tests/test_fill_pypy_ver.sh

 # Test Python version fill utility, for pypy
-[ "$(fill_pypy_ver 4)" == $LATEST_PP_4 ] || ingest "lpp4"
 [ "$(fill_pypy_ver 5)" == $LATEST_PP_5 ] || ingest "lpp5"
 [ "$(fill_pypy_ver 6)" == $LATEST_PP_6 ] || ingest "lpp6"
 [ "$(fill_pypy_ver 7)" == $LATEST_PP_7 ] || ingest "lpp7"
-[ "$(fill_pypy_ver 4.0)" == $LATEST_PP_4p0 ] || ingest
 [ "$(fill_pypy_ver 5.0)" == $LATEST_PP_5p0 ] || ingest
 [ "$(fill_pypy_ver 5.1)" == $LATEST_PP_5p1 ] || ingest
 [ "$(fill_pypy_ver 5.3)" == $LATEST_PP_5p3 ] || ingest
@@ -16,5 +14,4 @@
 [ "$(fill_pypy_ver 7.0)" == $LATEST_PP_7p0 ] || ingest
 [ "$(fill_pypy_ver 7.1)" == $LATEST_PP_7p1 ] || ingest
 [ "$(fill_pypy_ver 7.2)" == $LATEST_PP_7p2 ] || ingest
-[ "$(fill_pypy_ver 4.0.1)" == "4.0.1" ] || ingest
-[ "$(fill_pypy_ver 5.0.1)" == "5.0.1" ] || ingest
+[ "$(fill_pypy_ver 7.3)" == $LATEST_PP_7p3 ] || ingest

tests/test_library_builders.sh

@@ -50,7 +50,10 @@ if [ -z "$IS_OSX" ]; then
     suppress build_lzo
 fi
 suppress build_ragel
-suppress build_cfitsio
+if [ -z "$IS_OSX" ]; then
+    # already installed in the macOS image, so `brew install cfitsio` fails
+    suppress build_cfitsio
+fi
 suppress build_new_zlib
 suppress build_hdf5
 

tests/test_multibuild.sh

@@ -25,6 +25,7 @@ else
     source tests/test_manylinux_utils.sh
 fi
 if [ -n "$TEST_BUILDS" ]; then
+    MB_PYTHON_VERSION=${MB_PYTHON_VERSION:-3.7}
     if [ -n "$IS_OSX" ]; then
         # This checked in test_library_builders.
         # Will be set automatically by docker call in build_multilinux below.
@@ -40,6 +41,7 @@ if [ -n "$TEST_BUILDS" ]; then
             source tests/test_manylinux_utils_docker.sh
             source tests/test_library_builders.sh
         "
+        build_multilinux $my_plat "pip install simplejson"
     fi
 fi