Merge branch 'devel' into manylinux2014

.travis.yml

@@ -48,11 +48,11 @@ matrix:
     - os: osx
       env:
         - MB_PYTHON_VERSION=2.7
+        - MB_PYTHON_OSX_VER=10.6
         - TEST_BUILDS=1
     - os: osx
       env:
         - MB_PYTHON_VERSION=2.7
-        - MB_PYTHON_OSX_VER=10.9
         - TEST_BUILDS=1
     - os: osx
       language: objective-c
@@ -69,21 +69,21 @@ matrix:
     - os: osx
       env:
         - MB_PYTHON_VERSION=3.6
+        - MB_PYTHON_OSX_VER=10.6
         - VENV=venv
     - os: osx
       env:
         - MB_PYTHON_VERSION=3.6
         - VENV=venv
-        - MB_PYTHON_OSX_VER=10.9
     - os: osx
       env:
         - MB_PYTHON_VERSION=3.7
+        - MB_PYTHON_OSX_VER=10.6
         - VENV=venv
     - os: osx
       env:
         - MB_PYTHON_VERSION=3.7
         - VENV=venv
-        - MB_PYTHON_OSX_VER=10.9
     - os: osx
       env:
         - MB_PYTHON_VERSION=3.7

README.rst

@@ -8,10 +8,10 @@ wheels on the `AppVeyor <https://ci.appveyor.com/>`_ infrastructure.
 
 The Travis CI scripts are designed to build *and test*:
 
-* Dual 32/64-bit architecture macOS wheels built for macOS 10.6+;
-* 64-bit macOS wheels built for macOS 10.9+;
-* 64-bit ``manylinuxX_x86_64`` wheels, both narrow and wide Unicode builds, where `X` is any valid Manylinux version, such as `1`, or `2010`.
-* 32-bit ``manylinuxX_i686`` wheels, both narrow and wide Unicode builds.
+* Dual 32/64-bit architecture macOS wheels built for macOS 10.6+
+* 64-bit macOS wheels built for macOS 10.9+
+* 64-bit ``manylinuxX_x86_64`` wheels, both narrow and wide Unicode builds, where `X` is any valid Manylinux version, such as `1`, or `2010`
+* 32-bit ``manylinuxX_i686`` wheels, both narrow and wide Unicode builds
 
 You can currently build and test against Pythons 2.7, 3.5, 3.6, 3.7.
 
@@ -21,8 +21,8 @@ Travis CI configurations.
 
 The AppVeyor setup is designed to build *and test*:
 
-* 64-bit Windows ``win_amd64`` wheels;
-* 32-bit Windows ``win32`` wheels.
+* 64-bit Windows ``win_amd64`` wheels
+* 32-bit Windows ``win32`` wheels
 
 You can currently build and test against Pythons 2.7, 3.5, 3.6, 3.7.
 
@@ -42,34 +42,34 @@ sequence:
 macOS
 =====
 
-See ``multibuild/travis_osx_steps.sh``.
-
-For build and test phases, these bash scripts get sourced one after the other,
+These bash scripts get sourced one after the other,
 so that functions and variables defined in later scripts can overwrite
-functions and variables in earlier scripts:
+functions and variables in earlier scripts::
 
-* multibuild/common_utils.sh
-* multibuild/osx_utils.sh
-* env_vars.sh
-* multibuild/configure_build.sh
-* multibuild/library_builders.sh
-* config.sh
+    multibuild/common_utils.sh
+    multibuild/osx_utils.sh
+    env_vars.sh
+    multibuild/configure_build.sh
+    multibuild/library_builders.sh
+    config.sh
 
-See ``travis_osx_steps.sh`` to review source order.
+See ``multibuild/travis_osx_steps.sh`` to review the source order.
 
 The macOS build / test phases run on the macOS VM started by Travis CI.
-Therefore any environment variable defined in the ``.travis.yml`` or bash
+Therefore any environment variable defined in ``.travis.yml`` or the bash
 shell scripts listed above are available for your build and test.
 
-macOS builds may be targeted either at macOS 10.6+
-(dual arch 64 / 32 bit) or macOS 10.9+ (64b only). These depend on the
-corresponding build of python from https://www.python.org/downloads/mac-osx/.
-At the time of writing, 10.9+ / 64 bit builds are supported for Python
-versions 3.6.5 / 2.7.15 and above. If you want to build for an older version
-of Python, you'll have to target 10.6+ / dual arch.
+Build options are controlled mainly by the following environment
+variables:
+
+* ``MB_PYTHON_VER`` selects the Python version built for, in the format ``major.minor.patch`` for CPython, or ``pypy-major.minor`` for PyPy
+* ``MB_PYTHON_OSX_VER`` sets the minimum macOS SDK version targetted. For CPython it may be set to 10.9 or 10.6 (the default). It is currently ignored for PyPy builds.
+* ``PLAT`` sets the architecture(s) built, either ``x86_64`` or ``intel`` for 64-bit or 64/32-bit respectively. The default is the same as the Python version selected by ``MB_PYTHON_VER`` and ``MB_PYTHON_OSX_VER``: 64-bit for PyPy or CPython 10.9 builds, and 64/32-bit for CPython 10.6 builds. For normal usage you should not need to set this variable.
+
+Valid combinations of ``MB_PYTHON_VER`` and ``MB_PYTHON_OSX_VER`` for CPython correspond to Python versions available for download at `python.org <https://www.python.org/downloads/mac-osx/>`_.
 
 The ``build_wheel`` function builds the wheel, and the ``install_run``
-function installs the wheel and tests it.  Look in ``common_utils.sh`` for
+function installs the wheel and tests it.  Look in ``multibuild/common_utils.sh`` for
 default definitions of these functions.  See below for more details.
 
 Manylinux
@@ -149,7 +149,7 @@ default the function that is run on macOS, and in the Manylinux container for
 the build phase, is defined in ``multibuild/common_utils.sh``.  You can
 override the default function in the project ``config.sh`` file (see below).
 
-If you are building a wheel from pypi, rather than from a source repository,
+If you are building a wheel from PyPI, rather than from a source repository,
 you can use the ``build_index_wheel`` command, again defined in
 ``multibuild/common_utils.sh``.
 
@@ -194,12 +194,11 @@ To use these scripts
             # Commit from your-project that you want to build
             - BUILD_COMMIT=v0.1.0
             # pip dependencies to _build_ your project
-            - BUILD_DEPENDS="Cython numpy"
+            - BUILD_DEPENDS="cython numpy"
             # pip dependencies to _test_ your project.  Include any dependencies
             # that you need, that are also specified in BUILD_DEPENDS, this will be
             # a separate install.
             - TEST_DEPENDS="numpy scipy pytest"
-            - PLAT=x86_64
             - UNICODE_WIDTH=32
             - WHEELHOUSE_UPLOADER_USERNAME=travis-worker
             # Following generated with

configure_build.sh

@@ -9,14 +9,13 @@ if [ -n "$CONFIGURE_BUILD_SOURCED" ]; then
 fi
 CONFIGURE_BUILD_SOURCED=1
 
-PLAT="${PLAT:-x86_64}"
 BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}"
 
-# Default compilation flags for OSX
 # IS_OSX is defined in common_utils.sh
 if [ -n "$IS_OSX" ]; then
+    # Default compilation flags for OSX
     source $MULTIBUILD_DIR/osx_utils.sh
-    PLAT=$(macpython_arch_for_version $MB_PYTHON_VERSION)
+    PLAT=${PLAT:-$(macpython_arch_for_version $MB_PYTHON_VERSION)}
     if [[ $PLAT == intel ]]; then
         ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"}
     elif [[ $PLAT == x86_64 ]]; then
@@ -34,6 +33,8 @@ if [ -n "$IS_OSX" ]; then
     # Disable homebrew auto-update
     export HOMEBREW_NO_AUTO_UPDATE=1
 else
+    # default compilation flags for linux
+    PLAT="${PLAT:-x86_64}"
     # Strip all binaries after compilation.
     STRIP_FLAGS=${STRIP_FLAGS:-"-Wl,-strip-all"}
 

osx_utils.sh

@@ -8,12 +8,6 @@ source $MULTIBUILD_DIR/common_utils.sh
 
 MACPYTHON_URL=https://www.python.org/ftp/python
 MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
-MACPYTHON_DEFAULT_OSX="10.6"
-if [ "$(lex_ver $MB_PYTHON_VERSION)" -ge "$(lex_ver 3.8)" ]; then
-    # At 3.8 Python.org dropped the 10.6 installer.
-    MACPYTHON_DEFAULT_OSX="10.9"
-fi
-MB_PYTHON_OSX_VER=${MB_PYTHON_OSX_VER:-$MACPYTHON_DEFAULT_OSX}
 GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
 DOWNLOADS_SDIR=downloads
 WORKING_SDIR=working
@@ -96,6 +90,36 @@ function fill_pyver {
     fi
 }
 
+function macpython_sdk_list_for_version {
+    # return a list of SDK targets supported for a given CPython version
+    # Parameters
+    #   $py_version (python version in major.minor.extra format)
+    # eg
+    #  macpython_sdks_for_version 2.7.15
+    #  >> 10.6 10.9
+    local _ver=$(fill_pyver $1)
+    local _major=${_ver%%.*}
+    local _return
+
+    if [ "$_major" -eq "2" ]; then
+        _return="10.6"
+        [ $(lex_ver $_ver) -ge $(lex_ver 2.7.15) ] && _return="$_return 10.9"
+    elif [ "$_major" -eq "3" ]; then
+        [ $(lex_ver $_ver) -lt $(lex_ver 3.8)    ] && _return="10.6"
+        [ $(lex_ver $_ver) -ge $(lex_ver 3.6.5)  ] && _return="$_return 10.9"
+    else
+        echo "Error version=${_ver}, expecting 2.x or 3.x" 1>&2
+        exit 1
+    fi
+    echo $_return
+}
+
+function macpython_sdk_for_version {
+    # assumes the output of macpython_sdk_list_for_version is a list
+    # of SDK versions XX.Y in sorted order, eg "10.6 10.9" or "10.9"
+    echo $(macpython_sdk_list_for_version $1) | awk -F' ' '{print $NF}'
+}
+
 function pyinst_ext_for_version {
     # echo "pkg" or "dmg" depending on the passed Python version
     # Parameters
@@ -123,13 +147,11 @@ function pyinst_fname_for_version {
     # Parameters
     #   $py_version (Python version in major.minor.extra format)
     #   $py_osx_ver: {major.minor | not defined}
-    #       if defined, the macOS version that Python is built for, e.g.
-    #       "10.6" or "10.9", if not defined, uses the default
-    #       MACPYTHON_DEFAULT_OSX
-    #       Note: this is the version the Python is built for, and hence
-    #       the min version supported, NOT the version of the current system
+    #       if defined, the minimum macOS SDK version that Python is
+    #       built for, eg: "10.6" or "10.9", if not defined, infers
+    #       this from $py_version using macpython_sdk_for_version
     local py_version=$1
-    local py_osx_ver=${2:-$MACPYTHON_DEFAULT_OSX}
+    local py_osx_ver=${2:-$(macpython_sdk_for_version $py_version)}
     local inst_ext=$(pyinst_ext_for_version $py_version)
     echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}"
 }

tests/test_library_builders.sh

@@ -32,7 +32,7 @@ suppress build_swig
 # E.g. arb (below) requires a couple of other libraries.
 # Run here just for the output, even though they fail.
 (set +e ;
-    build_github fredrik-johansson/arb 2.16.0 ;
+    build_github fredrik-johansson/arb 2.17.0 ;
     build_github glennrp/libpng v1.6.37 ;
     build_github wbhart/mpir mpir-3.0.0
     )

tests/test_multibuild.sh

@@ -9,6 +9,7 @@ source tests/test_fill_submodule.sh
 
 if [ -n "$IS_OSX" ]; then
     source osx_utils.sh
+    MB_PYTHON_OSX_VER=${MB_PYTHON_OSX_VER:-$(macpython_sdk_for_version $MB_PYTHON_VERSION)}
 
     # To work round:
     # https://travis-ci.community/t/syntax-error-unexpected-keyword-rescue-expecting-keyword-end-in-homebrew/5623

tests/test_osx_utils.sh

@@ -9,12 +9,16 @@
 [ "$(pyinst_ext_for_version 3.5)" == pkg ] || ingest
 [ "$(pyinst_ext_for_version 3)" == pkg ] || ingest
 
-macos_ver="${MACPYTHON_DEFAULT_OSX}"
-[ "$(pyinst_fname_for_version 2.7.14)" == "python-2.7.14-macosx${macos_ver}.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 2.7.15)" == "python-2.7.15-macosx${macos_ver}.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 3.6.8)" == "python-3.6.8-macosx${macos_ver}.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 3.7.1)" == "python-3.7.1-macosx${macos_ver}.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 3.8.0)" == "python-3.8.0-macosx${macos_ver}.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 2.7.14)" == "python-2.7.14-macosx10.6.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 2.7.15)" == "python-2.7.15-macosx10.9.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.6.8)" == "python-3.6.8-macosx10.9.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.7.1)" == "python-3.7.1-macosx10.9.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.8.0)" == "python-3.8.0-macosx10.9.pkg" ] || ingest
+
+[ "$(pyinst_fname_for_version 2.7.14 10.6)" == "python-2.7.14-macosx10.6.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 2.7.15 10.6)" == "python-2.7.15-macosx10.6.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.6.8 10.6)" == "python-3.6.8-macosx10.6.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.7.1 10.6)" == "python-3.7.1-macosx10.6.pkg" ] || ingest
 
 [ "$(pyinst_fname_for_version 2.7.15 10.11)" == "python-2.7.15-macosx10.11.pkg" ] || ingest
 [ "$(pyinst_fname_for_version 3.7.1 10.12)" == "python-3.7.1-macosx10.12.pkg" ] || ingest
@@ -40,5 +44,21 @@ macos_ver="${MACPYTHON_DEFAULT_OSX}"
 [ "$(macpython_impl_for_version 3.7.2)" == "cp" ] || ingest
 [ "$(macpython_impl_for_version pypy-5.4)" == "pp" ] || ingest
 
+# Test lookup of available macOS SDK build targets from python version
+[ "$(macpython_sdk_list_for_version 3.8)"    == "10.9" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.7.5)"  == "10.6 10.9" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.7)"    == "10.6 10.9" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.6.5)"  == "10.6 10.9" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.6)"    == "10.6 10.9" ] || ingest
+[ "$(macpython_sdk_list_for_version 3.5)"    == "10.6" ] || ingest
+[ "$(macpython_sdk_list_for_version 2.7)"    == "10.6 10.9" ] || ingest
+[ "$(macpython_sdk_list_for_version 2.7.14)" == "10.6" ] || ingest
+[ "$(macpython_sdk_list_for_version 2.7.15)" == "10.6 10.9" ] || ingest
+
+[ "$(macpython_sdk_for_version 3.8)"    == "10.9" ] || ingest
+[ "$(macpython_sdk_for_version 3.5)"    == "10.6" ] || ingest
+[ "$(macpython_sdk_for_version 2.7)"    == "10.9" ] || ingest
+[ "$(macpython_sdk_for_version 2.7.14)" == "10.6" ] || ingest
+
 # Test pkg-config install
 install_pkg_config

travis_osx_steps.sh

@@ -10,6 +10,8 @@ ENV_VARS_PATH=${ENV_VARS_PATH:-env_vars.sh}
 
 # These load common_utils.sh
 source $MULTIBUILD_DIR/osx_utils.sh
+MB_PYTHON_OSX_VER=${MB_PYTHON_OSX_VER:-$(macpython_sdk_for_version $MB_PYTHON_VERSION)}
+
 if [ -r "$ENV_VARS_PATH" ]; then source "$ENV_VARS_PATH"; fi
 source $MULTIBUILD_DIR/configure_build.sh
 source $MULTIBUILD_DIR/library_builders.sh