Merge pull request #224 from robbuckley/cpython-10.9

Add support for python.org builds targeting osx 10.9

.travis.yml

@@ -41,6 +41,11 @@ matrix:
       env:
         - PYTHON_VERSION=2.7
         - TEST_BUILDS=1
+    - os: osx
+      env:
+        - PYTHON_VERSION=2.7
+        - MB_PYTHON_OSX_VER=10.9
+        - TEST_BUILDS=1
     - os: osx
       language: objective-c
       env:
@@ -64,10 +69,20 @@ matrix:
       env:
         - PYTHON_VERSION=3.6
         - VENV=venv
+    - os: osx
+      env:
+        - PYTHON_VERSION=3.6
+        - VENV=venv
+        - MB_PYTHON_OSX_VER=10.9
+    - os: osx
+      env:
+        - PYTHON_VERSION=3.7
+        - VENV=venv
     - os: osx
       env:
         - PYTHON_VERSION=3.7
         - VENV=venv
+        - MB_PYTHON_OSX_VER=10.9
     - os: osx
       env:
         - PYTHON_VERSION=3.7
@@ -119,6 +134,21 @@ matrix:
         - VENV=venv
     # Default OSX (xcode image) is 10.13 (xcode 9.4.1) as of 2018-08-03
     # See: https://docs.travis-ci.com/user/reference/osx/
+    - os: osx
+      osx_image: xcode10.1
+      env:
+        - PYTHON_VERSION=3.7
+        - TEST_BUILDS=1
+    - os: osx
+      osx_image: xcode10.1
+      env:
+        - PYTHON_VERSION=3.7
+        - MB_PYTHON_OSX_VER=10.9
+        - TEST_BUILDS=1
+    - os: osx
+      osx_image: xcode10
+      env:
+        - PYTHON_VERSION=3.7
     - os: osx
       osx_image: xcode10.1
       env:

README.rst

@@ -8,7 +8,8 @@ wheels on the `AppVeyor <https://ci.appveyor.com/>`_ infrastructure.
 
 The Travis CI scripts are designed to build *and test*:
 
-* Dual architecture macOS wheels;
+* Dual 32/64-bit architecture macOS wheels built for macOS 10.6+;
+* 64-bit macOS wheels built for macOS 10.9+;
 * 64-bit ``manylinux1_x86_64`` wheels, both narrow and wide Unicode builds;
 * 32-bit ``manylinux1_i686`` wheels, both narrow and wide Unicode builds.
 
@@ -56,10 +57,17 @@ functions and variables in earlier scripts:
 
 See ``travis_osx_steps.sh`` to review source order.
 
-The macOS build / test and phase are on the macOS VM started by Travis CI.
+The macOS build / test phases run on the macOS VM started by Travis CI.
 Therefore any environment variable defined in the ``.travis.yml`` or bash
 shell scripts listed above are available for your build and test.
 
+OS X builds may be targetted either at macOS 10.6+
+(dual arch 64 / 32 bit) or macOS 10.9+ (64b only). These depend on the
+corresponding build of python from https://www.python.org/downloads/mac-osx/.
+At the time of writing, 10.9+ / 64 bit builds are supported for Python
+versions 3.6.5 / 2.7.15 and above. If you want to build for an older version
+of Python, you'll have to target 10.6+ / dual arch.
+
 The ``build_wheel`` function builds the wheel, and the ``install_run``
 function installs the wheel and tests it.  Look in ``common_utils.sh`` for
 default definitions of these functions.  See below for more details.
@@ -187,7 +195,11 @@ To use these scripts
 
     language: python
     # The Travis Python version is unrelated to the version we build and test
-    # with.  This is set with the MB_PYTHON_VERSION variable.
+    # with.  This is set with the MB_PYTHON_VERSION variable. For some builds,
+    # For CPython macOS builds only, the minimum supported macOS version and
+    # architectures of any C extensions in the wheel are set with the variable
+    # MB_PYTHON_OSX_VER: 10.9 (64-bit only) or 10.6 (64/32-bit dual arch).
+    # All PyPy macOS builds are 64-bit only.
     python: 3.5
     dist: trusty
     services: docker
@@ -237,6 +249,10 @@ To use these scripts
           language: generic
           env:
             - MB_PYTHON_VERSION=2.7
+        - os: osx
+          env:
+            - MB_PYTHON_VERSION=2.7
+            - MB_PYTHON_OSX_VER=10.9
         - os: osx
           language: generic
           env:
@@ -249,6 +265,11 @@ To use these scripts
           language: generic
           env:
             - MB_PYTHON_VERSION=3.6
+        - os: osx
+          language: generic
+          env:
+            - MB_PYTHON_VERSION=3.7
+            - MB_PYTHON_OSX_VER=10.9
         - os: osx
           language: generic
           env:
@@ -293,6 +314,7 @@ To use these scripts
     install:
         - build_index_wheel $PROJECT_SPEC
 
+
 * Next create a ``config.sh`` for your project, that fills in any steps you
   need to do before building the wheel (such as building required libraries).
   You also need this file to specify how to run your tests::

configure_build.sh

@@ -15,8 +15,16 @@ BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}"
 # Default compilation flags for OSX
 # IS_OSX is defined in common_utils.sh
 if [ -n "$IS_OSX" ]; then
-    # Dual arch build by default
+    source $MULTIBUILD_DIR/osx_utils.sh
+    PLAT=$(macpython_arch_for_version $MB_PYTHON_VERSION)
+    if [[ $PLAT == intel ]]; then
         ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"}
+    elif [[ $PLAT == x86_64 ]]; then
+        ARCH_FLAGS=${ARCH_FLAGS:-"-arch x86_64"}
+    else
+        echo "invalid platform = '$PLAT', supported values are 'intel' or 'x86_64'"
+        exit 1
+    fi
     # Only set CFLAGS, FFLAGS if they are not already defined.  Build functions
     # can override the arch flags by setting CFLAGS, FFLAGS
     export CFLAGS="${CFLAGS:-$ARCH_FLAGS}"

osx_utils.sh

@@ -8,6 +8,8 @@ source $MULTIBUILD_DIR/common_utils.sh
 
 MACPYTHON_URL=https://www.python.org/ftp/python
 MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
+MACPYTHON_DEFAULT_OSX="10.6"
+MB_PYTHON_OSX_VER=${MB_PYTHON_OSX_VER:-$MACPYTHON_DEFAULT_OSX}
 GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
 DOWNLOADS_SDIR=downloads
 WORKING_SDIR=working
@@ -116,25 +118,96 @@ function pyinst_ext_for_version {
 }
 
 function pyinst_fname_for_version {
-    # echo filename for OSX installer file given Python version
+    # echo filename for OSX installer file given Python and minimum
+    # macOS versions
     # Parameters
-    #   $py_version (python version in major.minor.extra format)
+    #   $py_version (Python version in major.minor.extra format)
+    #   $py_osx_ver: {major.minor | not defined}
+    #       if defined, the macOS version that Python is built for, e.g.
+    #       "10.6" or "10.9", if not defined, uses the default
+    #       MACPYTHON_DEFAULT_OSX
+    #       Note: this is the version the Python is built for, and hence
+    #       the min version supported, NOT the version of the current system
     local py_version=$1
+    local py_osx_ver=${2:-$MACPYTHON_DEFAULT_OSX}
     local inst_ext=$(pyinst_ext_for_version $py_version)
-    local osx_ver=10.6
-    echo "python-$py_version-macosx${osx_ver}.$inst_ext"
+    echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}"
 }
 
-function install_macpython {
-    # Install Python and set $PYTHON_EXE to the installed executable
+function get_macpython_arch {
+    # echo arch (e.g. intel or x86_64), extracted from the distutils platform tag
+    # Parameters
+    #   $distutils_plat   PEP425 style platform tag, or if not provided, calls
+    #                       the function get_distutils_platform, provided by
+    #                       common_utils.sh. Fails if this is not a mac platform
+    #
+    # Note: MUST only be called after the version of Python used to build the
+    # target wheel has been installed and is on the path
+    local distutils_plat=${1:-$(get_distutils_platform)}
+    if [[ $distutils_plat =~ macosx-(10\.[0-9]+)-(.*) ]]; then
+        echo ${BASH_REMATCH[2]}
+    else
+        echo "Error parsing macOS distutils platform '$distutils_plat'"
+        exit 1
+    fi
+}
+
+function get_macpython_osx_ver {
+    # echo minimum macOS version (e.g. 10.9) from the distutils platform tag
+    # Parameters
+    #   $distutils_plat   PEP425 style platform tag, or if not provided, calls
+    #                       the function get_distutils_platform, provided by
+    #                       common_utils.sh. Fails if this is not a mac platform
+    #
+    # Note: MUST only be called after the version of Python used to build the
+    # target wheel has been installed and is on the path
+    local distutils_plat=${1:-$(get_distutils_platform)}
+    if [[ $distutils_plat =~ macosx-(10\.[0-9]+)-(.*) ]]; then
+        echo ${BASH_REMATCH[1]}
+    else
+        echo "Error parsing macOS distutils platform '$distutils_plat'"
+        exit 1
+    fi
+}
+
+function macpython_arch_for_version {
+    # echo arch (intel or x86_64) that a version of Python is expected
+    # to be built for
+    # Parameters
+    #   $py_ver     Python version, in the format (major.minor.patch) for
+    #               CPython, or pypy-(major.minor) for PyPy
+    #   $py_osx_ver minimum macOS version the target Python is built for
+    #               (major.minor)
+    local py_ver=$1
+    local py_osx_ver=${2:-$MB_PYTHON_OSX_VER}
+    check_var $1
+    if [[ $(macpython_impl_for_version $py_ver) == "cp" ]]; then
+        if [[ "$py_osx_ver" == "10.6" ]]; then
+            echo "intel"
+        elif [[ "$py_osx_ver" == "10.9" ]]; then
+            echo "x86_64"
+        else
+            echo "Unexpected CPython macOS version: ${py_osx_ver}, supported values: 10.6 and 10.9"
+            exit 1
+        fi
+    else
+        echo "x86_64"
+    fi
+}
+
+function macpython_impl_for_version {
+    # echo Python implementation (cp for CPython, pp for PyPy) given a
+    # suitably formatted version string
     # Parameters:
     #     $version : [implementation-]major[.minor[.patch]]
-    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
+    #         Python implementation, e.g. "3.6" for CPython or
+    #         "pypy-5.4" for PyPy
     local version=$1
+    check_var $1
     if [[ "$version" =~ pypy-([0-9\.]+) ]]; then
-        install_mac_pypy "${BASH_REMATCH[1]}"
+        echo pp
     elif [[ "$version" =~ ([0-9\.]+) ]]; then
-        install_mac_cpython "${BASH_REMATCH[1]}"
+        echo cp
     else
         echo "config error: Issue parsing this implementation in install_python:"
         echo "    version=$version"
@@ -142,15 +215,56 @@ function install_macpython {
     fi
 }
 
+function strip_macpython_ver_prefix {
+    # strip any implementation prefix from a Python version string
+    # Parameters:
+    #     $version : [implementation-]major[.minor[.patch]]
+    #         Python implementation, e.g. "3.6" for CPython or
+    #         "pypy-5.4" for PyPy
+    local version=$1
+    check_var $1
+    if [[ "$version" =~ (pypy-)?([0-9\.]+) ]]; then
+        echo ${BASH_REMATCH[2]}
+    fi
+}
+
+function install_macpython {
+    # Install Python and set $PYTHON_EXE to the installed executable
+    # Parameters:
+    #     $version : [implementation-]major[.minor[.patch]]
+    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
+    #     $py_osx_ver: {major.minor | not defined}
+    #       if defined, the macOS version that CPython is built for, e.g.
+    #       "10.6" or "10.9". Ignored for PyPy
+    local version=$1
+    local py_osx_ver=$2
+    local impl=$(macpython_impl_for_version $version)
+    local stripped_ver=$(strip_macpython_ver_prefix $version)
+    if [[ "$impl" == "pp" ]]; then
+        install_mac_pypy $stripped_ver
+    elif [[ "$impl" == "cp" ]]; then
+        install_mac_cpython $stripped_ver $py_osx_ver
+    else
+        echo "Unexpected Python impl: ${impl}"
+        exit 1
+    fi
+}
+
 function install_mac_cpython {
     # Installs Python.org Python
-    # Parameter $version
+    # Parameters
+    #   $py_version
     #       Version given in major or major.minor or major.minor.micro e.g
     #       "3" or "3.4" or "3.4.1".
-    # sets $PYTHON_EXE variable to python executable
+    #   $py_osx_ver
+    #       {major.minor | not defined}
+    #       if defined, the macOS version that Python is built for, e.g.
+    #        "10.6" or "10.9"
+    # sets $PYTHON_EXE variable to Python executable
     local py_version=$(fill_pyver $1)
+    local py_osx_ver=$2
     local py_stripped=$(strip_ver_suffix $py_version)
-    local py_inst=$(pyinst_fname_for_version $py_version)
+    local py_inst=$(pyinst_fname_for_version $py_version $py_osx_ver)
     local inst_path=$DOWNLOADS_SDIR/$py_inst
     mkdir -p $DOWNLOADS_SDIR
     curl $MACPYTHON_URL/$py_stripped/${py_inst} > $inst_path
@@ -258,6 +372,9 @@ function get_macpython_environment {
     #     $venv_dir : {directory_name|not defined}
     #         If defined - make virtualenv in this directory, set python / pip
     #         commands accordingly
+    #     $py_osx_ver: {major.minor | not defined}
+    #         if defined, the macOS version that Python is built for, e.g.
+    #         "10.6" or "10.9", if not defined, use the version from MB_PYTHON_OSX_VER
     #
     # Installs Python
     # Sets $PYTHON_EXE to path to Python executable
@@ -266,14 +383,14 @@ function get_macpython_environment {
     # Puts directory of $PYTHON_EXE on $PATH
     local version=$1
     local venv_dir=$2
-
+    local py_osx_ver=${3:-$MB_PYTHON_OSX_VER}
 
     if [ "$USE_CCACHE" == "1" ]; then
         activate_ccache
     fi
 
     remove_travis_ve_pip
-    install_macpython $version
+    install_macpython $version $py_osx_ver
     install_pip
 
     if [ -n "$venv_dir" ]; then
@@ -293,17 +410,24 @@ function install_delocate {
 
 function repair_wheelhouse {
     local wheelhouse=$1
-
     install_delocate
     delocate-wheel $wheelhouse/*.whl # copies library dependencies into wheel
     # Add platform tags to label wheels as compatible with OSX 10.9 and
-    # 10.10.  The wheels will be built against Python.org Python, and so will
-    # in fact be compatible with OSX >= 10.6.  pip < 6.0 doesn't realize
-    # this, so, in case users have older pip, add platform tags to specify
-    # compatibility with later OSX.  Not necessary for OSX released well
-    # after pip 6.0.  See:
+    # 10.10.  The wheels are built against Python.org Python, and so will
+    # in fact be compatible with either 10.6+ or 10.9+, depending on the value
+    # of MB_PYTHON_OSX_VER. pip < 6.0 doesn't realize this, so, in case users
+    # try to install have older pip, add platform tags to specify compatibility
+    # with later OSX. Not necessary for OSX released well after pip 6.0.  See:
     # https://github.com/MacPython/wiki/wiki/Spinning-wheels#question-will-pip-give-me-a-broken-wheel
+    local MAC_ARCH=$(get_macpython_arch)
+    if [[ "$MAC_ARCH" == "x86_64" ]]; then
+        delocate-addplat --rm-orig -p macosx_10_10_x86_64 $wheelhouse/*.whl
+    elif [[ "$MAC_ARCH" == "intel" ]]; then
         delocate-addplat --rm-orig -x 10_9 -x 10_10 $wheelhouse/*.whl
+    else
+        echo "Unexpected ARCH='$MAC_ARCH', supported values are 'x86_64' and 'intel'"
+        exit 1
+    fi
 }
 
 function install_pkg_config {

tests/test_multibuild.sh

@@ -8,7 +8,7 @@ source tests/test_common_utils.sh
 source tests/test_fill_submodule.sh
 if [ -n "$IS_OSX" ]; then
     source osx_utils.sh
-    get_macpython_environment $PYTHON_VERSION $VENV
+    get_macpython_environment $PYTHON_VERSION ${VENV:-""} $MB_PYTHON_OSX_VER
     source tests/test_python_install.sh
     source tests/test_fill_pyver.sh
     source tests/test_fill_pypy_ver.sh
@@ -19,6 +19,7 @@ else
 fi
 if [ -n "$TEST_BUILDS" ]; then
     if [ -n "$IS_OSX" ]; then
+        MB_PYTHON_VERSION=${MB_PYTHON_VERSION:-$PYTHON_VERSION}
         source tests/test_library_builders.sh
     elif [ ! -x "$(command -v docker)" ]; then
         echo "Skipping build tests; no docker available"

tests/test_osx_utils.sh

@@ -10,17 +10,35 @@
 [ "$(pyinst_ext_for_version 3.5.0)" == pkg ] || ingest
 [ "$(pyinst_ext_for_version 3.4)" == pkg ] || ingest
 [ "$(pyinst_ext_for_version 3)" == pkg ] || ingest
-[ "$(pyinst_fname_for_version 2.7.8)" == "python-2.7.8-macosx10.6.dmg" ] || ingest
-[ "$(pyinst_fname_for_version 2.7.9)" == "python-2.7.9-macosx10.6.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 3.4.1)" == "python-3.4.1-macosx10.6.dmg" ] || ingest
-[ "$(pyinst_fname_for_version 3.4.2)" == "python-3.4.2-macosx10.6.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 3.6.1)" == "python-3.6.1-macosx10.6.pkg" ] || ingest
+
+[ "$(pyinst_fname_for_version 2.7.14)" == "python-2.7.14-macosx10.6.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 2.7.15)" == "python-2.7.15-macosx10.6.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.6.8)" == "python-3.6.8-macosx10.6.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.7.1)" == "python-3.7.1-macosx10.6.pkg" ] || ingest
+
+[ "$(pyinst_fname_for_version 2.7.15 10.9)" == "python-2.7.15-macosx10.9.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.7.1 10.9)" == "python-3.7.1-macosx10.9.pkg" ] || ingest
 
 # Test utilities for getting Python version versions
 [ "$(get_py_digit)" == "${cpython_version:0:1}" ] || ingest
 [ "$(get_py_mm)" == "${cpython_version:0:3}" ] || ingest
-[ "$(get_py_mm_nodot)" == $(echo "${cpython_version:0:3}" | tr -d .) ] || \
-    ingest
+[ "$(get_py_mm_nodot)" == $(echo "${cpython_version:0:3}" | tr -d .) ] || ingest
+
+# test lookup of arch from Python macOS target build
+[ "$(macpython_arch_for_version 2.7 10.6)" == "intel" ] || ingest
+[ "$(macpython_arch_for_version 2.7 10.9)" == "x86_64" ] || ingest
+[ "$(macpython_arch_for_version pypy-2.7)" == "x86_64" ] || ingest
+
+# test lookup of arch / min macOS versions from installed Python distutils tag
+[ "$(get_macpython_arch macosx-10.6-intel)" == "intel" ] || ingest
+[ "$(get_macpython_arch macosx-10.6-x86_64)" == "x86_64" ] || ingest
+[ "$(get_macpython_osx_ver macosx-10.6-intel)" == "10.6" ] || ingest
+
+# test utilities for extracting version and impl from Python version string
+[ "$(strip_macpython_ver_prefix 3.7.2)" == "3.7.2" ] || ingest
+[ "$(strip_macpython_ver_prefix pypy-5.4)" == "5.4" ] || ingest
+[ "$(macpython_impl_for_version 3.7.2)" == "cp" ] || ingest
+[ "$(macpython_impl_for_version pypy-5.4)" == "pp" ] || ingest
 
 # Test pkg-config install
 install_pkg_config

tests/test_python_install.sh

@@ -22,8 +22,10 @@ then
 fi
 python_mm="${cpython_version:0:1}.${cpython_version:2:1}"
 
-# Remove implementation prefix
+# extract implementation prefix and version
 if [[ "$PYTHON_VERSION" =~ (pypy-)?([0-9\.]+) ]]; then
+    _impl=${BASH_REMATCH[1]:-"cp"}
+    requested_impl=${_impl:0:2}
     requested_version=${BASH_REMATCH[2]}
 else
     ingest "Error parsing PYTHON_VERSION=$PYTHON_VERSION"
@@ -58,3 +60,13 @@ else # not virtualenv
         ingest "Wrong macpython pip '$PIP_CMD'"
     fi
 fi
+
+# check macOS version and arch are as expected
+distutils_plat=$($PYTHON_EXE -c "import distutils.util; print(distutils.util.get_platform())")
+expected_arch=$(macpython_arch_for_version $PYTHON_VERSION)
+if [[ $requested_impl == 'cp' ]]; then
+    expected_tag="macosx-$MB_PYTHON_OSX_VER-$expected_arch"
+else
+    expected_tag="macosx-10.[0-9]+-$expected_arch"
+fi
+[[ $distutils_plat =~ $expected_tag ]] || ingest
\ No newline at end of file

tests/test_supported_wheels.sh

@@ -10,6 +10,15 @@ lpv=$(lex_ver $PYTHON_VERSION)
 if [ $lpv -ge $(lex_ver 3.5) ] || [ $lpv -lt $(lex_ver 3) ]; then
     for whl in wheel==0.31.1 wheel==0.32.0 wheel; do
         $pip_install -U $whl
-        $PYTHON_EXE supported_wheels.py tornado-5.1-cp27-cp27m-macosx_10_6_intel.whl tornado-5.1-cp36-cp36m-macosx_10_13_x86_64.whl texext-0.6.1-cp36-none-any.whl
+        $PYTHON_EXE supported_wheels.py \
+        tornado-5.1-cp27-cp27m-macosx_10_6_intel.whl \
+        tornado-5.1-cp27-cp27m-macosx_10_9_intel.whl \
+        tornado-5.1-cp27-cp27m-macosx_10_9_x86_64.whl \
+        tornado-5.1-cp27-cp27m-macosx_10_13_x86_64.whl \
+        tornado-5.1-cp36-cp36m-macosx_10_6_intel.whl \
+        tornado-5.1-cp36-cp36m-macosx_10_9_intel.whl \
+        tornado-5.1-cp36-cp36m-macosx_10_9_x86_64.whl \
+        tornado-5.1-cp36-cp36m-macosx_10_13_x86_64.whl \
+        texext-0.6.1-cp36-none-any.whl
     done
 fi