Merge pull request #275 from matthew-brett/default-installer-3.8

MRG  default macOS ver, brew fix, refactor

This PR does three things:

* Sets the default Python installer / MB_PYTHON_OSX_VER for Python 3.8, so
  users don't have do this in their travis.yml files.
* Adds a brew update fix to work round a syntax error on Homebrew on macOS.
* Refactors our .travis.yml to use MB_PYTHON_VERSION, rather than
  PYTHON_VERSION.  This matches the way that users will write their travis.yml
  files.

.travis.yml

@@ -6,6 +6,11 @@ cache:
   directories:
    - $HOME/.ccache
 
+env:
+    global:
+        # Always set Python version
+        - MB_PYTHON_VERSION=3.7
+
 matrix:
   include:
     - os: linux
@@ -36,177 +41,173 @@ matrix:
     # OSX builds
     - os: osx
       env:
-        - PYTHON_VERSION=2.7
+        - MB_PYTHON_VERSION=2.7
         - TEST_BUILDS=1
     - os: osx
       env:
-        - PYTHON_VERSION=2.7
+        - MB_PYTHON_VERSION=2.7
         - MB_PYTHON_OSX_VER=10.9
         - TEST_BUILDS=1
     - os: osx
       language: objective-c
       env:
-        - PYTHON_VERSION=2.7
+        - MB_PYTHON_VERSION=2.7
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=3.6
+        - MB_PYTHON_VERSION=3.6
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=3.6
+        - MB_PYTHON_VERSION=3.6
         - VENV=venv
         - MB_PYTHON_OSX_VER=10.9
     - os: osx
       env:
-        - PYTHON_VERSION=3.7
+        - MB_PYTHON_VERSION=3.7
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=3.7
+        - MB_PYTHON_VERSION=3.7
         - VENV=venv
         - MB_PYTHON_OSX_VER=10.9
     - os: osx
       env:
-        - PYTHON_VERSION=3.7
+        - MB_PYTHON_VERSION=3.7
         - VENV=venv
         - USE_CCACHE=1
     - os: osx
       env:
-        - PYTHON_VERSION=3.8
+        - MB_PYTHON_VERSION=3.8
         - VENV=venv
-        - MB_PYTHON_OSX_VER=10.9
-        - MB_ML_VER=2010
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-4.0
+        - MB_PYTHON_VERSION=pypy-4.0
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-5.0
+        - MB_PYTHON_VERSION=pypy-5.0
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-5.1
+        - MB_PYTHON_VERSION=pypy-5.1
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-5.3
+        - MB_PYTHON_VERSION=pypy-5.3
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-5.4
+        - MB_PYTHON_VERSION=pypy-5.4
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-5.6
+        - MB_PYTHON_VERSION=pypy-5.6
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-5.7
+        - MB_PYTHON_VERSION=pypy-5.7
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-5.8
+        - MB_PYTHON_VERSION=pypy-5.8
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-5.9
+        - MB_PYTHON_VERSION=pypy-5.9
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-6.0
+        - MB_PYTHON_VERSION=pypy-6.0
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-7.0
+        - MB_PYTHON_VERSION=pypy-7.0
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-7.1
+        - MB_PYTHON_VERSION=pypy-7.1
         - VENV=venv
     - os: osx
       env:
-        - PYTHON_VERSION=pypy-7.2
+        - MB_PYTHON_VERSION=pypy-7.2
         - VENV=venv
     # Default OSX (xcode image) is 10.13 (xcode 9.4.1) as of 2018-08-03
     # See: https://docs.travis-ci.com/user/reference/osx/
     - os: osx
       osx_image: xcode10.1
       env:
-        - PYTHON_VERSION=3.7
+        - MB_PYTHON_VERSION=3.7
         - TEST_BUILDS=1
     - os: osx
       osx_image: xcode10.1
       env:
-        - PYTHON_VERSION=3.7
+        - MB_PYTHON_VERSION=3.7
         - MB_PYTHON_OSX_VER=10.9
         - TEST_BUILDS=1
     - os: osx
       osx_image: xcode10
       env:
-        - PYTHON_VERSION=3.7
+        - MB_PYTHON_VERSION=3.7
     - os: osx
       osx_image: xcode10
       env:
-        - PYTHON_VERSION=3.8
-        - MB_PYTHON_OSX_VER=10.9
-        - MB_ML_VER=2010
+        - MB_PYTHON_VERSION=3.8
     - os: osx
       osx_image: xcode10.1
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode10
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode9.4
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode9.3
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode9.2
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode9.1
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode9
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode8.3
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode8.2
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode8.1
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode8
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
     - os: osx
       osx_image: xcode7.3
       env:
-        - PYTHON_VERSION=3.5
+        - MB_PYTHON_VERSION=3.5
 
 script:
   - export ENV_VARS_PATH="tests/env_vars.sh"

osx_utils.sh

@@ -9,6 +9,10 @@ source $MULTIBUILD_DIR/common_utils.sh
 MACPYTHON_URL=https://www.python.org/ftp/python
 MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
 MACPYTHON_DEFAULT_OSX="10.6"
+if [ "$(lex_ver $MB_PYTHON_VERSION)" -ge "$(lex_ver 3.8)" ]; then
+    # At 3.8 Python.org dropped the 10.6 installer.
+    MACPYTHON_DEFAULT_OSX="10.9"
+fi
 MB_PYTHON_OSX_VER=${MB_PYTHON_OSX_VER:-$MACPYTHON_DEFAULT_OSX}
 GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
 DOWNLOADS_SDIR=downloads

tests/test_multibuild.sh

@@ -6,9 +6,15 @@ source tests/utils.sh
 
 source tests/test_common_utils.sh
 source tests/test_fill_submodule.sh
+
 if [ -n "$IS_OSX" ]; then
     source osx_utils.sh
-    get_macpython_environment $PYTHON_VERSION ${VENV:-""} $MB_PYTHON_OSX_VER
+
+    # To work round:
+    # https://travis-ci.community/t/syntax-error-unexpected-keyword-rescue-expecting-keyword-end-in-homebrew/5623
+    brew update
+
+    get_macpython_environment $MB_PYTHON_VERSION ${VENV:-""} $MB_PYTHON_OSX_VER
     source tests/test_python_install.sh
     source tests/test_fill_pyver.sh
     source tests/test_fill_pypy_ver.sh
@@ -19,7 +25,9 @@ else
 fi
 if [ -n "$TEST_BUILDS" ]; then
     if [ -n "$IS_OSX" ]; then
-        MB_PYTHON_VERSION=${MB_PYTHON_VERSION:-$PYTHON_VERSION}
+        # This checked in test_library_builders.
+        # Will be set automatically by docker call in build_multilinux below.
+        PYTHON_VERSION=${MB_PYTHON_VERSION}
         source tests/test_library_builders.sh
     elif [ ! -x "$(command -v docker)" ]; then
         echo "Skipping build tests; no docker available"

tests/test_osx_utils.sh

@@ -9,13 +9,15 @@
 [ "$(pyinst_ext_for_version 3.5)" == pkg ] || ingest
 [ "$(pyinst_ext_for_version 3)" == pkg ] || ingest
 
-[ "$(pyinst_fname_for_version 2.7.14)" == "python-2.7.14-macosx10.6.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 2.7.15)" == "python-2.7.15-macosx10.6.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 3.6.8)" == "python-3.6.8-macosx10.6.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 3.7.1)" == "python-3.7.1-macosx10.6.pkg" ] || ingest
-
-[ "$(pyinst_fname_for_version 2.7.15 10.9)" == "python-2.7.15-macosx10.9.pkg" ] || ingest
-[ "$(pyinst_fname_for_version 3.7.1 10.9)" == "python-3.7.1-macosx10.9.pkg" ] || ingest
+macos_ver="${MACPYTHON_DEFAULT_OSX}"
+[ "$(pyinst_fname_for_version 2.7.14)" == "python-2.7.14-macosx${macos_ver}.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 2.7.15)" == "python-2.7.15-macosx${macos_ver}.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.6.8)" == "python-3.6.8-macosx${macos_ver}.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.7.1)" == "python-3.7.1-macosx${macos_ver}.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.8.0)" == "python-3.8.0-macosx${macos_ver}.pkg" ] || ingest
+
+[ "$(pyinst_fname_for_version 2.7.15 10.11)" == "python-2.7.15-macosx10.11.pkg" ] || ingest
+[ "$(pyinst_fname_for_version 3.7.1 10.12)" == "python-3.7.1-macosx10.12.pkg" ] || ingest
 
 # Test utilities for getting Python version versions
 [ "$(get_py_digit)" == "${cpython_version:0:1}" ] || ingest

tests/test_python_install.sh

@@ -23,12 +23,12 @@ fi
 python_mm="${cpython_version:0:1}.${cpython_version:2:1}"
 
 # extract implementation prefix and version
-if [[ "$PYTHON_VERSION" =~ (pypy-)?([0-9\.]+) ]]; then
+if [[ "$MB_PYTHON_VERSION" =~ (pypy-)?([0-9\.]+) ]]; then
     _impl=${BASH_REMATCH[1]:-"cp"}
     requested_impl=${_impl:0:2}
     requested_version=${BASH_REMATCH[2]}
 else
-    ingest "Error parsing PYTHON_VERSION=$PYTHON_VERSION"
+    ingest "Error parsing MB_PYTHON_VERSION=$MB_PYTHON_VERSION"
 fi
 
 # simple regex match, a 2.7 pattern will match 2.7.11, but not 2
@@ -63,7 +63,7 @@ fi
 
 # check macOS version and arch are as expected
 distutils_plat=$($PYTHON_EXE -c "import distutils.util; print(distutils.util.get_platform())")
-expected_arch=$(macpython_arch_for_version $PYTHON_VERSION)
+expected_arch=$(macpython_arch_for_version $MB_PYTHON_VERSION)
 if [[ $requested_impl == 'cp' ]]; then
     expected_tag="macosx-$MB_PYTHON_OSX_VER-$expected_arch"
 else

tests/test_supported_wheels.sh

@@ -6,7 +6,7 @@ else
     pip_install="$PIP_CMD install"
 fi
 # Current wheel versions not available for older Pythons
-lpv=$(lex_ver $PYTHON_VERSION)
+lpv=$(lex_ver $MB_PYTHON_VERSION)
 if [ $lpv -ge $(lex_ver 3.5) ] || [ $lpv -lt $(lex_ver 3) ]; then
     for whl in wheel==0.31.1 wheel==0.32.0 wheel; do
         $pip_install -U $whl

travis_osx_steps.sh

@@ -20,6 +20,11 @@ source $MULTIBUILD_DIR/library_builders.sh
 function before_install {
     export CC=clang
     export CXX=clang++
+
+    # To work round:
+    # https://travis-ci.community/t/syntax-error-unexpected-keyword-rescue-expecting-keyword-end-in-homebrew/5623
+    brew update
+
     get_macpython_environment $MB_PYTHON_VERSION venv
     source venv/bin/activate
     pip install --upgrade pip wheel