osx_utils.sh 12.8 KB
Newer Older
Matthew Brett's avatar
Matthew Brett committed
1
2
3
4
#!/bin/bash
# Use with ``source osx_utils.sh``
set -e

5
6
7
8
# Get our own location on this filesystem, load common utils
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
source $MULTIBUILD_DIR/common_utils.sh

Matthew Brett's avatar
Matthew Brett committed
9
10
MACPYTHON_URL=https://www.python.org/ftp/python
MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
11
MACPYTHON_DEFAULT_OSX="10.6"
12
MB_PYTHON_OSX_VER=${MB_PYTHON_OSX_VER:-$MACPYTHON_DEFAULT_OSX}
Matthew Brett's avatar
Matthew Brett committed
13
14
15
16
GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
DOWNLOADS_SDIR=downloads
WORKING_SDIR=working

Andrew Murray's avatar
Andrew Murray committed
17
# As of 20 October 2018 - latest Python of each version with binary download
Matthew Brett's avatar
Matthew Brett committed
18
# available.
19
# See: https://www.python.org/downloads/mac-osx/
Andrew Murray's avatar
Andrew Murray committed
20
LATEST_2p7=2.7.15
Matthew Brett's avatar
Matthew Brett committed
21
LATEST_3p4=3.4.4
Andrew Murray's avatar
Andrew Murray committed
22
LATEST_3p5=3.5.4
Andrew Murray's avatar
Andrew Murray committed
23
24
LATEST_3p6=3.6.7
LATEST_3p7=3.7.1
Matthew Brett's avatar
Matthew Brett committed
25

Matthew Brett's avatar
Matthew Brett committed
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

function check_python {
    if [ -z "$PYTHON_EXE" ]; then
        echo "PYTHON_EXE variable not defined"
        exit 1
    fi
}

function check_pip {
    if [ -z "$PIP_CMD" ]; then
        echo "PIP_CMD variable not defined"
        exit 1
    fi
}

function check_var {
    if [ -z "$1" ]; then
        echo "required variable not defined"
        exit 1
    fi
}

function get_py_digit {
    check_python
    $PYTHON_EXE -c "import sys; print(sys.version_info[0])"
}

function get_py_mm {
    check_python
    $PYTHON_EXE -c "import sys; print('{0}.{1}'.format(*sys.version_info[0:2]))"
}

function get_py_mm_nodot {
    check_python
    $PYTHON_EXE -c "import sys; print('{0}{1}'.format(*sys.version_info[0:2]))"
}

function get_py_prefix {
    check_python
    $PYTHON_EXE -c "import sys; print(sys.prefix)"
}

function fill_pyver {
    # Convert major or major.minor format to major.minor.micro
    #
    # Hence:
    # 2 -> 2.7.11  (depending on LATEST_2p7 value)
    # 2.7 -> 2.7.11  (depending on LATEST_2p7 value)
    local ver=$1
    check_var $ver
    if [[ $ver =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then
        # Major.minor.micro format already
        echo $ver
    elif [ $ver == 2 ] || [ $ver == "2.7" ]; then
        echo $LATEST_2p7
Andrew Murray's avatar
Andrew Murray committed
81
82
83
    elif [ $ver == 3 ] || [ $ver == "3.7" ]; then
        echo $LATEST_3p7
    elif [ $ver == "3.6" ]; then
Matthew Brett's avatar
Matthew Brett committed
84
85
        echo $LATEST_3p6
    elif [ $ver == "3.5" ]; then
Matthew Brett's avatar
Matthew Brett committed
86
87
88
89
        echo $LATEST_3p5
    elif [ $ver == "3.4" ]; then
        echo $LATEST_3p4
    else
90
        echo "Can't fill version $ver" 1>&2
Matthew Brett's avatar
Matthew Brett committed
91
92
93
94
95
96
97
98
99
100
101
102
        exit 1
    fi
}

function pyinst_ext_for_version {
    # echo "pkg" or "dmg" depending on the passed Python version
    # Parameters
    #   $py_version (python version in major.minor.extra format)
    #
    # Earlier Python installers are .dmg, later are .pkg.
    local py_version=$1
    check_var $py_version
103
    py_version=$(fill_pyver $py_version)
Matthew Brett's avatar
Matthew Brett committed
104
105
    local py_0=${py_version:0:1}
    if [ $py_0 -eq 2 ]; then
106
        if [ "$(lex_ver $py_version)" -ge "$(lex_ver 2.7.9)" ]; then
Matthew Brett's avatar
Matthew Brett committed
107
108
109
110
111
            echo "pkg"
        else
            echo "dmg"
        fi
    elif [ $py_0 -ge 3 ]; then
112
        if [ "$(lex_ver $py_version)" -ge "$(lex_ver 3.4.2)" ]; then
Matthew Brett's avatar
Matthew Brett committed
113
114
115
116
117
118
119
            echo "pkg"
        else
            echo "dmg"
        fi
    fi
}

120
function pyinst_fname_for_version {
121
122
    # echo filename for OSX installer file given Python and minimum
    # macosx versions
123
124
    # Parameters
    #   $py_version (python version in major.minor.extra format)
125
126
    #   $py_osx_ver: {major.minor | not defined}
    #       if defined, the macosx version that python is built for, e.g.
127
    #       "10.6" or "10.9", if not defined, uses the default MACPYTHON_DEFAULT_OSX
128
129
    #       Note: this is the version the Python is built for, and hence
    #       the min version supported, NOT the version of the current system
130
    local py_version=$1
131
    local py_osx_ver=${2:-$MACPYTHON_DEFAULT_OSX}
132
    local inst_ext=$(pyinst_ext_for_version $py_version)
133
    echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}"
134
135
}

136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
function get_macpython_arch {
    # get arch (e.g. intel or x86_64) from the disutils platform tag
    distutils_plat=${1:-$(get_distutils_platform)}
    if [[ $distutils_plat =~ macosx-(10\.[0-9]+)-(.*) ]]; then
        echo ${BASH_REMATCH[2]}
    else
        echo "Error parsing distutils platform '$distutils_plat'"
        exit 1
    fi
}

function get_macpython_osx_ver {
    # get minimum macOS version (e.g. 10.9) from the disutils platform tag
    distutils_plat=${1:-$(get_distutils_platform)}
    if [[ $distutils_plat =~ macosx-(10\.[0-9]+)-(.*) ]]; then
        echo ${BASH_REMATCH[1]}
    else
        echo "Error parsing distutils platform '$distutils_plat'"
        exit 1
    fi
}
157

158
159
160
function mac_cpython_arch_for_osx_ver {
    # echo arch (intel or x86_64) for cpython python.org builds targetted for
    # the given minimum macOS version
161
    # Parameters
162
    #   $py_osx_ver (major.minor | not defined}
163
164
165
166
    #       the macosx version that python is built for, e.g.
    #       "10.6" or "10.9", or MB_PYTHON_OSX_VER if undefined
    #
    py_osx_ver=${1:-$MB_PYTHON_OSX_VER}
Rob Buckley's avatar
Rob Buckley committed
167
    if [[ "$py_osx_ver" == "10.6" ]]; then
168
        echo "intel"
Rob Buckley's avatar
Rob Buckley committed
169
    elif [[ "$py_osx_ver" == "10.9" ]]; then
Rob Buckley's avatar
Rob Buckley committed
170
        echo "x86_64"
171
    else
Rob Buckley's avatar
Rob Buckley committed
172
        echo "Unexpected python osx version: ${py_osx_ver}, supported values: 10.6 and 10.9"
173
174
175
176
        exit 1
    fi
}

177
178
179
180
181
function install_macpython {
    # Install Python and set $PYTHON_EXE to the installed executable
    # Parameters:
    #     $version : [implementation-]major[.minor[.patch]]
    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
182
183
184
    #     $py_osx_ver: {major.minor | not defined}
    #       if defined, the macosx version that cpython is built for, e.g.
    #       "10.6" or "10.9". Ignored for pypy
185
    local version=$1
186
    local py_osx_ver=$2
187
    if [[ "$version" =~ pypy-([0-9\.]+) ]]; then
188
        install_mac_pypy "${BASH_REMATCH[1]}"
189
    elif [[ "$version" =~ ([0-9\.]+) ]]; then
190
        install_mac_cpython "${BASH_REMATCH[1]}" $py_osx_ver
Kyle Stewart's avatar
Kyle Stewart committed
191
    else
Matt Bachmann's avatar
Matt Bachmann committed
192
        echo "config error: Issue parsing this implementation in install_python:"
193
        echo "    version=$version"
Kyle Stewart's avatar
Kyle Stewart committed
194
195
196
197
        exit 1
    fi
}

198
function install_mac_cpython {
Matthew Brett's avatar
Matthew Brett committed
199
    # Installs Python.org Python
200
201
202
203
    # Parameters
    #   $py_version
    #       Version given in major or major.minor or major.minor.micro e.g
    #       "3" or "3.4" or "3.4.1".
204
    #   $py_osx_ver
205
206
207
    #       {major.minor | not defined}
    #       if defined, the macosx version that python is built for, e.g.
    #        "10.6" or "10.9"
Matthew Brett's avatar
Matthew Brett committed
208
209
    # sets $PYTHON_EXE variable to python executable
    local py_version=$(fill_pyver $1)
Rob Buckley's avatar
Rob Buckley committed
210
    local py_osx_ver=$2
211
    local py_stripped=$(strip_ver_suffix $py_version)
212
    local py_inst=$(pyinst_fname_for_version $py_version $py_osx_ver)
Matthew Brett's avatar
Matthew Brett committed
213
214
    local inst_path=$DOWNLOADS_SDIR/$py_inst
    mkdir -p $DOWNLOADS_SDIR
215
    curl $MACPYTHON_URL/$py_stripped/${py_inst} > $inst_path
216
    if [ "${py_inst: -3}" == "dmg" ]; then
Matthew Brett's avatar
Matthew Brett committed
217
218
219
220
221
222
        hdiutil attach $inst_path -mountpoint /Volumes/Python
        inst_path=/Volumes/Python/Python.mpkg
    fi
    sudo installer -pkg $inst_path -target /
    local py_mm=${py_version:0:3}
    PYTHON_EXE=$MACPYTHON_PY_PREFIX/$py_mm/bin/python$py_mm
223
224
225
226
227
    # Install certificates for Python 3.6
    local inst_cmd="/Applications/Python ${py_mm}/Install Certificates.command"
    if [ -e "$inst_cmd" ]; then
        sh "$inst_cmd"
    fi
Matthew Brett's avatar
Matthew Brett committed
228
229
}

Kyle Stewart's avatar
Kyle Stewart committed
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
function install_mac_pypy {
    # Installs pypy.org PyPy
    # Parameter $version
    # Version given in major or major.minor or major.minor.micro e.g
    # "3" or "3.4" or "3.4.1".
    # sets $PYTHON_EXE variable to python executable
    local py_version=$(fill_pypy_ver $1)
    local py_build=$(get_pypy_build_prefix $py_version)$py_version-osx64
    local py_zip=$py_build.tar.bz2
    local zip_path=$DOWNLOADS_SDIR/$py_zip
    mkdir -p $DOWNLOADS_SDIR
    wget -nv $PYPY_URL/${py_zip} -P $DOWNLOADS_SDIR
    untar $zip_path
    PYTHON_EXE=$(realpath $py_build/bin/pypy)
}

Matthew Brett's avatar
Matthew Brett committed
246
247
248
249
250
251
252
253
function install_pip {
    # Generic install pip
    # Gets needed version from version implied by $PYTHON_EXE
    # Installs pip into python given by $PYTHON_EXE
    # Assumes pip will be installed into same directory as $PYTHON_EXE
    check_python
    mkdir -p $DOWNLOADS_SDIR
    local py_mm=`get_py_mm`
254
    local get_pip_path=$DOWNLOADS_SDIR/get-pip.py
Hugo's avatar
Hugo committed
255
    curl $GET_PIP_URL > $get_pip_path
256
257
    # Travis VMS now install pip for system python by default - force install
    # even if installed already.
258
    sudo $PYTHON_EXE $get_pip_path --ignore-installed $pip_args
259
260
261
262
263
264
    PIP_CMD="sudo $(dirname $PYTHON_EXE)/pip$py_mm"
    # Append pip_args if present (avoiding trailing space cf using variable
    # above).
    if [ -n "$pip_args" ]; then
        PIP_CMD="$PIP_CMD $pip_args"
    fi
Matthew Brett's avatar
Matthew Brett committed
265
266
267
268
269
270
271
272
273
274
275
}

function install_virtualenv {
    # Generic install of virtualenv
    # Installs virtualenv into python given by $PYTHON_EXE
    # Assumes virtualenv will be installed into same directory as $PYTHON_EXE
    check_pip
    # Travis VMS install virtualenv for system python by default - force
    # install even if installed already
    $PIP_CMD install virtualenv --ignore-installed
    check_python
276
    VIRTUALENV_CMD="$(dirname $PYTHON_EXE)/virtualenv"
Matthew Brett's avatar
Matthew Brett committed
277
278
279
280
281
282
283
284
285
286
287
288
289
}

function make_workon_venv {
    # Make a virtualenv in given directory ('venv' default)
    # Set $PYTHON_EXE, $PIP_CMD to virtualenv versions
    # Parameter $venv_dir
    #    directory for virtualenv
    local venv_dir=$1
    if [ -z "$venv_dir" ]; then
        venv_dir="venv"
    fi
    venv_dir=`abspath $venv_dir`
    check_python
xoviat's avatar
xoviat committed
290
    $PYTHON_EXE -m virtualenv $venv_dir
Matthew Brett's avatar
Matthew Brett committed
291
292
293
294
295
296
    PYTHON_EXE=$venv_dir/bin/python
    PIP_CMD=$venv_dir/bin/pip
}

function remove_travis_ve_pip {
    # Remove travis installs of virtualenv and pip
xoviat's avatar
xoviat committed
297
298
    # FIXME: What if virtualenv is installed but pip is not?
    if [ "$(sudo which virtualenv)" == /usr/local/bin/virtualenv ] && [ "$(sudo which pip)" == /usr/local/bin/pip ]; then
Matthew Brett's avatar
Matthew Brett committed
299
300
301
302
303
304
305
306
        sudo pip uninstall -y virtualenv;
    fi
    if [ "$(sudo which pip)" == /usr/local/bin/pip ]; then
        sudo pip uninstall -y pip;
    fi
}

function set_py_vars {
307
    # Used by terryfy project; left here for back-compatibility
Matthew Brett's avatar
Matthew Brett committed
308
309
310
311
    export PATH="`dirname $PYTHON_EXE`:$PATH"
    export PYTHON_EXE PIP_CMD
}

312
function get_macpython_environment {
Matthew Brett's avatar
Matthew Brett committed
313
314
    # Set up MacPython environment
    # Parameters:
315
316
    #     $version : [implementation-]major[.minor[.patch]]
    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
Matthew Brett's avatar
Matthew Brett committed
317
318
319
    #     $venv_dir : {directory_name|not defined}
    #         If defined - make virtualenv in this directory, set python / pip
    #         commands accordingly
320
321
322
323
    #     $py_osx_ver: {major.minor | not defined}
    #         if defined, the macosx version that python is built for, e.g.
    #         "10.6" or "10.9", if not defined, use the version from the
    #         environment variable MB_PYTHON_OSX_VER
Matthew Brett's avatar
Matthew Brett committed
324
325
326
327
328
329
    #
    # Installs Python
    # Sets $PYTHON_EXE to path to Python executable
    # Sets $PIP_CMD to full command for pip (including sudo if necessary)
    # If $venv_dir defined, Sets $VIRTUALENV_CMD to virtualenv executable
    # Puts directory of $PYTHON_EXE on $PATH
330
    local version=$1
331
332
    local venv_dir=$2
    local py_osx_ver=${3:-$MB_PYTHON_OSX_VER}
xoviat's avatar
xoviat committed
333
334
335
336

    if [ "$USE_CCACHE" == "1" ]; then
        activate_ccache
    fi
337

Matthew Brett's avatar
Matthew Brett committed
338
    remove_travis_ve_pip
339
    install_macpython $version $py_osx_ver
Matthew Brett's avatar
Matthew Brett committed
340
    install_pip
xoviat's avatar
xoviat committed
341

Matthew Brett's avatar
Matthew Brett committed
342
343
344
    if [ -n "$venv_dir" ]; then
        install_virtualenv
        make_workon_venv $venv_dir
345
346
347
        source $venv_dir/bin/activate
    else
        export PATH="`dirname $PYTHON_EXE`:$PATH"
Matthew Brett's avatar
Matthew Brett committed
348
    fi
349
    export PYTHON_EXE PIP_CMD
Matthew Brett's avatar
Matthew Brett committed
350
}
351

352
function install_delocate {
353
354
    check_pip
    $PIP_CMD install delocate
355
356
357
358
359
}

function repair_wheelhouse {
    local wheelhouse=$1
    install_delocate
360
    delocate-wheel $wheelhouse/*.whl # copies library dependencies into wheel
361
    # Add platform tags to label wheels as compatible with OSX 10.9 and
362
363
364
    # 10.10.  The wheels are built against Python.org Python, and so will
    # in fact be compatible with either 10.6+ or 10.9+, depending on the value
    # of MB_PYTHON_OSX_VER. pip < 6.0 doesn't realize this, so, in case users
365
366
    # try to install have older pip, add platform tags to specify compatibility
    # with later OSX. Not necessary for OSX released well after pip 6.0.  See:
367
    # https://github.com/MacPython/wiki/wiki/Spinning-wheels#question-will-pip-give-me-a-broken-wheel
368
369
    local MAC_ARCH=$(get_macpython_arch)
    if [[ "$MAC_ARCH" == "x86_64" ]]; then
370
        delocate-addplat --rm-orig -p macosx_10_10_x86_64 $wheelhouse/*.whl
371
    elif [[ "$MAC_ARCH" == "intel" ]]; then
372
        delocate-addplat --rm-orig -x 10_9 -x 10_10 $wheelhouse/*.whl
373
    else
374
        echo "Unexpected ARCH='$MAC_ARCH', supported values are 'x86_64' and 'intel'"
375
376
        exit 1
    fi
377
}
378
379
380
381
382
383
384

function install_pkg_config {
    # Install pkg-config avoiding error from homebrew
    # See :
    # https://github.com/matthew-brett/multibuild/issues/24#issue-221951587
    command -v pkg-config > /dev/null 2>&1 || brew install pkg-config
}
xoviat's avatar
xoviat committed
385
386

function activate_ccache {
xoviat's avatar
xoviat committed
387

xoviat's avatar
xoviat committed
388
389
390
391
392
    brew install ccache
    export PATH=/usr/local/opt/ccache/libexec:$PATH
    export CCACHE_CPP2=1

    # Prove to the developer that ccache is activated
xoviat's avatar
xoviat committed
393
    echo "Using C compiler: $(which clang)"
394
}