osx_utils.sh 10.9 KB
Newer Older
Matthew Brett's avatar
Matthew Brett committed
1
2
3
4
#!/bin/bash
# Use with ``source osx_utils.sh``
set -e

5
6
7
8
# Get our own location on this filesystem, load common utils
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
source $MULTIBUILD_DIR/common_utils.sh

Matthew Brett's avatar
Matthew Brett committed
9
10
11
12
13
14
MACPYTHON_URL=https://www.python.org/ftp/python
MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
DOWNLOADS_SDIR=downloads
WORKING_SDIR=working

Andrew Murray's avatar
Andrew Murray committed
15
# As of 15 October 2017 - latest Python of each version with binary download
Matthew Brett's avatar
Matthew Brett committed
16
# available.
17
# See: https://www.python.org/downloads/mac-osx/
Andrew Murray's avatar
Andrew Murray committed
18
LATEST_2p7=2.7.14
Matthew Brett's avatar
Matthew Brett committed
19
20
21
22
LATEST_2p6=2.6.6
LATEST_3p2=3.2.5
LATEST_3p3=3.3.5
LATEST_3p4=3.4.4
Andrew Murray's avatar
Andrew Murray committed
23
24
LATEST_3p5=3.5.4
LATEST_3p6=3.6.3
Matthew Brett's avatar
Matthew Brett committed
25

Matthew Brett's avatar
Matthew Brett committed
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

function check_python {
    if [ -z "$PYTHON_EXE" ]; then
        echo "PYTHON_EXE variable not defined"
        exit 1
    fi
}

function check_pip {
    if [ -z "$PIP_CMD" ]; then
        echo "PIP_CMD variable not defined"
        exit 1
    fi
}

function check_var {
    if [ -z "$1" ]; then
        echo "required variable not defined"
        exit 1
    fi
}

function get_py_digit {
    check_python
    $PYTHON_EXE -c "import sys; print(sys.version_info[0])"
}

function get_py_mm {
    check_python
    $PYTHON_EXE -c "import sys; print('{0}.{1}'.format(*sys.version_info[0:2]))"
}

function get_py_mm_nodot {
    check_python
    $PYTHON_EXE -c "import sys; print('{0}{1}'.format(*sys.version_info[0:2]))"
}

function get_py_prefix {
    check_python
    $PYTHON_EXE -c "import sys; print(sys.prefix)"
}

function fill_pyver {
    # Convert major or major.minor format to major.minor.micro
    #
    # Hence:
    # 2 -> 2.7.11  (depending on LATEST_2p7 value)
    # 2.7 -> 2.7.11  (depending on LATEST_2p7 value)
    local ver=$1
    check_var $ver
    if [[ $ver =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then
        # Major.minor.micro format already
        echo $ver
    elif [ $ver == 2 ] || [ $ver == "2.7" ]; then
        echo $LATEST_2p7
    elif [ $ver == "2.6" ]; then
        echo $LATEST_2p6
Matthew Brett's avatar
Matthew Brett committed
83
84
85
    elif [ $ver == 3 ] || [ $ver == "3.6" ]; then
        echo $LATEST_3p6
    elif [ $ver == "3.5" ]; then
Matthew Brett's avatar
Matthew Brett committed
86
87
88
89
90
91
92
93
        echo $LATEST_3p5
    elif [ $ver == "3.4" ]; then
        echo $LATEST_3p4
    elif [ $ver == "3.3" ]; then
        echo $LATEST_3p3
    elif [ $ver == "3.2" ]; then
        echo $LATEST_3p2
    else
94
        echo "Can't fill version $ver" 1>&2
Matthew Brett's avatar
Matthew Brett committed
95
96
97
98
99
100
101
102
103
104
105
106
        exit 1
    fi
}

function pyinst_ext_for_version {
    # echo "pkg" or "dmg" depending on the passed Python version
    # Parameters
    #   $py_version (python version in major.minor.extra format)
    #
    # Earlier Python installers are .dmg, later are .pkg.
    local py_version=$1
    check_var $py_version
107
    py_version=$(fill_pyver $py_version)
Matthew Brett's avatar
Matthew Brett committed
108
109
    local py_0=${py_version:0:1}
    if [ $py_0 -eq 2 ]; then
110
        if [ "$(lex_ver $py_version)" -ge "$(lex_ver 2.7.9)" ]; then
Matthew Brett's avatar
Matthew Brett committed
111
112
113
114
115
            echo "pkg"
        else
            echo "dmg"
        fi
    elif [ $py_0 -ge 3 ]; then
116
        if [ "$(lex_ver $py_version)" -ge "$(lex_ver 3.4.2)" ]; then
Matthew Brett's avatar
Matthew Brett committed
117
118
119
120
121
122
123
            echo "pkg"
        else
            echo "dmg"
        fi
    fi
}

124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
function pyinst_fname_for_version {
    # echo filename for OSX installer file given Python version
    # Parameters
    #   $py_version (python version in major.minor.extra format)
    local py_version=$1
    local inst_ext=$(pyinst_ext_for_version $py_version)
    # Python 2.6 has OSX 10.3 suffix
    if [ "$(lex_ver $py_version)" -le "$(lex_ver 2.6.6)" ]; then
        local osx_ver=10.3
    else
        local osx_ver=10.6
    fi
    echo "python-$py_version-macosx${osx_ver}.$inst_ext"
}

139
140
141
142
143
144
145
function install_macpython {
    # Install Python and set $PYTHON_EXE to the installed executable
    # Parameters:
    #     $version : [implementation-]major[.minor[.patch]]
    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
    local version=$1
    if [[ "$version" =~ pypy-([0-9\.]+) ]]; then
146
        install_mac_pypy "${BASH_REMATCH[1]}"
147
148
    elif [[ "$version" =~ ([0-9\.]+) ]]; then
        install_mac_cpython "${BASH_REMATCH[1]}"
Kyle Stewart's avatar
Kyle Stewart committed
149
    else
Matt Bachmann's avatar
Matt Bachmann committed
150
        echo "config error: Issue parsing this implementation in install_python:"
151
        echo "    version=$version"
Kyle Stewart's avatar
Kyle Stewart committed
152
153
154
155
        exit 1
    fi
}

156
function install_mac_cpython {
Matthew Brett's avatar
Matthew Brett committed
157
158
159
160
161
162
    # Installs Python.org Python
    # Parameter $version
    # Version given in major or major.minor or major.minor.micro e.g
    # "3" or "3.4" or "3.4.1".
    # sets $PYTHON_EXE variable to python executable
    local py_version=$(fill_pyver $1)
163
    local py_stripped=$(strip_ver_suffix $py_version)
164
    local py_inst=$(pyinst_fname_for_version $py_version)
Matthew Brett's avatar
Matthew Brett committed
165
166
    local inst_path=$DOWNLOADS_SDIR/$py_inst
    mkdir -p $DOWNLOADS_SDIR
167
    curl $MACPYTHON_URL/$py_stripped/${py_inst} > $inst_path
168
    if [ "${py_inst: -3}" == "dmg" ]; then
Matthew Brett's avatar
Matthew Brett committed
169
170
171
172
173
174
        hdiutil attach $inst_path -mountpoint /Volumes/Python
        inst_path=/Volumes/Python/Python.mpkg
    fi
    sudo installer -pkg $inst_path -target /
    local py_mm=${py_version:0:3}
    PYTHON_EXE=$MACPYTHON_PY_PREFIX/$py_mm/bin/python$py_mm
175
176
177
178
179
    # Install certificates for Python 3.6
    local inst_cmd="/Applications/Python ${py_mm}/Install Certificates.command"
    if [ -e "$inst_cmd" ]; then
        sh "$inst_cmd"
    fi
Matthew Brett's avatar
Matthew Brett committed
180
181
}

Kyle Stewart's avatar
Kyle Stewart committed
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
function install_mac_pypy {
    # Installs pypy.org PyPy
    # Parameter $version
    # Version given in major or major.minor or major.minor.micro e.g
    # "3" or "3.4" or "3.4.1".
    # sets $PYTHON_EXE variable to python executable
    local py_version=$(fill_pypy_ver $1)
    local py_build=$(get_pypy_build_prefix $py_version)$py_version-osx64
    local py_zip=$py_build.tar.bz2
    local zip_path=$DOWNLOADS_SDIR/$py_zip
    mkdir -p $DOWNLOADS_SDIR
    wget -nv $PYPY_URL/${py_zip} -P $DOWNLOADS_SDIR
    untar $zip_path
    PYTHON_EXE=$(realpath $py_build/bin/pypy)
}

Matthew Brett's avatar
Matthew Brett committed
198
199
200
201
202
203
204
205
function install_pip {
    # Generic install pip
    # Gets needed version from version implied by $PYTHON_EXE
    # Installs pip into python given by $PYTHON_EXE
    # Assumes pip will be installed into same directory as $PYTHON_EXE
    check_python
    mkdir -p $DOWNLOADS_SDIR
    curl $GET_PIP_URL > $DOWNLOADS_SDIR/get-pip.py
206
    # Python 2.6 will fail SSL check
Matthew Brett's avatar
Matthew Brett committed
207
    local py_mm=`get_py_mm`
208
209
210
211
212
213
    if [ "$py_mm" == "2.6" ]; then
        local pip_args="--trusted-host=pypi.python.org"
    fi
    # Travis VMS now install pip for system python by default - force install
    # even if installed already.
    sudo $PYTHON_EXE $DOWNLOADS_SDIR/get-pip.py --ignore-installed $pip_args
214
215
216
217
218
219
    PIP_CMD="sudo $(dirname $PYTHON_EXE)/pip$py_mm"
    # Append pip_args if present (avoiding trailing space cf using variable
    # above).
    if [ -n "$pip_args" ]; then
        PIP_CMD="$PIP_CMD $pip_args"
    fi
Matthew Brett's avatar
Matthew Brett committed
220
221
222
223
224
225
226
227
228
229
230
}

function install_virtualenv {
    # Generic install of virtualenv
    # Installs virtualenv into python given by $PYTHON_EXE
    # Assumes virtualenv will be installed into same directory as $PYTHON_EXE
    check_pip
    # Travis VMS install virtualenv for system python by default - force
    # install even if installed already
    $PIP_CMD install virtualenv --ignore-installed
    check_python
231
    VIRTUALENV_CMD="$(dirname $PYTHON_EXE)/virtualenv"
Matthew Brett's avatar
Matthew Brett committed
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
}

function make_workon_venv {
    # Make a virtualenv in given directory ('venv' default)
    # Set $PYTHON_EXE, $PIP_CMD to virtualenv versions
    # Parameter $venv_dir
    #    directory for virtualenv
    local venv_dir=$1
    if [ -z "$venv_dir" ]; then
        venv_dir="venv"
    fi
    venv_dir=`abspath $venv_dir`
    check_python
    $VIRTUALENV_CMD --python=$PYTHON_EXE $venv_dir
    PYTHON_EXE=$venv_dir/bin/python
    PIP_CMD=$venv_dir/bin/pip
248
249
250
    if [ "$(get_py_mm)" == "2.6" ]; then
        PIP_CMD="$PIP_CMD --trusted-host=pypi.python.org"
    fi
Matthew Brett's avatar
Matthew Brett committed
251
252
253
254
255
256
257
258
259
260
261
262
263
}

function remove_travis_ve_pip {
    # Remove travis installs of virtualenv and pip
    if [ "$(sudo which virtualenv)" == /usr/local/bin/virtualenv ]; then
        sudo pip uninstall -y virtualenv;
    fi
    if [ "$(sudo which pip)" == /usr/local/bin/pip ]; then
        sudo pip uninstall -y pip;
    fi
}

function set_py_vars {
264
    # Used by terryfy project; left here for back-compatibility
Matthew Brett's avatar
Matthew Brett committed
265
266
267
268
    export PATH="`dirname $PYTHON_EXE`:$PATH"
    export PYTHON_EXE PIP_CMD
}

269
function get_macpython_environment {
xoviat's avatar
xoviat committed
270
    if [ "$USE_CCACHE" == "1" ]; then
xoviat's avatar
xoviat committed
271
        activate_ccache
272
    fi
Matthew Brett's avatar
Matthew Brett committed
273
274
    # Set up MacPython environment
    # Parameters:
275
276
    #     $version : [implementation-]major[.minor[.patch]]
    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
Matthew Brett's avatar
Matthew Brett committed
277
278
279
280
281
282
283
284
285
    #     $venv_dir : {directory_name|not defined}
    #         If defined - make virtualenv in this directory, set python / pip
    #         commands accordingly
    #
    # Installs Python
    # Sets $PYTHON_EXE to path to Python executable
    # Sets $PIP_CMD to full command for pip (including sudo if necessary)
    # If $venv_dir defined, Sets $VIRTUALENV_CMD to virtualenv executable
    # Puts directory of $PYTHON_EXE on $PATH
286
287
    local version=$1
    local venv_dir=$2
Matthew Brett's avatar
Matthew Brett committed
288
    remove_travis_ve_pip
289
    install_macpython $version
Matthew Brett's avatar
Matthew Brett committed
290
    install_pip
xoviat's avatar
xoviat committed
291
    export HOMEBREW_NO_AUTO_UPDATE=1
Matthew Brett's avatar
Matthew Brett committed
292
293
294
    if [ -n "$venv_dir" ]; then
        install_virtualenv
        make_workon_venv $venv_dir
295
296
297
        source $venv_dir/bin/activate
    else
        export PATH="`dirname $PYTHON_EXE`:$PATH"
Matthew Brett's avatar
Matthew Brett committed
298
    fi
299
    export PYTHON_EXE PIP_CMD
Matthew Brett's avatar
Matthew Brett committed
300
}
301

302
function install_delocate {
303
    check_pip
304
305
306
    if [ $(lex_ver $(get_py_mm)) -lt $(lex_ver 2.7) ]; then
        # Wheel 0.30 doesn't work for Python 2.6; see:
        # https://github.com/pypa/wheel/issues/193
307
        $PIP_CMD install "wheel<=0.29"
308
    fi
309
    $PIP_CMD install delocate
310
311
312
313
314
315
}

function repair_wheelhouse {
    local wheelhouse=$1

    install_delocate
316

317
    delocate-listdeps $wheelhouse/*.whl # lists library dependencies
318
319
320
321
    # repair_wheelhouse can take more than 10 minutes without generating output
    # but jobs that do not generate output within 10 minutes are aborted by travis-ci.
    # Echoing something here solves the problem.
    echo in repair_wheelhouse, executing delocate-wheel
322
    delocate-wheel $wheelhouse/*.whl # copies library dependencies into wheel
323
324
325
326
327
328
329
    # Add platform tags to label wheels as compatible with OSX 10.9 and
    # 10.10.  The wheels will be built against Python.org Python, and so will
    # in fact be compatible with OSX >= 10.6.  pip < 6.0 doesn't realize
    # this, so, in case users have older pip, add platform tags to specify
    # compatibility with later OSX.  Not necessary for OSX released well
    # after pip 6.0.  See:
    # https://github.com/MacPython/wiki/wiki/Spinning-wheels#question-will-pip-give-me-a-broken-wheel
330
331
    delocate-addplat --rm-orig -x 10_9 -x 10_10 $wheelhouse/*.whl
}
332
333
334
335
336
337
338

function install_pkg_config {
    # Install pkg-config avoiding error from homebrew
    # See :
    # https://github.com/matthew-brett/multibuild/issues/24#issue-221951587
    command -v pkg-config > /dev/null 2>&1 || brew install pkg-config
}
xoviat's avatar
xoviat committed
339
340

function activate_ccache {
xoviat's avatar
xoviat committed
341
    brew update
xoviat's avatar
xoviat committed
342
343
344
345
346
347
348
349
    brew install ccache
    export PATH=/usr/local/opt/ccache/libexec:$PATH
    export CCACHE_MAXSIZE=200M
    export CCACHE_CPP2=1

    # Prove to the developer that ccache is activated
    which clang
}