osx_utils.sh 10.7 KB
Newer Older
Matthew Brett's avatar
Matthew Brett committed
1
2
3
4
#!/bin/bash
# Use with ``source osx_utils.sh``
set -e

5
6
7
8
# Get our own location on this filesystem, load common utils
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
source $MULTIBUILD_DIR/common_utils.sh

Matthew Brett's avatar
Matthew Brett committed
9
10
MACPYTHON_URL=https://www.python.org/ftp/python
MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
11
MACPYTHON_DEFAULT_OSX="10.6"
Matthew Brett's avatar
Matthew Brett committed
12
13
14
15
GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
DOWNLOADS_SDIR=downloads
WORKING_SDIR=working

Andrew Murray's avatar
Andrew Murray committed
16
# As of 20 October 2018 - latest Python of each version with binary download
Matthew Brett's avatar
Matthew Brett committed
17
# available.
18
# See: https://www.python.org/downloads/mac-osx/
Andrew Murray's avatar
Andrew Murray committed
19
LATEST_2p7=2.7.15
Matthew Brett's avatar
Matthew Brett committed
20
LATEST_3p4=3.4.4
Andrew Murray's avatar
Andrew Murray committed
21
LATEST_3p5=3.5.4
Andrew Murray's avatar
Andrew Murray committed
22
23
LATEST_3p6=3.6.7
LATEST_3p7=3.7.1
Matthew Brett's avatar
Matthew Brett committed
24

Matthew Brett's avatar
Matthew Brett committed
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

function check_python {
    if [ -z "$PYTHON_EXE" ]; then
        echo "PYTHON_EXE variable not defined"
        exit 1
    fi
}

function check_pip {
    if [ -z "$PIP_CMD" ]; then
        echo "PIP_CMD variable not defined"
        exit 1
    fi
}

function check_var {
    if [ -z "$1" ]; then
        echo "required variable not defined"
        exit 1
    fi
}

function get_py_digit {
    check_python
    $PYTHON_EXE -c "import sys; print(sys.version_info[0])"
}

function get_py_mm {
    check_python
    $PYTHON_EXE -c "import sys; print('{0}.{1}'.format(*sys.version_info[0:2]))"
}

function get_py_mm_nodot {
    check_python
    $PYTHON_EXE -c "import sys; print('{0}{1}'.format(*sys.version_info[0:2]))"
}

function get_py_prefix {
    check_python
    $PYTHON_EXE -c "import sys; print(sys.prefix)"
}

function fill_pyver {
    # Convert major or major.minor format to major.minor.micro
    #
    # Hence:
    # 2 -> 2.7.11  (depending on LATEST_2p7 value)
    # 2.7 -> 2.7.11  (depending on LATEST_2p7 value)
    local ver=$1
    check_var $ver
    if [[ $ver =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then
        # Major.minor.micro format already
        echo $ver
    elif [ $ver == 2 ] || [ $ver == "2.7" ]; then
        echo $LATEST_2p7
Andrew Murray's avatar
Andrew Murray committed
80
81
82
    elif [ $ver == 3 ] || [ $ver == "3.7" ]; then
        echo $LATEST_3p7
    elif [ $ver == "3.6" ]; then
Matthew Brett's avatar
Matthew Brett committed
83
84
        echo $LATEST_3p6
    elif [ $ver == "3.5" ]; then
Matthew Brett's avatar
Matthew Brett committed
85
86
87
88
        echo $LATEST_3p5
    elif [ $ver == "3.4" ]; then
        echo $LATEST_3p4
    else
89
        echo "Can't fill version $ver" 1>&2
Matthew Brett's avatar
Matthew Brett committed
90
91
92
93
94
95
96
97
98
99
100
101
        exit 1
    fi
}

function pyinst_ext_for_version {
    # echo "pkg" or "dmg" depending on the passed Python version
    # Parameters
    #   $py_version (python version in major.minor.extra format)
    #
    # Earlier Python installers are .dmg, later are .pkg.
    local py_version=$1
    check_var $py_version
102
    py_version=$(fill_pyver $py_version)
Matthew Brett's avatar
Matthew Brett committed
103
104
    local py_0=${py_version:0:1}
    if [ $py_0 -eq 2 ]; then
105
        if [ "$(lex_ver $py_version)" -ge "$(lex_ver 2.7.9)" ]; then
Matthew Brett's avatar
Matthew Brett committed
106
107
108
109
110
            echo "pkg"
        else
            echo "dmg"
        fi
    elif [ $py_0 -ge 3 ]; then
111
        if [ "$(lex_ver $py_version)" -ge "$(lex_ver 3.4.2)" ]; then
Matthew Brett's avatar
Matthew Brett committed
112
113
114
115
116
117
118
            echo "pkg"
        else
            echo "dmg"
        fi
    fi
}

119
120
121
122
function pyinst_fname_for_version {
    # echo filename for OSX installer file given Python version
    # Parameters
    #   $py_version (python version in major.minor.extra format)
123
    #   $osx_ver    (macosx version in major.minor format, defaults to MACPYTHON_DEFAULT_OSX)
124
    local py_version=$1
125
    local osx_ver=${2:-$MACPYTHON_DEFAULT_OSX}
126
127
128
129
    local inst_ext=$(pyinst_ext_for_version $py_version)
    echo "python-$py_version-macosx${osx_ver}.$inst_ext"
}

130
131
132
133
134
function install_macpython {
    # Install Python and set $PYTHON_EXE to the installed executable
    # Parameters:
    #     $version : [implementation-]major[.minor[.patch]]
    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
135
    #     $osx_version: { 10.6|10.9 }. Ignored for pypy
136
    local version=$1
137
    local osx_version=$2
138
    if [[ "$version" =~ pypy-([0-9\.]+) ]]; then
139
        install_mac_pypy "${BASH_REMATCH[1]}"
140
    elif [[ "$version" =~ ([0-9\.]+) ]]; then
141
        install_mac_cpython "${BASH_REMATCH[1]}" $osx_version
Kyle Stewart's avatar
Kyle Stewart committed
142
    else
Matt Bachmann's avatar
Matt Bachmann committed
143
        echo "config error: Issue parsing this implementation in install_python:"
144
        echo "    version=$version"
Kyle Stewart's avatar
Kyle Stewart committed
145
146
147
148
        exit 1
    fi
}

149
function install_mac_cpython {
Matthew Brett's avatar
Matthew Brett committed
150
    # Installs Python.org Python
151
152
153
154
155
    # Parameters
    #   $py_version
    #       Version given in major or major.minor or major.minor.micro e.g
    #       "3" or "3.4" or "3.4.1".
    #   $osx_version = { 10.6 | 10.9 }
Matthew Brett's avatar
Matthew Brett committed
156
157
    # sets $PYTHON_EXE variable to python executable
    local py_version=$(fill_pyver $1)
158
    local osx_version=$2
159
    local py_stripped=$(strip_ver_suffix $py_version)
160
    local py_inst=$(pyinst_fname_for_version $py_version $osx_version)
Matthew Brett's avatar
Matthew Brett committed
161
162
    local inst_path=$DOWNLOADS_SDIR/$py_inst
    mkdir -p $DOWNLOADS_SDIR
163
    curl $MACPYTHON_URL/$py_stripped/${py_inst} > $inst_path
164
    if [ "${py_inst: -3}" == "dmg" ]; then
Matthew Brett's avatar
Matthew Brett committed
165
166
167
168
169
170
        hdiutil attach $inst_path -mountpoint /Volumes/Python
        inst_path=/Volumes/Python/Python.mpkg
    fi
    sudo installer -pkg $inst_path -target /
    local py_mm=${py_version:0:3}
    PYTHON_EXE=$MACPYTHON_PY_PREFIX/$py_mm/bin/python$py_mm
171
172
173
174
175
    # Install certificates for Python 3.6
    local inst_cmd="/Applications/Python ${py_mm}/Install Certificates.command"
    if [ -e "$inst_cmd" ]; then
        sh "$inst_cmd"
    fi
Matthew Brett's avatar
Matthew Brett committed
176
177
}

Kyle Stewart's avatar
Kyle Stewart committed
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
function install_mac_pypy {
    # Installs pypy.org PyPy
    # Parameter $version
    # Version given in major or major.minor or major.minor.micro e.g
    # "3" or "3.4" or "3.4.1".
    # sets $PYTHON_EXE variable to python executable
    local py_version=$(fill_pypy_ver $1)
    local py_build=$(get_pypy_build_prefix $py_version)$py_version-osx64
    local py_zip=$py_build.tar.bz2
    local zip_path=$DOWNLOADS_SDIR/$py_zip
    mkdir -p $DOWNLOADS_SDIR
    wget -nv $PYPY_URL/${py_zip} -P $DOWNLOADS_SDIR
    untar $zip_path
    PYTHON_EXE=$(realpath $py_build/bin/pypy)
}

Matthew Brett's avatar
Matthew Brett committed
194
195
196
197
198
199
200
201
function install_pip {
    # Generic install pip
    # Gets needed version from version implied by $PYTHON_EXE
    # Installs pip into python given by $PYTHON_EXE
    # Assumes pip will be installed into same directory as $PYTHON_EXE
    check_python
    mkdir -p $DOWNLOADS_SDIR
    local py_mm=`get_py_mm`
202
    local get_pip_path=$DOWNLOADS_SDIR/get-pip.py
Hugo's avatar
Hugo committed
203
    curl $GET_PIP_URL > $get_pip_path
204
205
    # Travis VMS now install pip for system python by default - force install
    # even if installed already.
206
    sudo $PYTHON_EXE $get_pip_path --ignore-installed $pip_args
207
208
209
210
211
212
    PIP_CMD="sudo $(dirname $PYTHON_EXE)/pip$py_mm"
    # Append pip_args if present (avoiding trailing space cf using variable
    # above).
    if [ -n "$pip_args" ]; then
        PIP_CMD="$PIP_CMD $pip_args"
    fi
Matthew Brett's avatar
Matthew Brett committed
213
214
215
216
217
218
219
220
221
222
223
}

function install_virtualenv {
    # Generic install of virtualenv
    # Installs virtualenv into python given by $PYTHON_EXE
    # Assumes virtualenv will be installed into same directory as $PYTHON_EXE
    check_pip
    # Travis VMS install virtualenv for system python by default - force
    # install even if installed already
    $PIP_CMD install virtualenv --ignore-installed
    check_python
224
    VIRTUALENV_CMD="$(dirname $PYTHON_EXE)/virtualenv"
Matthew Brett's avatar
Matthew Brett committed
225
226
227
228
229
230
231
232
233
234
235
236
237
}

function make_workon_venv {
    # Make a virtualenv in given directory ('venv' default)
    # Set $PYTHON_EXE, $PIP_CMD to virtualenv versions
    # Parameter $venv_dir
    #    directory for virtualenv
    local venv_dir=$1
    if [ -z "$venv_dir" ]; then
        venv_dir="venv"
    fi
    venv_dir=`abspath $venv_dir`
    check_python
xoviat's avatar
xoviat committed
238
    $PYTHON_EXE -m virtualenv $venv_dir
Matthew Brett's avatar
Matthew Brett committed
239
240
241
242
243
244
    PYTHON_EXE=$venv_dir/bin/python
    PIP_CMD=$venv_dir/bin/pip
}

function remove_travis_ve_pip {
    # Remove travis installs of virtualenv and pip
xoviat's avatar
xoviat committed
245
246
    # FIXME: What if virtualenv is installed but pip is not?
    if [ "$(sudo which virtualenv)" == /usr/local/bin/virtualenv ] && [ "$(sudo which pip)" == /usr/local/bin/pip ]; then
Matthew Brett's avatar
Matthew Brett committed
247
248
249
250
251
252
253
254
        sudo pip uninstall -y virtualenv;
    fi
    if [ "$(sudo which pip)" == /usr/local/bin/pip ]; then
        sudo pip uninstall -y pip;
    fi
}

function set_py_vars {
255
    # Used by terryfy project; left here for back-compatibility
Matthew Brett's avatar
Matthew Brett committed
256
257
258
259
    export PATH="`dirname $PYTHON_EXE`:$PATH"
    export PYTHON_EXE PIP_CMD
}

260
function get_macpython_environment {
Matthew Brett's avatar
Matthew Brett committed
261
262
    # Set up MacPython environment
    # Parameters:
263
264
    #     $version : [implementation-]major[.minor[.patch]]
    #         The Python implementation to install, e.g. "3.6" or "pypy-5.4"
Matthew Brett's avatar
Matthew Brett committed
265
266
267
    #     $venv_dir : {directory_name|not defined}
    #         If defined - make virtualenv in this directory, set python / pip
    #         commands accordingly
268
    #     $osx_version: {10.6 | 10.9}
Matthew Brett's avatar
Matthew Brett committed
269
270
271
272
273
274
    #
    # Installs Python
    # Sets $PYTHON_EXE to path to Python executable
    # Sets $PIP_CMD to full command for pip (including sudo if necessary)
    # If $venv_dir defined, Sets $VIRTUALENV_CMD to virtualenv executable
    # Puts directory of $PYTHON_EXE on $PATH
275
    local version=$1
276
277
    local osx_version=$2
    local venv_dir=$3
xoviat's avatar
xoviat committed
278
279
280
281
282

    if [ "$USE_CCACHE" == "1" ]; then
        activate_ccache
    fi
    
Matthew Brett's avatar
Matthew Brett committed
283
    remove_travis_ve_pip
284
    install_macpython $version $osx_version
Matthew Brett's avatar
Matthew Brett committed
285
    install_pip
xoviat's avatar
xoviat committed
286

Matthew Brett's avatar
Matthew Brett committed
287
288
289
    if [ -n "$venv_dir" ]; then
        install_virtualenv
        make_workon_venv $venv_dir
290
291
292
        source $venv_dir/bin/activate
    else
        export PATH="`dirname $PYTHON_EXE`:$PATH"
Matthew Brett's avatar
Matthew Brett committed
293
    fi
294
    export PYTHON_EXE PIP_CMD
Matthew Brett's avatar
Matthew Brett committed
295
}
296

297
function install_delocate {
298
299
    check_pip
    $PIP_CMD install delocate
300
301
302
303
304
305
}

function repair_wheelhouse {
    local wheelhouse=$1

    install_delocate
306
    delocate-wheel $wheelhouse/*.whl # copies library dependencies into wheel
307
    # Add platform tags to label wheels as compatible with OSX 10.9 and
308
309
310
311
312
    # 10.10.  The wheels are built against Python.org Python, and so will
    # in fact be compatible with either 10.6+ or 10.9+, depending on the value
    # of MB_PYTHON_OSX_VER. pip < 6.0 doesn't realize this, so, in case users
    # have older pip, add platform tags to specify compatibility with later OSX.
    # Not necessary for OSX released well after pip 6.0.  See:
313
    # https://github.com/MacPython/wiki/wiki/Spinning-wheels#question-will-pip-give-me-a-broken-wheel
314
    if [ $MB_PYTHON_OSX_VER == "10.6" ]; then
315
        # assume that 10.6-based python is dual arch (32/64-bit)
316
317
        delocate-addplat --rm-orig -x 10_9 -x 10_10 $wheelhouse/*.whl
    elif [ $MB_PYTHON_OSX_VER == "10.9" ]; then
318
        # assume that 10.9-based python is 64-bit arch only
319
320
321
322
323
        delocate-addplat --rm-orig -p macosx_10_10_x86_64 $wheelhouse/*.whl
    else
        echo "Invalid python macosx version $MB_PYTHON_OSX_VER" 1>&2
        exit 1
    fi
324
}
325
326
327
328
329
330
331

function install_pkg_config {
    # Install pkg-config avoiding error from homebrew
    # See :
    # https://github.com/matthew-brett/multibuild/issues/24#issue-221951587
    command -v pkg-config > /dev/null 2>&1 || brew install pkg-config
}
xoviat's avatar
xoviat committed
332
333

function activate_ccache {
xoviat's avatar
xoviat committed
334

xoviat's avatar
xoviat committed
335
336
337
338
339
    brew install ccache
    export PATH=/usr/local/opt/ccache/libexec:$PATH
    export CCACHE_CPP2=1

    # Prove to the developer that ccache is activated
xoviat's avatar
xoviat committed
340
    echo "Using C compiler: $(which clang)"
xoviat's avatar
xoviat committed
341
}