common_utils.sh 21.1 KB
Newer Older
1
#!/bin/bash
2
# Utilities for both OSX and Docker Linux
3
# Python should be on the PATH
4
5
6
7
8
9
10
11

# Only source common_utils once
if [ -n "$COMMON_UTILS_SOURCED" ]; then
    return
fi
COMMON_UTILS_SOURCED=1

# Turn on exit-if-error
12
13
set -e

14
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
mattip's avatar
mattip committed
15
DOWNLOADS_SDIR=downloads
mattip's avatar
mattip committed
16
PYPY_URL=https://downloads.python.org/pypy
Matthew Brett's avatar
Matthew Brett committed
17
18
# For back-compatibility.  We use the "ensurepip" module now
# instead of get-pip.py
mattip's avatar
mattip committed
19
20
GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py

21
22
23
24
25
26
# Unicode width, default 32. Used here and in travis_linux_steps.sh
# In docker_build_wrap.sh it is passed in when calling "docker run"
# The docker test images also use it when choosing the python to run
# with, so it is passed in when calling "docker run" for tests.
UNICODE_WIDTH=${UNICODE_WIDTH:-32}

27
28
29
# Default Manylinux version
MB_ML_VER=${MB_ML_VER:-2014}

mattip's avatar
mattip committed
30
31
32
if [ $(uname) == "Darwin" ]; then
  IS_MACOS=1; IS_OSX=1;
else
mattip's avatar
mattip committed
33
34
  # In the manylinux_2_24 image, based on Debian9, "python" is not installed
  # so link in something for the various system calls before PYTHON_EXE is set
mattip's avatar
mattip committed
35
36
37
  which python || export PATH=/opt/python/cp39-cp39/bin:$PATH
fi

John Jones's avatar
John Jones committed
38
if [ "$MB_ML_LIBC" == "musllinux" ]; then
John Jones's avatar
John Jones committed
39
40
  IS_ALPINE=1;
fi
41

Matthew Brett's avatar
Matthew Brett committed
42
43
44
45
# Work round bug in travis xcode image described at
# https://github.com/direnv/direnv/issues/210
shell_session_update() { :; }

xoviat's avatar
xoviat committed
46
47
48
49
# Workaround for https://github.com/travis-ci/travis-ci/issues/8703
# suggested by Thomas K at
# https://github.com/travis-ci/travis-ci/issues/8703#issuecomment-347881274
unset -f cd
Matthew Brett's avatar
Matthew Brett committed
50
51
unset -f pushd
unset -f popd
xoviat's avatar
xoviat committed
52

xoviat's avatar
xoviat committed
53
54
55
56
57
function start_spinner {
    if [ -n "$MB_SPINNER_PID" ]; then
        return
    fi

xoviat's avatar
xoviat committed
58
    >&2 echo "Building libraries..."
59
60
    # Start a process that runs as a keep-alive
    # to avoid travis quitting if there is no output
xoviat's avatar
xoviat committed
61
    (while true; do
xoviat's avatar
xoviat committed
62
        sleep 60
xoviat's avatar
xoviat committed
63
        >&2 echo "Still building..."
xoviat's avatar
xoviat committed
64
65
    done) &
    MB_SPINNER_PID=$!
xoviat's avatar
xoviat committed
66
    disown
xoviat's avatar
xoviat committed
67
}
xoviat's avatar
xoviat committed
68

xoviat's avatar
xoviat committed
69
70
71
72
function stop_spinner {
    if [ ! -n "$MB_SPINNER_PID" ]; then
        return
    fi
73

xoviat's avatar
xoviat committed
74
75
    kill $MB_SPINNER_PID
    unset MB_SPINNER_PID
xoviat's avatar
xoviat committed
76
77

    >&2 echo "Building libraries finished."
xoviat's avatar
xoviat committed
78
}
xoviat's avatar
xoviat committed
79

80
function abspath {
81
    # Can work with any Python; need not be our installed Python.
82
83
84
    python -c "import os.path; print(os.path.abspath('$1'))"
}

85
86
function relpath {
    # Path of first input relative to second (or $PWD if not specified)
87
    # Can work with any Python; need not be our installed Python.
88
89
90
    python -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))"
}

Matthew Brett's avatar
Matthew Brett committed
91
function realpath {
92
    # Can work with any Python; need not be our installed Python.
Matthew Brett's avatar
Matthew Brett committed
93
94
95
    python -c "import os; print(os.path.realpath('$1'))"
}

96
97
98
99
100
101
102
103
function lex_ver {
    # Echoes dot-separated version string padded with zeros
    # Thus:
    # 3.2.1 -> 003002001
    # 3     -> 003000000
    echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}'
}

104
105
106
107
108
109
110
111
112
113
114
115
function unlex_ver {
    # Reverses lex_ver to produce major.minor.micro
    # Thus:
    # 003002001 -> 3.2.1
    # 003000000 -> 3.0.0
    echo "$((10#${1:0:3}+0)).$((10#${1:3:3}+0)).$((10#${1:6:3}+0))"
}

function strip_ver_suffix {
    echo $(unlex_ver $(lex_ver $1))
}

116
function is_function {
117
118
    # Echo "true" if input argument string is a function
    # Allow errors during "set -e" blocks.
Matthew Brett's avatar
Matthew Brett committed
119
    (set +e; $(declare -Ff "$1" > /dev/null) && echo true)
120
121
}

122
function gh_clone {
123
124
125
    git clone https://github.com/$1
}

Denver Ogaro's avatar
Denver Ogaro committed
126
127
128
129
# gh-clone was renamed to gh_clone, so we have this alias for
# backwards compatibility.
alias gh-clone=gh_clone

130
131
132
function set_opts {
    # Set options from input options string (in $- format).
    local opts=$1
mattip's avatar
mattip committed
133
    local chars="exhmBH"
134
135
136
137
138
139
    for (( i=0; i<${#chars}; i++ )); do
        char=${chars:$i:1}
        [ -n "${opts//[^${char}]/}" ] && set -$char || set +$char
    done
}

xoviat's avatar
xoviat committed
140
function suppress {
141
142
143
144
145
146
147
    # Run a command, show output only if return code not 0.
    # Takes into account state of -e option.
    # Compare
    # https://unix.stackexchange.com/questions/256120/how-can-i-suppress-output-only-if-the-command-succeeds#256122
    # Set -e stuff agonized over in
    # https://unix.stackexchange.com/questions/296526/set-e-in-a-subshell
    local tmp=$(mktemp tmp.XXXXXXXXX) || return
Matthew Brett's avatar
Matthew Brett committed
148
    local errexit_set
149
    echo "Running $@"
Matthew Brett's avatar
Matthew Brett committed
150
    if [[ $- = *e* ]]; then errexit_set=true; fi
151
    set +e
Matthew Brett's avatar
Matthew Brett committed
152
    ( if [[ -n $errexit_set ]]; then set -e; fi; "$@"  > "$tmp" 2>&1 ) ; ret=$?
153
154
    [ "$ret" -eq 0 ] || cat "$tmp"
    rm -f "$tmp"
Matthew Brett's avatar
Matthew Brett committed
155
    if [[ -n $errexit_set ]]; then set -e; fi
156
    return "$ret"
xoviat's avatar
xoviat committed
157
158
}

robbuckley's avatar
robbuckley committed
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
function expect_return {
  # Run a command, succeeding (returning 0) only if the commend returns a specified code
  # Parameters
  #   retcode   expected return code (which may be zero)
  #   command   the command called
  #
  #   any further arguments are passed to the called command
  #
  # Returns 1 if called with less than 2 arguments
  (( $# < 2 )) && echo "Must have at least 2 arguments" && return 1
  local retcode=$1
  local retval
  ( "${@:2}" ) || retval=$?
  [[ $retcode == ${retval:-0} ]] && return 0
  return ${retval:-1}
}

function cmd_notexit {
    # wraps a command, capturing its return code and preventing it
    # from exiting the shell. Handles -e / +e modes.
    # Parameters
    #    cmd - command
    #    any further parameters are passed to the wrapped command
    # If called without an argument, it will exit the shell with an error
    local cmd=$1
    if [ -z "$cmd" ];then echo "no command"; exit 1; fi
    if [[ $- = *e* ]]; then errexit_set=true; fi
    set +e
    ("${@:1}") ; retval=$?
    [[ -n $errexit_set ]] && set -e
    return $retval
}

192
193
194
function rm_mkdir {
    # Remove directory if present, then make directory
    local path=$1
195
196
    if [ -z "$path" ]; then echo "Need not-empty path"; exit 1; fi
    if [ -d "$path" ]; then rm -rf $path; fi
197
198
199
    mkdir $path
}

200
201
202
203
204
function untar {
    local in_fname=$1
    if [ -z "$in_fname" ];then echo "in_fname not defined"; exit 1; fi
    local extension=${in_fname##*.}
    case $extension in
205
206
207
        tar) tar -xf $in_fname ;;
        gz|tgz) tar -zxf $in_fname ;;
        bz2) tar -jxf $in_fname ;;
208
        zip) unzip -qq $in_fname ;;
209
210
211
        xz) if [ -n "$IS_MACOS" ]; then
              tar -xf $in_fname
            else
212
213
214
              if [[ ! $(type -P "unxz") ]]; then
                echo xz must be installed to uncompress file; exit 1
              fi
215
216
              unxz -c $in_fname | tar -xf -
            fi ;;
217
218
219
220
        *) echo Did not recognize extension $extension; exit 1 ;;
    esac
}

221
function install_rsync {
mattip's avatar
mattip committed
222
    # install rsync via package manager
mattip's avatar
mattip committed
223
    if [ -n "$IS_MACOS" ]; then
224
225
        # macOS. The colon in the next line is the null command
        :
John Jones's avatar
John Jones committed
226
227
    elif [ -n "$IS_ALPINE" ]; then
        [[ $(type -P rsync) ]] || apk add rsync
mattip's avatar
mattip committed
228
    elif [[ $MB_ML_VER == "_2_24" ]]; then
mattip's avatar
mattip committed
229
        # debian:9 based distro
mattip's avatar
mattip committed
230
        [[ $(type -P rsync) ]] || apt-get install -y rsync
mattip's avatar
mattip committed
231
    else
mattip's avatar
mattip committed
232
        # centos based distro
mattip's avatar
mattip committed
233
        [[ $(type -P rsync) ]] || yum_install rsync
234
235
236
    fi
}

237
238
239
240
241
242
function fetch_unpack {
    # Fetch input archive name from input URL
    # Parameters
    #    url - URL from which to fetch archive
    #    archive_fname (optional) archive name
    #
243
244
    # Echos unpacked directory and file names.
    #
245
246
247
248
249
250
    # If `archive_fname` not specified then use basename from `url`
    # If `archive_fname` already present at download location, use that instead.
    local url=$1
    if [ -z "$url" ];then echo "url not defined"; exit 1; fi
    local archive_fname=${2:-$(basename $url)}
    local arch_sdir="${ARCHIVE_SDIR:-archives}"
251
252
253
254
255
256
    if [ -z "$IS_MACOS" ]; then
        local extension=${archive_fname##*.}
        if [ "$extension" == "xz" ]; then
            ensure_xz
        fi
    fi
257
258
259
    # Make the archive directory in case it doesn't exist
    mkdir -p $arch_sdir
    local out_archive="${arch_sdir}/${archive_fname}"
260
    # If the archive is not already in the archives directory, get it.
261
    if [ ! -f "$out_archive" ]; then
262
263
264
        # Source it from multibuild archives if available.
        local our_archive="${MULTIBUILD_DIR}/archives/${archive_fname}"
        if [ -f "$our_archive" ]; then
265
            ln -s $our_archive $out_archive
266
267
268
269
        else
            # Otherwise download it.
            curl -L $url > $out_archive
        fi
270
    fi
271
272
    # Unpack archive, refreshing contents, echoing dir and file
    # names.
273
    rm_mkdir arch_tmp
274
    install_rsync
275
276
277
278
    (cd arch_tmp && \
        untar ../$out_archive && \
        ls -1d * &&
        rsync --delete -ah * ..)
279
280
}

281
282
283
284
285
function clean_code {
    local repo_dir=${1:-$REPO_DIR}
    local build_commit=${2:-$BUILD_COMMIT}
    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
    [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1
286
287
288
289
290
    # The package $repo_dir may be a submodule. git submodules do not
    # have a .git directory. If $repo_dir is copied around, tools like
    # Versioneer which require that it be a git repository are unable
    # to determine the version.  Give submodule proper git directory
    fill_submodule "$repo_dir"
291
292
293
294
295
296
297
298
    (cd $repo_dir \
        && git fetch origin \
        && git checkout $build_commit \
        && git clean -fxd \
        && git reset --hard \
        && git submodule update --init --recursive)
}

299
300
301
302
303
function build_wheel_cmd {
    # Builds wheel with named command, puts into $WHEEL_SDIR
    #
    # Parameters:
    #     cmd  (optional, default "pip_wheel_cmd"
Andrew Murray's avatar
Andrew Murray committed
304
    #        Name of command for building wheel
305
    #     repo_dir  (optional, default $REPO_DIR)
306
307
308
309
310
311
    #
    # Depends on
    #     REPO_DIR  (or via input argument)
    #     WHEEL_SDIR  (optional, default "wheelhouse")
    #     BUILD_DEPENDS (optional, default "")
    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
312
313
    local cmd=${1:-pip_wheel_cmd}
    local repo_dir=${2:-$REPO_DIR}
314
315
    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
John Jones's avatar
John Jones committed
316
    mkdir -p "$wheelhouse"
xoviat's avatar
xoviat committed
317
318
319
    start_spinner
    if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
    stop_spinner
320
    if [ -n "$BUILD_DEPENDS" ]; then
Isuru Fernando's avatar
Isuru Fernando committed
321
        pip install $(pip_opts) $BUILD_DEPENDS
322
323
    fi
    (cd $repo_dir && $cmd $wheelhouse)
324
325
326
    repair_wheelhouse $wheelhouse
}

327
328
function pip_wheel_cmd {
    local abs_wheelhouse=$1
Isuru Fernando's avatar
Isuru Fernando committed
329
    pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps .
330
331
332
333
334
335
336
337
338
339
}

function bdist_wheel_cmd {
    # Builds wheel with bdist_wheel, puts into wheelhouse
    #
    # It may sometimes be useful to use bdist_wheel for the wheel building
    # process.  For example, versioneer has problems with versions which are
    # fixed with bdist_wheel:
    # https://github.com/warner/python-versioneer/issues/121
    local abs_wheelhouse=$1
340
341
    check_python
    $PYTHON_EXE setup.py bdist_wheel
342
343
344
    cp dist/*.whl $abs_wheelhouse
}

345
function wrap_wheel_builder {
Isuru Fernando's avatar
Isuru Fernando committed
346
    # Wrapper for build commands, overwritten by macOS for universal2 or arm64 wheel building
347
    $@
348
349
}

350
function build_pip_wheel {
351
    # Standard wheel building command with pip wheel
352
    wrap_wheel_builder build_wheel_cmd "pip_wheel_cmd" $@
353
354
355
356
}

function build_bdist_wheel {
    # Wheel building with bdist_wheel. See bdist_wheel_cmd
357
    wrap_wheel_builder build_wheel_cmd "bdist_wheel_cmd" $@
358
359
}

360
361
function build_wheel {
    # Set default building method to pip
362
    wrap_wheel_builder build_pip_wheel $@
363
364
}

365
function build_index_wheel_cmd {
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
    # Builds wheel from some index, usually pypi
    #
    # Parameters:
    #     project_spec
    #        requirement to install, e.g. "tornado" or "tornado==4.4.1"
    #     *args
    #        Any other arguments to be passed to pip `install`  and `wheel`
    #        commands.
    #
    # Depends on
    #     WHEEL_SDIR  (optional, default "wheelhouse")
    #     BUILD_DEPENDS (optional, default "")
    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
    #
    # You can also override `pip_opts` command to set indices other than pypi
    local project_spec=$1
    [ -z "$project_spec" ] && echo "project_spec not defined" && exit 1
    # Discard first argument to pass remainder to pip
    shift
    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
xoviat's avatar
xoviat committed
386
    start_spinner
387
    if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
xoviat's avatar
xoviat committed
388
    stop_spinner
389
    if [ -n "$BUILD_DEPENDS" ]; then
Isuru Fernando's avatar
Isuru Fernando committed
390
        pip install $(pip_opts) $@ $BUILD_DEPENDS
391
    fi
Isuru Fernando's avatar
Isuru Fernando committed
392
    pip wheel $(pip_opts) $@ -w $wheelhouse --no-deps $project_spec
393
394
395
    repair_wheelhouse $wheelhouse
}

396
function build_index_wheel {
397
    wrap_wheel_builder build_index_wheel_cmd $@
398
399
}

400
401
function pip_opts {
    [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL"
402
}
403

Isuru Fernando's avatar
Isuru Fernando committed
404
405
406
407
408
409
function get_os {
    # Report OS as given by uname
    # Use any Python that comes to hand.
    python -c 'import platform; print(platform.uname()[0])'
}

410
411
function get_platform {
    # Report platform as given by uname
412
413
    # Use any Python that comes to hand.
    python -c 'import platform; print(platform.uname()[4])'
414
415
}

416
417
418
if [ "$(get_platform)" == x86_64 ] || \
    [ "$(get_platform)" == i686 ]; then IS_X86=1; fi

419
420
421
function get_distutils_platform {
    # Report platform as given by distutils get_platform.
    # This is the platform tag that pip will use.
422
423
    check_python
    $PYTHON_EXE -c "import distutils.util; print(distutils.util.get_platform())"
424
425
}

426
427
function install_wheel {
    # Install test dependencies and built wheel
428
    #
429
    # Pass any input flags to pip install steps
430
    #
431
    # Depends on:
432
433
434
435
    #     WHEEL_SDIR  (optional, default "wheelhouse")
    #     TEST_DEPENDS  (optional, default "")
    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
436
    check_pip
437
    if [ -n "$TEST_DEPENDS" ]; then
438
        while read TEST_DEPENDENCY; do
mattip's avatar
mattip committed
439
            $PIP_CMD install $(pip_opts) $@ $TEST_DEPENDENCY
440
        done <<< "$TEST_DEPENDS"
441
    fi
442
443
444
445

    check_python
    check_pip

mattip's avatar
mattip committed
446
    $PIP_CMD install packaging
mattip's avatar
mattip committed
447
    local supported_wheels=$($PYTHON_EXE $MULTIBUILD_DIR/supported_wheels.py $wheelhouse/*.whl)
448
449
450
451
    if [ -z "$supported_wheels" ]; then
        echo "ERROR: no supported wheels found"
        exit 1
    fi
452
    # Install compatible wheel
mattip's avatar
mattip committed
453
    $PIP_CMD install $(pip_opts) $@ $supported_wheels
454
}
455
456
457
458
459
460

function install_run {
    # Depends on function `run_tests` defined in `config.sh`
    install_wheel
    mkdir tmp_for_test
    (cd tmp_for_test && run_tests)
Matthew Brett's avatar
Matthew Brett committed
461
    rmdir tmp_for_test  2>/dev/null || echo "Cannot remove tmp_for_test"
462
}
463
464
465
466

function fill_submodule {
    # Restores .git directory to submodule, if necessary
    # See:
Andrew Murray's avatar
Andrew Murray committed
467
    # https://stackoverflow.com/questions/41776331/is-there-a-way-to-reconstruct-a-git-directory-for-a-submodule
468
469
470
471
472
473
474
    local repo_dir="$1"
    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
    local git_loc="$repo_dir/.git"
    # For ordinary submodule, .git is a file.
    [ -d "$git_loc" ] && return
    # Need to recreate .git directory for submodule
    local origin_url=$(cd "$repo_dir" && git config --get remote.origin.url)
Matthew Brett's avatar
Matthew Brett committed
475
476
    local repo_copy="$repo_dir-$RANDOM"
    git clone --recursive "$repo_dir" "$repo_copy"
477
    rm -rf "$repo_dir"
Matthew Brett's avatar
Matthew Brett committed
478
    mv "${repo_copy}" "$repo_dir"
479
480
    (cd "$repo_dir" && git remote set-url origin $origin_url)
}
Kyle Stewart's avatar
Kyle Stewart committed
481

mattip's avatar
mattip committed
482
# The latest versions of PyPy.
Kyle Stewart's avatar
Kyle Stewart committed
483
484
485
486
487
LATEST_PP_5p0=5.0.1
LATEST_PP_5p1=5.1.1
LATEST_PP_5p3=5.3.1
LATEST_PP_5p4=5.4.1
LATEST_PP_5p6=5.6.0
Andrew Murray's avatar
Andrew Murray committed
488
LATEST_PP_5p7=5.7.1
Andrew Murray's avatar
Andrew Murray committed
489
490
LATEST_PP_5p8=5.8.0
LATEST_PP_5p9=5.9.0
Andrew Murray's avatar
Andrew Murray committed
491
LATEST_PP_5=$LATEST_PP_5p9
Kyle Stewart's avatar
Kyle Stewart committed
492

Andrew Murray's avatar
Andrew Murray committed
493
494
495
LATEST_PP_6p0=6.0.0
LATEST_PP_6=$LATEST_PP_6p0

Andrew Murray's avatar
Andrew Murray committed
496
LATEST_PP_7p0=7.0.0
Andrew Murray's avatar
Andrew Murray committed
497
LATEST_PP_7p1=7.1.1
Andrew Murray's avatar
Andrew Murray committed
498
LATEST_PP_7p2=7.2.0
mattip's avatar
mattip committed
499
LATEST_PP_7p3=7.3.3
mattip's avatar
mattip committed
500
LATEST_PP_7=$LATEST_PP_7p3
Andrew Murray's avatar
Andrew Murray committed
501

Kyle Stewart's avatar
Kyle Stewart committed
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
function unroll_version {
    # Convert major or major.minor format to major.minor.micro using the above
    # values recursively
    # Parameters:
    #   $prefix : one of LATEST_PP or LATEST_PP3
    #   $version : major[.minor[.patch]]
    # Hence:
    #   LATEST_PP 5 -> 5.7.0
    #   LATEST 2.7 -> 2.7.11
    local prefix=$1
    local ver=$2
    local latest=${prefix}_${ver//./p}
    if [ -n "${!latest}" ]; then
        echo $(unroll_version ${prefix} ${!latest})
    else
        echo $ver
    fi
}

mattip's avatar
mattip committed
521
522
523
524
525
526
527
528
529
function install_pypy {
    # Installs pypy.org PyPy
    # Parameter $version
    # Version given in major or major.minor or major.minor.micro e.g
    # "3" or "3.7" or "3.7.1".
    # Uses $PLAT
    # sets $PYTHON_EXE variable to python executable

    local version=$1
530
    case "$PLAT" in
Andrew Murray's avatar
Andrew Murray committed
531
    "x86_64")  if [ -n "$IS_MACOS" ]; then
532
533
534
535
536
537
538
539
540
541
                   suffix="osx64";
               else
                   suffix="linux64";
               fi;;
    "i686")    suffix="linux32";;
    "ppc64le") suffix="ppc64le";;
    "s390x")    suffix="s390x";;
    "aarch64")  suffix="aarch64";;
    *) echo unknown platform "$PLAT"; exit 1;;
    esac
mattip's avatar
mattip committed
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561

    # Need to convert pypy-7.2 to pypy2.7-v7.2.0 and pypy3.6-7.3 to pypy3.6-v7.3.0
    local prefix=$(get_pypy_build_prefix $version)
    # since prefix is pypy3.6v7.2 or pypy2.7v7.2, grab the 4th (0-index) letter
    local major=${prefix:4:1}
    # get the pypy version 7.2.0
    local py_version=$(fill_pypy_ver $(echo $version | cut -f2 -d-))

    local py_build=$prefix$py_version-$suffix
    local py_zip=$py_build.tar.bz2
    local zip_path=$DOWNLOADS_SDIR/$py_zip
    mkdir -p $DOWNLOADS_SDIR
    wget -nv $PYPY_URL/${py_zip} -P $DOWNLOADS_SDIR
    untar $zip_path
    # bug/feature: pypy package for pypy3 only has bin/pypy3 :(
    if [ "$major" == "3" ] && [ ! -x "$py_build/bin/pypy" ]; then
        ln $py_build/bin/pypy3 $py_build/bin/pypy
    fi
    PYTHON_EXE=$(realpath $py_build/bin/pypy)
    $PYTHON_EXE -mensurepip
562
    $PYTHON_EXE -mpip install --upgrade pip setuptools wheel
mattip's avatar
mattip committed
563
564
565
566
567
568
    if [ "$major" == "3" ] && [ ! -x "$py_build/bin/pip" ]; then
        ln $py_build/bin/pip3 $py_build/bin/pip
    fi
    PIP_CMD=pip
}

Kyle Stewart's avatar
Kyle Stewart committed
569
570
571
572
573
574
575
576
577
578
579
580
function fill_pypy_ver {
    # Convert major or major.minor format to major.minor.micro
    # Parameters:
    #   $version : major[.minor[.patch]]
    # Hence:
    #   5 -> 5.7.0
    echo $(unroll_version LATEST_PP $1)
}

function get_pypy_build_prefix {
    # Return the file prefix of a PyPy file
    # Parameters:
mattip's avatar
mattip committed
581
    #   $version : pypy version number, for example pypy-7.2 or pypy3.6-7.2
Kyle Stewart's avatar
Kyle Stewart committed
582
    local version=$1
mattip's avatar
mattip committed
583
584
585
586
587
    if [[ $version =~ pypy([0-9]+)\.([0-9]+)-([0-9]+)\.([0-9]+) ]]; then
        local py_major=${BASH_REMATCH[1]}
        local py_minor=${BASH_REMATCH[2]}
        echo "pypy$py_major.$py_minor-v"
    elif [[ $version =~ ([0-9]+)\.([0-9]+) ]]; then
Kyle Stewart's avatar
Kyle Stewart committed
588
589
        local major=${BASH_REMATCH[1]}
        local minor=${BASH_REMATCH[2]}
Andrew Murray's avatar
Andrew Murray committed
590
591
592
        if (( $major > 6 )); then
            echo "pypy2.7-v"
        elif (( $major > 5 || ($major == 5 && $minor >= 3) )); then
Kyle Stewart's avatar
Kyle Stewart committed
593
594
595
596
597
            echo "pypy2-v"
        else
            echo "pypy-"
        fi
    else
mattip's avatar
mattip committed
598
        echo "error: expected version like pypy-7.2 or pypy3.6-7.2, got $1" 1>&2
Kyle Stewart's avatar
Kyle Stewart committed
599
600
601
        exit 1
    fi
}
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619

retry () {
    # Retry command (with arguments) up to 5 times
    # https://gist.github.com/fungusakafungus/1026804
    local retry_max=5
    local count=$retry_max
    while [ $count -gt 0 ]; do
        "$@" && break
        count=$(($count - 1))
        sleep 1
    done

    [ $count -eq 0 ] && {
        echo "Retry failed [$retry_max]: $@" >&2
        return 1
    }
    return 0
}
mattip's avatar
mattip committed
620
621
622

function install_pip {
    # Generic install pip
Matthew Brett's avatar
Matthew Brett committed
623
624
625
    echo "Deprecated - please see pip installs within the individual"
    echo "install functions for each Python type."
    echo "Multibuild itself no longer uses this function."
mattip's avatar
mattip committed
626
627
628
629
630
631
632
633
634
635
636
637
    # Gets needed version from version implied by $PYTHON_EXE
    # Installs pip into python given by $PYTHON_EXE
    # Assumes pip will be installed into same directory as $PYTHON_EXE
    check_python
    mkdir -p $DOWNLOADS_SDIR
    local py_mm=`get_py_mm`
    local get_pip_path=$DOWNLOADS_SDIR/get-pip.py
    curl $GET_PIP_URL > $get_pip_path
    # Travis VMS now install pip for system python by default - force install
    # even if installed already.
    $PYTHON_EXE $get_pip_path --ignore-installed $pip_args
    PIP_CMD=$(dirname $PYTHON_EXE)/pip$py_mm
mattip's avatar
mattip committed
638
639
640
641
    if [ "$USER" != "root" ]; then
        # inside a docker, there is no sudo but the user is already root
        PIP_CMD="sudo $PIP_CMD"
    fi
mattip's avatar
mattip committed
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
    # Append pip_args if present (avoiding trailing space cf using variable
    # above).
    if [ -n "$pip_args" ]; then
        PIP_CMD="$PIP_CMD $pip_args"
    fi
}

function check_python {
    if [ -z "$PYTHON_EXE" ]; then
        echo "PYTHON_EXE variable not defined"
        exit 1
    fi
}

function check_pip {
    if [ -z "$PIP_CMD" ]; then
        echo "PIP_CMD variable not defined"
        exit 1
    fi
}

function get_py_mm {
    check_python
    $PYTHON_EXE -c "import sys; print('{0}.{1}'.format(*sys.version_info[0:2]))"
}

668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
function cpython_path {
    # Return path to cpython given
    # * version (of form "2.7")
    # * u_width ("16" or "32" default "32")
    #
    # For back-compatibility "u" as u_width also means "32"
    local py_ver="${1:-2.7}"
    local abi_suff=m
    local u_width="${2:-${UNICODE_WIDTH}}"
    local u_suff=u
    # Python 3.8 and up no longer uses the PYMALLOC 'm' suffix
    # https://github.com/pypa/wheel/pull/303
    if [ $(lex_ver $py_ver) -ge $(lex_ver 3.8) ]; then
        abi_suff=""
    fi
    # Back-compatibility
    if [ "$u_width" == "u" ]; then u_width=32; fi
    # For Python >= 3.4, "u" suffix not meaningful
    if [ $(lex_ver $py_ver) -ge $(lex_ver 3.4) ] ||
        [ "$u_width" == "16" ]; then
        u_suff=""
mattip's avatar
typo  
mattip committed
689
    elif [ "$u_width" == "" ]; then
mattip's avatar
mattip committed
690
        u_width="32"
691
692
693
694
695
696
697
    elif [ "$u_width" != "32" ]; then
        echo "Incorrect u_width value $u_width"
        exit 1
    fi
    local no_dots=$(echo $py_ver | tr -d .)
    echo "/opt/python/cp${no_dots}-cp${no_dots}$abi_suff${u_suff}"
}