common_utils.sh 12.6 KB
Newer Older
1
#!/bin/bash
2
# Utilities for both OSX and Docker Linux
3
# Python should be on the PATH
4
5
6
7
8
9
10
11

# Only source common_utils once
if [ -n "$COMMON_UTILS_SOURCED" ]; then
    return
fi
COMMON_UTILS_SOURCED=1

# Turn on exit-if-error
12
13
set -e

14
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
15
if [ $(uname) == "Darwin" ]; then IS_OSX=1; fi
16

Matthew Brett's avatar
Matthew Brett committed
17
18
19
20
# Work round bug in travis xcode image described at
# https://github.com/direnv/direnv/issues/210
shell_session_update() { :; }

xoviat's avatar
xoviat committed
21
22
23
24
# Workaround for https://github.com/travis-ci/travis-ci/issues/8703
# suggested by Thomas K at
# https://github.com/travis-ci/travis-ci/issues/8703#issuecomment-347881274
unset -f cd
Matthew Brett's avatar
Matthew Brett committed
25
26
unset -f pushd
unset -f popd
xoviat's avatar
xoviat committed
27

xoviat's avatar
xoviat committed
28
29
30
31
32
function start_spinner {
    if [ -n "$MB_SPINNER_PID" ]; then
        return
    fi

xoviat's avatar
xoviat committed
33
    >&2 echo "Building libraries..."
34
35
    # Start a process that runs as a keep-alive
    # to avoid travis quitting if there is no output
xoviat's avatar
xoviat committed
36
    (while true; do
xoviat's avatar
xoviat committed
37
        sleep 60
xoviat's avatar
xoviat committed
38
        >&2 echo "Still building..."
xoviat's avatar
xoviat committed
39
40
    done) &
    MB_SPINNER_PID=$!
xoviat's avatar
xoviat committed
41
    disown
xoviat's avatar
xoviat committed
42
}
xoviat's avatar
xoviat committed
43

xoviat's avatar
xoviat committed
44
45
46
47
48
49
50
function stop_spinner {
    if [ ! -n "$MB_SPINNER_PID" ]; then
        return
    fi
    
    kill $MB_SPINNER_PID
    unset MB_SPINNER_PID
xoviat's avatar
xoviat committed
51
52

    >&2 echo "Building libraries finished."
xoviat's avatar
xoviat committed
53
}
xoviat's avatar
xoviat committed
54

55
56
57
58
function abspath {
    python -c "import os.path; print(os.path.abspath('$1'))"
}

59
60
61
62
63
function relpath {
    # Path of first input relative to second (or $PWD if not specified)
    python -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))"
}

Matthew Brett's avatar
Matthew Brett committed
64
65
66
67
function realpath {
    python -c "import os; print(os.path.realpath('$1'))"
}

68
69
70
71
72
73
74
75
function lex_ver {
    # Echoes dot-separated version string padded with zeros
    # Thus:
    # 3.2.1 -> 003002001
    # 3     -> 003000000
    echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}'
}

76
77
78
79
80
81
82
83
84
85
86
87
function unlex_ver {
    # Reverses lex_ver to produce major.minor.micro
    # Thus:
    # 003002001 -> 3.2.1
    # 003000000 -> 3.0.0
    echo "$((10#${1:0:3}+0)).$((10#${1:3:3}+0)).$((10#${1:6:3}+0))"
}

function strip_ver_suffix {
    echo $(unlex_ver $(lex_ver $1))
}

88
function is_function {
89
90
    # Echo "true" if input argument string is a function
    # Allow errors during "set -e" blocks.
Matthew Brett's avatar
Matthew Brett committed
91
    (set +e; $(declare -Ff "$1" > /dev/null) && echo true)
92
93
}

94
95
96
97
function gh-clone {
    git clone https://github.com/$1
}

98
99
100
101
102
103
104
105
106
107
function set_opts {
    # Set options from input options string (in $- format).
    local opts=$1
    local chars="exhimBH"
    for (( i=0; i<${#chars}; i++ )); do
        char=${chars:$i:1}
        [ -n "${opts//[^${char}]/}" ] && set -$char || set +$char
    done
}

xoviat's avatar
xoviat committed
108
function suppress {
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    # Run a command, show output only if return code not 0.
    # Takes into account state of -e option.
    # Compare
    # https://unix.stackexchange.com/questions/256120/how-can-i-suppress-output-only-if-the-command-succeeds#256122
    # Set -e stuff agonized over in
    # https://unix.stackexchange.com/questions/296526/set-e-in-a-subshell
    local tmp=$(mktemp tmp.XXXXXXXXX) || return
    local opts=$-
    echo "Running $@"
    set +e
    ( set_opts $opts ; $@  > "$tmp" 2>&1 ) ; ret=$?
    [ "$ret" -eq 0 ] || cat "$tmp"
    rm -f "$tmp"
    set_opts $opts
    return "$ret"
xoviat's avatar
xoviat committed
124
125
}

126
127
128
function rm_mkdir {
    # Remove directory if present, then make directory
    local path=$1
129
130
    if [ -z "$path" ]; then echo "Need not-empty path"; exit 1; fi
    if [ -d "$path" ]; then rm -rf $path; fi
131
132
133
    mkdir $path
}

134
135
136
137
138
function untar {
    local in_fname=$1
    if [ -z "$in_fname" ];then echo "in_fname not defined"; exit 1; fi
    local extension=${in_fname##*.}
    case $extension in
139
140
141
        tar) tar -xf $in_fname ;;
        gz|tgz) tar -zxf $in_fname ;;
        bz2) tar -jxf $in_fname ;;
142
        zip) unzip $in_fname ;;
143
        xz) unxz -c $in_fname | tar -xf ;;
144
145
146
147
        *) echo Did not recognize extension $extension; exit 1 ;;
    esac
}

148
149
150
151
152
153
function install_rsync {
    if [ -z "$IS_OSX" ]; then
        [[ $(type -P rsync) ]] || yum install -y rsync
    fi
}

154
155
156
157
158
159
function fetch_unpack {
    # Fetch input archive name from input URL
    # Parameters
    #    url - URL from which to fetch archive
    #    archive_fname (optional) archive name
    #
160
161
    # Echos unpacked directory and file names.
    #
162
163
164
165
166
167
168
169
170
171
172
173
174
    # If `archive_fname` not specified then use basename from `url`
    # If `archive_fname` already present at download location, use that instead.
    local url=$1
    if [ -z "$url" ];then echo "url not defined"; exit 1; fi
    local archive_fname=${2:-$(basename $url)}
    local arch_sdir="${ARCHIVE_SDIR:-archives}"
    # Make the archive directory in case it doesn't exist
    mkdir -p $arch_sdir
    local out_archive="${arch_sdir}/${archive_fname}"
    # Fetch the archive if it does not exist
    if [ ! -f "$out_archive" ]; then
        curl -L $url > $out_archive
    fi
175
176
    # Unpack archive, refreshing contents, echoing dir and file
    # names.
177
    rm_mkdir arch_tmp
178
    install_rsync
179
180
181
182
    (cd arch_tmp && \
        untar ../$out_archive && \
        ls -1d * &&
        rsync --delete -ah * ..)
183
184
}

185
186
187
188
189
function clean_code {
    local repo_dir=${1:-$REPO_DIR}
    local build_commit=${2:-$BUILD_COMMIT}
    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
    [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1
190
191
192
193
194
    # The package $repo_dir may be a submodule. git submodules do not
    # have a .git directory. If $repo_dir is copied around, tools like
    # Versioneer which require that it be a git repository are unable
    # to determine the version.  Give submodule proper git directory
    fill_submodule "$repo_dir"
195
196
197
198
199
200
201
202
    (cd $repo_dir \
        && git fetch origin \
        && git checkout $build_commit \
        && git clean -fxd \
        && git reset --hard \
        && git submodule update --init --recursive)
}

203
204
205
206
207
function build_wheel_cmd {
    # Builds wheel with named command, puts into $WHEEL_SDIR
    #
    # Parameters:
    #     cmd  (optional, default "pip_wheel_cmd"
Andrew Murray's avatar
Andrew Murray committed
208
    #        Name of command for building wheel
209
    #     repo_dir  (optional, default $REPO_DIR)
210
211
212
213
214
215
    #
    # Depends on
    #     REPO_DIR  (or via input argument)
    #     WHEEL_SDIR  (optional, default "wheelhouse")
    #     BUILD_DEPENDS (optional, default "")
    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
216
217
    local cmd=${1:-pip_wheel_cmd}
    local repo_dir=${2:-$REPO_DIR}
218
219
    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
xoviat's avatar
xoviat committed
220
221
222
    start_spinner
    if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
    stop_spinner
223
224
225
226
    if [ -n "$BUILD_DEPENDS" ]; then
        pip install $(pip_opts) $BUILD_DEPENDS
    fi
    (cd $repo_dir && $cmd $wheelhouse)
227
228
229
    repair_wheelhouse $wheelhouse
}

230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
function pip_wheel_cmd {
    local abs_wheelhouse=$1
    pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps .
}

function bdist_wheel_cmd {
    # Builds wheel with bdist_wheel, puts into wheelhouse
    #
    # It may sometimes be useful to use bdist_wheel for the wheel building
    # process.  For example, versioneer has problems with versions which are
    # fixed with bdist_wheel:
    # https://github.com/warner/python-versioneer/issues/121
    local abs_wheelhouse=$1
    python setup.py bdist_wheel
    cp dist/*.whl $abs_wheelhouse
}

247
function build_pip_wheel {
248
249
250
251
252
253
254
255
256
    # Standard wheel building command with pip wheel
    build_wheel_cmd "pip_wheel_cmd" $@
}

function build_bdist_wheel {
    # Wheel building with bdist_wheel. See bdist_wheel_cmd
    build_wheel_cmd "bdist_wheel_cmd" $@
}

257
258
259
260
261
function build_wheel {
    # Set default building method to pip
    build_pip_wheel $@
}

262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
function build_index_wheel {
    # Builds wheel from some index, usually pypi
    #
    # Parameters:
    #     project_spec
    #        requirement to install, e.g. "tornado" or "tornado==4.4.1"
    #     *args
    #        Any other arguments to be passed to pip `install`  and `wheel`
    #        commands.
    #
    # Depends on
    #     WHEEL_SDIR  (optional, default "wheelhouse")
    #     BUILD_DEPENDS (optional, default "")
    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
    #
    # You can also override `pip_opts` command to set indices other than pypi
    local project_spec=$1
    [ -z "$project_spec" ] && echo "project_spec not defined" && exit 1
    # Discard first argument to pass remainder to pip
    shift
    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
xoviat's avatar
xoviat committed
283
    start_spinner
284
    if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
xoviat's avatar
xoviat committed
285
    stop_spinner
286
287
288
    if [ -n "$BUILD_DEPENDS" ]; then
        pip install $(pip_opts) $@ $BUILD_DEPENDS
    fi
Matthew Brett's avatar
Matthew Brett committed
289
    pip wheel $(pip_opts) $@ -w $wheelhouse --no-deps $project_spec
290
291
292
    repair_wheelhouse $wheelhouse
}

293
294
function pip_opts {
    [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL"
295
}
296

297
298
299
300
301
function get_platform {
    # Report platform as given by uname
    python -c 'import platform; print(platform.uname()[4])'
}

302
303
304
305
306
307
function get_distutils_platform {
    # Report platform as given by distutils get_platform.
    # This is the platform tag that pip will use.
    python -c "import distutils.util; print(distutils.util.get_platform())"
}

308
309
function install_wheel {
    # Install test dependencies and built wheel
310
    #
311
    # Pass any input flags to pip install steps
312
    #
313
    # Depends on:
314
315
316
317
    #     WHEEL_SDIR  (optional, default "wheelhouse")
    #     TEST_DEPENDS  (optional, default "")
    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
318
    if [ -n "$TEST_DEPENDS" ]; then
319
        pip install $(pip_opts) $@ $TEST_DEPENDS
320
321
    fi
    # Install compatible wheel
322
    pip install $(pip_opts) $@ \
323
        $(python $MULTIBUILD_DIR/supported_wheels.py $wheelhouse/*.whl)
324
}
325
326
327
328
329
330
331

function install_run {
    # Depends on function `run_tests` defined in `config.sh`
    install_wheel
    mkdir tmp_for_test
    (cd tmp_for_test && run_tests)
}
332
333
334
335

function fill_submodule {
    # Restores .git directory to submodule, if necessary
    # See:
Andrew Murray's avatar
Andrew Murray committed
336
    # https://stackoverflow.com/questions/41776331/is-there-a-way-to-reconstruct-a-git-directory-for-a-submodule
337
338
339
340
341
342
343
    local repo_dir="$1"
    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
    local git_loc="$repo_dir/.git"
    # For ordinary submodule, .git is a file.
    [ -d "$git_loc" ] && return
    # Need to recreate .git directory for submodule
    local origin_url=$(cd "$repo_dir" && git config --get remote.origin.url)
Matthew Brett's avatar
Matthew Brett committed
344
345
    local repo_copy="$repo_dir-$RANDOM"
    git clone --recursive "$repo_dir" "$repo_copy"
346
    rm -rf "$repo_dir"
Matthew Brett's avatar
Matthew Brett committed
347
    mv "${repo_copy}" "$repo_dir"
348
349
    (cd "$repo_dir" && git remote set-url origin $origin_url)
}
Kyle Stewart's avatar
Kyle Stewart committed
350
351
352

PYPY_URL=https://bitbucket.org/pypy/pypy/downloads

Andrew Murray's avatar
Andrew Murray committed
353
# As of 2018-04-25, the latest verions of PyPy.
Kyle Stewart's avatar
Kyle Stewart committed
354
355
356
LATEST_PP_1=1.9

LATEST_PP_2p0=2.0.2
357
358
# No minor version numbers for 2.1
LATEST_PP_2p1=2.1
Kyle Stewart's avatar
Kyle Stewart committed
359
360
361
362
363
LATEST_PP_2p2=2.2.1
LATEST_PP_2p3=2.3.1
LATEST_PP_2p4=2.4.0
LATEST_PP_2p5=2.5.1
LATEST_PP_2p6=2.6.1
364
LATEST_PP_2=$LATEST_PP_2p6
Kyle Stewart's avatar
Kyle Stewart committed
365
366

LATEST_PP_4p0=4.0.1
367
LATEST_PP_4=$LATEST_PP_4p0
Kyle Stewart's avatar
Kyle Stewart committed
368
369
370
371
372
373

LATEST_PP_5p0=5.0.1
LATEST_PP_5p1=5.1.1
LATEST_PP_5p3=5.3.1
LATEST_PP_5p4=5.4.1
LATEST_PP_5p6=5.6.0
Andrew Murray's avatar
Andrew Murray committed
374
LATEST_PP_5p7=5.7.1
Andrew Murray's avatar
Andrew Murray committed
375
376
LATEST_PP_5p8=5.8.0
LATEST_PP_5p9=5.9.0
Andrew Murray's avatar
Andrew Murray committed
377
378
LATEST_PP_5p10=5.10.1
LATEST_PP_5=$LATEST_PP_5p10
Kyle Stewart's avatar
Kyle Stewart committed
379

Andrew Murray's avatar
Andrew Murray committed
380
381
382
LATEST_PP_6p0=6.0.0
LATEST_PP_6=$LATEST_PP_6p0

Kyle Stewart's avatar
Kyle Stewart committed
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
function unroll_version {
    # Convert major or major.minor format to major.minor.micro using the above
    # values recursively
    # Parameters:
    #   $prefix : one of LATEST_PP or LATEST_PP3
    #   $version : major[.minor[.patch]]
    # Hence:
    #   LATEST_PP 5 -> 5.7.0
    #   LATEST 2.7 -> 2.7.11
    local prefix=$1
    local ver=$2
    local latest=${prefix}_${ver//./p}
    if [ -n "${!latest}" ]; then
        echo $(unroll_version ${prefix} ${!latest})
    else
        echo $ver
    fi
}

function fill_pypy_ver {
    # Convert major or major.minor format to major.minor.micro
    # Parameters:
    #   $version : major[.minor[.patch]]
    # Hence:
    #   5 -> 5.7.0
    echo $(unroll_version LATEST_PP $1)
}

function get_pypy_build_prefix {
    # Return the file prefix of a PyPy file
    # Parameters:
    #   $version : pypy2 version number
    local version=$1
    if [[ $version =~ ([0-9]+)\.([0-9]+) ]]; then
        local major=${BASH_REMATCH[1]}
        local minor=${BASH_REMATCH[2]}
        if (( $major > 5 || ($major == 5 && $minor >= 3) )); then
            echo "pypy2-v"
        else
            echo "pypy-"
        fi
    else
        echo "error: expected version number, got $1" 1>&2
        exit 1
    fi
}
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446

retry () {
    # Retry command (with arguments) up to 5 times
    # https://gist.github.com/fungusakafungus/1026804
    local retry_max=5
    local count=$retry_max
    while [ $count -gt 0 ]; do
        "$@" && break
        count=$(($count - 1))
        sleep 1
    done

    [ $count -eq 0 ] && {
        echo "Retry failed [$retry_max]: $@" >&2
        return 1
    }
    return 0
}