common_utils.sh 10.8 KB
Newer Older
1
#!/bin/bash
2
# Utilities for both OSX and Docker Linux
3
# Python should be on the PATH
4
5
6
7
8
9
10
11

# Only source common_utils once
if [ -n "$COMMON_UTILS_SOURCED" ]; then
    return
fi
COMMON_UTILS_SOURCED=1

# Turn on exit-if-error
12
13
set -e

14
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
15
if [ $(uname) == "Darwin" ]; then IS_OSX=1; fi
16

Matthew Brett's avatar
Matthew Brett committed
17
18
19
20
# Work round bug in travis xcode image described at
# https://github.com/direnv/direnv/issues/210
shell_session_update() { :; }

21
22
23
if [ -n "$MB_KEEP_ALIVE" ]; then
    # Start a process that runs as a keep-alive
    # to avoid travis quitting if there is no output
xoviat's avatar
xoviat committed
24

25
26
    (while true; do >&2 echo "Travis-CI keep-alive"; sleep 480; done) &
fi
xoviat's avatar
xoviat committed
27

28
29
30
31
function abspath {
    python -c "import os.path; print(os.path.abspath('$1'))"
}

32
33
34
35
36
function relpath {
    # Path of first input relative to second (or $PWD if not specified)
    python -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))"
}

Matthew Brett's avatar
Matthew Brett committed
37
38
39
40
function realpath {
    python -c "import os; print(os.path.realpath('$1'))"
}

41
42
43
44
45
46
47
48
function lex_ver {
    # Echoes dot-separated version string padded with zeros
    # Thus:
    # 3.2.1 -> 003002001
    # 3     -> 003000000
    echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}'
}

49
50
51
52
53
54
55
56
57
58
59
60
function unlex_ver {
    # Reverses lex_ver to produce major.minor.micro
    # Thus:
    # 003002001 -> 3.2.1
    # 003000000 -> 3.0.0
    echo "$((10#${1:0:3}+0)).$((10#${1:3:3}+0)).$((10#${1:6:3}+0))"
}

function strip_ver_suffix {
    echo $(unlex_ver $(lex_ver $1))
}

61
function is_function {
62
63
64
    # Echo "true" if input argument string is a function
    # Allow errors during "set -e" blocks.
    (set +e; echo $($(declare -Ff "$1") > /dev/null && echo true))
65
66
}

67
68
69
70
function gh-clone {
    git clone https://github.com/$1
}

xoviat's avatar
xoviat committed
71
72
73
74
75
76
77
78
function suppress {
    # Suppress the output of a bash command unless it fails
    out=$( ( $@ ) 2>&1 )
    ret=$?
    [ "$ret" -eq 0 ] || >&2 echo "$out" # if $? (the return of the last run command) is not zero, cat the temp file
    return "$ret" # return the exit status of the command
}

79
80
81
function rm_mkdir {
    # Remove directory if present, then make directory
    local path=$1
82
83
    if [ -z "$path" ]; then echo "Need not-empty path"; exit 1; fi
    if [ -d "$path" ]; then rm -rf $path; fi
84
85
86
    mkdir $path
}

87
88
89
90
91
function untar {
    local in_fname=$1
    if [ -z "$in_fname" ];then echo "in_fname not defined"; exit 1; fi
    local extension=${in_fname##*.}
    case $extension in
92
93
94
        tar) tar -xf $in_fname ;;
        gz|tgz) tar -zxf $in_fname ;;
        bz2) tar -jxf $in_fname ;;
95
        zip) unzip $in_fname ;;
96
        xz) unxz -c $in_fname | tar -xf ;;
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
        *) echo Did not recognize extension $extension; exit 1 ;;
    esac
}

function fetch_unpack {
    # Fetch input archive name from input URL
    # Parameters
    #    url - URL from which to fetch archive
    #    archive_fname (optional) archive name
    #
    # If `archive_fname` not specified then use basename from `url`
    # If `archive_fname` already present at download location, use that instead.
    local url=$1
    if [ -z "$url" ];then echo "url not defined"; exit 1; fi
    local archive_fname=${2:-$(basename $url)}
    local arch_sdir="${ARCHIVE_SDIR:-archives}"
    # Make the archive directory in case it doesn't exist
    mkdir -p $arch_sdir
    local out_archive="${arch_sdir}/${archive_fname}"
    # Fetch the archive if it does not exist
    if [ ! -f "$out_archive" ]; then
        curl -L $url > $out_archive
    fi
    # Unpack archive, refreshing contents
    rm_mkdir arch_tmp
    (cd arch_tmp && untar ../$out_archive && rsync --delete -avh * ..)
}

125
126
127
128
129
function clean_code {
    local repo_dir=${1:-$REPO_DIR}
    local build_commit=${2:-$BUILD_COMMIT}
    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
    [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1
130
131
132
133
134
    # The package $repo_dir may be a submodule. git submodules do not
    # have a .git directory. If $repo_dir is copied around, tools like
    # Versioneer which require that it be a git repository are unable
    # to determine the version.  Give submodule proper git directory
    fill_submodule "$repo_dir"
135
136
137
138
139
140
141
142
    (cd $repo_dir \
        && git fetch origin \
        && git checkout $build_commit \
        && git clean -fxd \
        && git reset --hard \
        && git submodule update --init --recursive)
}

143
144
145
146
147
function build_wheel_cmd {
    # Builds wheel with named command, puts into $WHEEL_SDIR
    #
    # Parameters:
    #     cmd  (optional, default "pip_wheel_cmd"
Andrew Murray's avatar
Andrew Murray committed
148
    #        Name of command for building wheel
149
    #     repo_dir  (optional, default $REPO_DIR)
150
151
152
153
154
155
    #
    # Depends on
    #     REPO_DIR  (or via input argument)
    #     WHEEL_SDIR  (optional, default "wheelhouse")
    #     BUILD_DEPENDS (optional, default "")
    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
156
157
    local cmd=${1:-pip_wheel_cmd}
    local repo_dir=${2:-$REPO_DIR}
158
159
    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
160
    if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
161
162
163
164
    if [ -n "$BUILD_DEPENDS" ]; then
        pip install $(pip_opts) $BUILD_DEPENDS
    fi
    (cd $repo_dir && $cmd $wheelhouse)
165
166
167
    repair_wheelhouse $wheelhouse
}

168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
function pip_wheel_cmd {
    local abs_wheelhouse=$1
    pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps .
}

function bdist_wheel_cmd {
    # Builds wheel with bdist_wheel, puts into wheelhouse
    #
    # It may sometimes be useful to use bdist_wheel for the wheel building
    # process.  For example, versioneer has problems with versions which are
    # fixed with bdist_wheel:
    # https://github.com/warner/python-versioneer/issues/121
    local abs_wheelhouse=$1
    python setup.py bdist_wheel
    cp dist/*.whl $abs_wheelhouse
}

185
function build_pip_wheel {
186
187
188
189
190
191
192
193
194
    # Standard wheel building command with pip wheel
    build_wheel_cmd "pip_wheel_cmd" $@
}

function build_bdist_wheel {
    # Wheel building with bdist_wheel. See bdist_wheel_cmd
    build_wheel_cmd "bdist_wheel_cmd" $@
}

195
196
197
198
199
function build_wheel {
    # Set default building method to pip
    build_pip_wheel $@
}

200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
function build_index_wheel {
    # Builds wheel from some index, usually pypi
    #
    # Parameters:
    #     project_spec
    #        requirement to install, e.g. "tornado" or "tornado==4.4.1"
    #     *args
    #        Any other arguments to be passed to pip `install`  and `wheel`
    #        commands.
    #
    # Depends on
    #     WHEEL_SDIR  (optional, default "wheelhouse")
    #     BUILD_DEPENDS (optional, default "")
    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
    #
    # You can also override `pip_opts` command to set indices other than pypi
    local project_spec=$1
    [ -z "$project_spec" ] && echo "project_spec not defined" && exit 1
    # Discard first argument to pass remainder to pip
    shift
    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
    if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
    if [ -n "$BUILD_DEPENDS" ]; then
        pip install $(pip_opts) $@ $BUILD_DEPENDS
    fi
Matthew Brett's avatar
Matthew Brett committed
225
    pip wheel $(pip_opts) $@ -w $wheelhouse --no-deps $project_spec
226
227
228
    repair_wheelhouse $wheelhouse
}

229
230
function pip_opts {
    [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL"
231
}
232

233
234
235
236
237
function get_platform {
    # Report platform as given by uname
    python -c 'import platform; print(platform.uname()[4])'
}

238
239
function install_wheel {
    # Install test dependencies and built wheel
240
    #
241
    # Pass any input flags to pip install steps
242
    #
243
    # Depends on:
244
245
246
247
    #     WHEEL_SDIR  (optional, default "wheelhouse")
    #     TEST_DEPENDS  (optional, default "")
    #     MANYLINUX_URL (optional, default "") (via pip_opts function)
    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
248
    if [ -n "$TEST_DEPENDS" ]; then
249
        pip install $(pip_opts) $@ $TEST_DEPENDS
250
251
    fi
    # Install compatible wheel
252
    pip install $(pip_opts) $@ \
253
        $(python $MULTIBUILD_DIR/supported_wheels.py $wheelhouse/*.whl)
254
}
255
256
257
258
259
260
261

function install_run {
    # Depends on function `run_tests` defined in `config.sh`
    install_wheel
    mkdir tmp_for_test
    (cd tmp_for_test && run_tests)
}
262
263
264
265
266
267
268
269
270
271
272
273

function fill_submodule {
    # Restores .git directory to submodule, if necessary
    # See:
    # http://stackoverflow.com/questions/41776331/is-there-a-way-to-reconstruct-a-git-directory-for-a-submodule
    local repo_dir="$1"
    [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
    local git_loc="$repo_dir/.git"
    # For ordinary submodule, .git is a file.
    [ -d "$git_loc" ] && return
    # Need to recreate .git directory for submodule
    local origin_url=$(cd "$repo_dir" && git config --get remote.origin.url)
Matthew Brett's avatar
Matthew Brett committed
274
275
    local repo_copy="$repo_dir-$RANDOM"
    git clone --recursive "$repo_dir" "$repo_copy"
276
    rm -rf "$repo_dir"
Matthew Brett's avatar
Matthew Brett committed
277
    mv "${repo_copy}" "$repo_dir"
278
279
    (cd "$repo_dir" && git remote set-url origin $origin_url)
}
Kyle Stewart's avatar
Kyle Stewart committed
280
281
282
283
284
285
286

PYPY_URL=https://bitbucket.org/pypy/pypy/downloads

# As of 2017-03-25, the latest verions of PyPy.
LATEST_PP_1=1.9

LATEST_PP_2p0=2.0.2
287
288
# No minor version numbers for 2.1
LATEST_PP_2p1=2.1
Kyle Stewart's avatar
Kyle Stewart committed
289
290
291
292
293
LATEST_PP_2p2=2.2.1
LATEST_PP_2p3=2.3.1
LATEST_PP_2p4=2.4.0
LATEST_PP_2p5=2.5.1
LATEST_PP_2p6=2.6.1
294
LATEST_PP_2=$LATEST_PP_2p6
Kyle Stewart's avatar
Kyle Stewart committed
295
296

LATEST_PP_4p0=4.0.1
297
LATEST_PP_4=$LATEST_PP_4p0
Kyle Stewart's avatar
Kyle Stewart committed
298
299
300
301
302
303
304

LATEST_PP_5p0=5.0.1
LATEST_PP_5p1=5.1.1
LATEST_PP_5p3=5.3.1
LATEST_PP_5p4=5.4.1
LATEST_PP_5p6=5.6.0
LATEST_PP_5p7=5.7.0
305
LATEST_PP_5=$LATEST_PP_5p7
Kyle Stewart's avatar
Kyle Stewart committed
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352

function unroll_version {
    # Convert major or major.minor format to major.minor.micro using the above
    # values recursively
    # Parameters:
    #   $prefix : one of LATEST_PP or LATEST_PP3
    #   $version : major[.minor[.patch]]
    # Hence:
    #   LATEST_PP 5 -> 5.7.0
    #   LATEST 2.7 -> 2.7.11
    local prefix=$1
    local ver=$2
    local latest=${prefix}_${ver//./p}
    if [ -n "${!latest}" ]; then
        echo $(unroll_version ${prefix} ${!latest})
    else
        echo $ver
    fi
}

function fill_pypy_ver {
    # Convert major or major.minor format to major.minor.micro
    # Parameters:
    #   $version : major[.minor[.patch]]
    # Hence:
    #   5 -> 5.7.0
    echo $(unroll_version LATEST_PP $1)
}

function get_pypy_build_prefix {
    # Return the file prefix of a PyPy file
    # Parameters:
    #   $version : pypy2 version number
    local version=$1
    if [[ $version =~ ([0-9]+)\.([0-9]+) ]]; then
        local major=${BASH_REMATCH[1]}
        local minor=${BASH_REMATCH[2]}
        if (( $major > 5 || ($major == 5 && $minor >= 3) )); then
            echo "pypy2-v"
        else
            echo "pypy-"
        fi
    else
        echo "error: expected version number, got $1" 1>&2
        exit 1
    fi
}
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370

retry () {
    # Retry command (with arguments) up to 5 times
    # https://gist.github.com/fungusakafungus/1026804
    local retry_max=5
    local count=$retry_max
    while [ $count -gt 0 ]; do
        "$@" && break
        count=$(($count - 1))
        sleep 1
    done

    [ $count -eq 0 ] && {
        echo "Retry failed [$retry_max]: $@" >&2
        return 1
    }
    return 0
}