library_builders.sh 12 KB
Newer Older
1
# Find, load common utilties
2
# Defines IS_OSX, fetch_unpack
3
4
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
source $MULTIBUILD_DIR/common_utils.sh
xoviat's avatar
xoviat committed
5
6
7
8
9
source $MULTIBUILD_DIR/_gfortran_utils.sh

# For OpenBLAS
PLAT="${PLAT:x86_64}"
GF_LIB_URL="https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com"
10

11
# Recipes for building some libaries
Matthew Brett's avatar
Matthew Brett committed
12
OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.2.18}"
13
# We use system zlib by default - see build_new_zlib
Matthew Brett's avatar
Matthew Brett committed
14
ZLIB_VERSION="${ZLIB_VERSION:-1.2.10}"
15
16
17
LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.21}"
BZIP2_VERSION="${BZIP2_VERSION:-1.0.6}"
FREETYPE_VERSION="${FREETYPE_VERSION:-2.6.3}"
Matthew Brett's avatar
Matthew Brett committed
18
TIFF_VERSION="${TIFF_VERSION:-4.0.6}"
19
JPEG_VERSION="${JPEG_VERSION:-9b}"
20
21
22
23
24
25
OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}"
LCMS2_VERSION="${LCMS2_VERSION:-2.7}"
GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}"
LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}"
XZ_VERSION="${XZ_VERSION:-5.2.2}"
LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}"
Matthew Brett's avatar
Matthew Brett committed
26
SZIP_VERSION="${SZIP_VERSION:-2.1}"
Andrea Bedini's avatar
Andrea Bedini committed
27
HDF5_VERSION="${HDF5_VERSION:-1.8.18}"
28
LIBAEC_VERSION="${LIBAEC_VERSION:-0.3.3}"
29
LZO_VERSION=${LZO_VERSION:-2.09}
Rolando Espinoza's avatar
Rolando Espinoza committed
30
LZF_VERSION="${LZF_VERSION:-3.6}"
31
BLOSC_VERSION=${BLOSC_VERSION:-1.10.2}
Rolando Espinoza's avatar
Rolando Espinoza committed
32
SNAPPY_VERSION="${SNAPPY_VERSION:-1.1.3}"
33
CURL_VERSION=${CURL_VERSION:-7.49.1}
34
NETCDF_VERSION=${NETCDF_VERSION:-4.4.1.1}
xoviat's avatar
xoviat committed
35
SWIG_VERSION=${SWIG_VERSION:-3.0.12}
36
PCRE_VERSION=${PCRE_VERSION:-8.38}
37
SUITESPARSE_VERSION=${SUITESPARSE_VERSION:-4.5.6}
38
39
40
41
LIBTOOL_VERSION=${LIBTOOL_VERSION:-2.4.6}
RAGEL_VERSION=${RAGEL_VERSION:-6.10}
FLEX_VERSION=${FLEX_VERSION:-2.6.4}
BISON_VERSION=${BISON_VERSION:-3.0.4}
xoviat's avatar
xoviat committed
42
43
NETCDF_VERSION=${NETCDF_VERSION:-4.5.0}

Matthew Brett's avatar
Matthew Brett committed
44
45
46
OPENSSL_ROOT=openssl-1.0.2l
# Hash from https://www.openssl.org/source/openssl-1.0.2?.tar.gz.sha256
OPENSSL_HASH=ce07195b659e75f4e1db43552860070061f156a98bb37b672b101ba6e3ddf30c
47
OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source
48

Rolando Espinoza's avatar
Rolando Espinoza committed
49

Matthew Brett's avatar
Matthew Brett committed
50
BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}"
51
ARCHIVE_SDIR=${ARCHIVE_DIR:-archives}
52

53
54
55
# Set default library compilation flags for OSX
# IS_OSX defined in common_utils.sh
if [ -n "$IS_OSX" ]; then
56
    # Dual arch build by default
57
    ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"}
58
59
60
    # Only set CFLAGS, FFLAGS if they are not already defined.  Build functions
    # can override the arch flags by setting CFLAGS, FFLAGS
    export CFLAGS="${CFLAGS:-$ARCH_FLAGS}"
61
    export CXXFLAGS="${CXXFLAGS:-$ARCH_FLAGS}"
62
    export FFLAGS="${FFLAGS:-$ARCH_FLAGS}"
63
fi
64
65

function build_simple {
xoviat's avatar
xoviat committed
66
67
68
    # Example: build_simple libpng $LIBPNG_VERSION \
    #               http://download.sourceforge.net/libpng tar.gz \
    #               --additional --configure --arguments
69
70
71
    local name=$1
    local version=$2
    local url=$3
72
    local ext=${4:-tar.gz}
73
    local configure_args=${@:5}
74
75
76
77
    if [ -e "${name}-stamp" ]; then
        return
    fi
    local name_version="${name}-${version}"
78
79
    local archive=${name_version}.${ext}
    fetch_unpack $url/$archive
Matthew Brett's avatar
Matthew Brett committed
80
    (cd $name_version \
81
        && ./configure --prefix=$BUILD_PREFIX $configure_args \
Matthew Brett's avatar
Matthew Brett committed
82
83
        && make \
        && make install)
84
85
86
    touch "${name}-stamp"
}

87
88
89
90
91
92
function build_github {
    # Example: build_github fredrik-johansson/arb 2.11.1
    local path=$1
    local version=$2
    local configure_args=${@:3}
    local name=`basename "$path"`
93
94
95
96
97
98
99
    # This is tricky. If the version name starts with a "v",
    # then the archive name will not start with a "v"
    if [[ $version == v* ]]; then 
        local name_version="${name}-${version:1}"
    else
        local name_version="${name}-${version}"
    fi
100
101
102
103
104
105
106
107
108
109
110
    if [ -e "${name}-stamp" ]; then
        return
    fi
    fetch_unpack "https://github.com/${path}/archive/${version}.tar.gz"
    (cd $name_version \
        && ./configure --prefix=$BUILD_PREFIX $configure_args \
        && make \
        && make install)
    touch "${name}-stamp"
}

111
112
function build_openblas {
    if [ -e openblas-stamp ]; then return; fi
113
114
115
    if [ -n "$IS_OSX" ]; then
        brew install openblas
        brew link --force openblas
116
    else
117
        mkdir -p $ARCHIVE_SDIR
xoviat's avatar
xoviat committed
118
119
120
        local plat=${1:-$PLAT}
        local tar_path=$(abspath $(_mb_get_gf_lib "openblas-${OPENBLAS_VERSION}" "$plat"))
        (cd / && tar zxf $tar_path)
121
    fi
122
123
124
125
126
    touch openblas-stamp
}

function build_zlib {
    # Gives an old but safe version
127
    if [ -n "$IS_OSX" ]; then return; fi  # OSX has zlib already
128
    if [ -e zlib-stamp ]; then return; fi
129
    yum install -y zlib-devel
130
131
132
133
134
    touch zlib-stamp
}

function build_new_zlib {
    # Careful, this one may cause yum to segfault
135
136
    # Fossils directory should also contain latest
    build_simple zlib $ZLIB_VERSION http://zlib.net/fossils
137
138
139
140
}

function build_jpeg {
    if [ -e jpeg-stamp ]; then return; fi
141
142
    fetch_unpack http://ijg.org/files/jpegsrc.v${JPEG_VERSION}.tar.gz
    (cd jpeg-${JPEG_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
143
144
145
        && ./configure --prefix=$BUILD_PREFIX \
        && make \
        && make install)
146
147
148
149
150
151
152
153
154
    touch jpeg-stamp
}

function build_libpng {
    build_zlib
    build_simple libpng $LIBPNG_VERSION http://download.sourceforge.net/libpng
}

function build_bzip2 {
155
    if [ -n "$IS_OSX" ]; then return; fi  # OSX has bzip2 libs already
156
    if [ -e bzip2-stamp ]; then return; fi
157
    fetch_unpack http://bzip.org/${BZIP2_VERSION}/bzip2-${BZIP2_VERSION}.tar.gz
Matthew Brett's avatar
Matthew Brett committed
158
159
160
    (cd bzip2-${BZIP2_VERSION} \
        && make -f Makefile-libbz2_so \
        && make install PREFIX=$BUILD_PREFIX)
161
162
163
164
165
166
167
    touch bzip2-stamp
}

function build_tiff {
    build_zlib
    build_jpeg
    build_xz
Matthew Brett's avatar
Matthew Brett committed
168
    build_simple tiff $TIFF_VERSION http://download.osgeo.org/libtiff
169
170
}

171
function get_cmake {
Matthew Brett's avatar
Matthew Brett committed
172
    local cmake=cmake
173
    if [ -n "$IS_OSX" ]; then
174
        brew install cmake > /dev/null
175
    else
176
        yum install -y cmake28 > /dev/null
Matthew Brett's avatar
Matthew Brett committed
177
        cmake=cmake28
178
    fi
179
180
181
182
183
    echo $cmake
}

function build_openjpeg {
    if [ -e openjpeg-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
184
185
186
187
    build_zlib
    build_libpng
    build_tiff
    build_lcms2
188
    local cmake=$(get_cmake)
189
    local archive_prefix="v"
190
    local directory_prefix="openjpeg-"
191
192
    if [ $(lex_ver $OPENJPEG_VERSION) -lt $(lex_ver 2.1.1) ]; then
        archive_prefix="version."
193
        directory_prefix="openjpeg-version."
194
195
    fi
    fetch_unpack https://github.com/uclouvain/openjpeg/archive/${archive_prefix}${OPENJPEG_VERSION}.tar.gz
196
    (cd ${directory_prefix}${OPENJPEG_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
197
198
        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \
        && make install)
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
    touch openjpeg-stamp
}

function build_lcms2 {
    build_tiff
    build_simple lcms2 $LCMS2_VERSION http://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION
}

function build_giflib {
    build_simple giflib $GIFLIB_VERSION http://downloads.sourceforge.net/project/giflib
}

function build_xz {
    build_simple xz $XZ_VERSION http://tukaani.org/xz
}

function build_libwebp {
    build_libpng
    build_tiff
    build_giflib
219
    build_simple libwebp $LIBWEBP_VERSION \
xoviat's avatar
xoviat committed
220
        https://storage.googleapis.com/downloads.webmproject.org/releases/webp/ tar.gz \
221
        --enable-libwebpmux --enable-libwebpdemux 
222
223
224
225
226
227
228
229
230
231
232
}

function build_freetype {
    build_libpng
    build_bzip2
    build_simple freetype $FREETYPE_VERSION http://download.savannah.gnu.org/releases/freetype
}

function build_libyaml {
    build_simple yaml $LIBYAML_VERSION http://pyyaml.org/download/libyaml
}
Matthew Brett's avatar
Matthew Brett committed
233
234
235
236

function build_szip {
    # Build szip without encoding (patent restrictions)
    build_zlib
237
    build_simple szip $SZIP_VERSION \
xoviat's avatar
xoviat committed
238
        https://www.hdfgroup.org/ftp/lib-external/szip/ tar.gz \
239
        --enable-encoding=no
Matthew Brett's avatar
Matthew Brett committed
240
241
242
243
}

function build_hdf5 {
    if [ -e hdf5-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
244
    build_zlib
245
246
    # libaec is a drop-in replacement for szip
    build_libaec
Matthew Brett's avatar
Matthew Brett committed
247
    local hdf5_url=https://www.hdfgroup.org/ftp/HDF5/releases
248
249
    local short=$(echo $HDF5_VERSION | awk -F "." '{printf "%d.%d", $1, $2}')
    fetch_unpack $hdf5_url/hdf5-$short/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz
Matthew Brett's avatar
Matthew Brett committed
250
    (cd hdf5-$HDF5_VERSION \
Matthew Brett's avatar
Matthew Brett committed
251
        && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \
Matthew Brett's avatar
Matthew Brett committed
252
253
254
255
        && make \
        && make install)
    touch hdf5-stamp
}
256
257
258
259
260
261

function build_libaec {
    if [ -e libaec-stamp ]; then return; fi
    local root_name=libaec-0.3.3
    local tar_name=${root_name}.tar.gz
    # Note URL will change for each version
262
    fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/48398bd5b7bc05a3edb3325abfeac864/${tar_name}
263
    (cd $root_name \
Matthew Brett's avatar
Matthew Brett committed
264
        && ./configure --prefix=$BUILD_PREFIX \
265
266
267
268
        && make \
        && make install)
    touch libaec-stamp
}
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285

function build_blosc {
    if [ -e blosc-stamp ]; then return; fi
    local cmake=$(get_cmake)
    fetch_unpack https://github.com/Blosc/c-blosc/archive/v${BLOSC_VERSION}.tar.gz
    (cd c-blosc-${BLOSC_VERSION} \
        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \
        && make install)
    if [ -n "$IS_OSX" ]; then
        # Fix blosc library id bug
        for lib in $(ls ${BUILD_PREFIX}/lib/libblosc*.dylib); do
            install_name_tool -id $lib $lib
        done
    fi
    touch blosc-stamp
}

Rolando Espinoza's avatar
Rolando Espinoza committed
286
287
288
289
function build_snappy {
    build_simple snappy $SNAPPY_VERSION https://github.com/google/snappy/releases/download/$SNAPPY_VERSION
}

290
291
292
293
294
295
296
297
298
function build_lzo {
    if [ -e lzo-stamp ]; then return; fi
    fetch_unpack http://www.oberhumer.com/opensource/lzo/download/lzo-${LZO_VERSION}.tar.gz
    (cd lzo-${LZO_VERSION} \
        && ./configure --prefix=$BUILD_PREFIX --enable-shared \
        && make \
        && make install)
    touch lzo-stamp
}
299

Rolando Espinoza's avatar
Rolando Espinoza committed
300
301
302
303
function build_lzf {
    build_simple liblzf $LZF_VERSION http://dist.schmorp.de/liblzf
}

304
305
function build_curl {
    if [ -e curl-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
306
    local flags="--prefix=$BUILD_PREFIX"
Matthew Brett's avatar
Matthew Brett committed
307
    if [ -n "$IS_OSX" ]; then
308
        flags="$flags --with-darwinssl"
309
    else  # manylinux
310
        flags="$flags --with-ssl"
Matthew Brett's avatar
Matthew Brett committed
311
312
        build_openssl
    fi
313
314
    fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz
    (cd curl-${CURL_VERSION} \
315
316
317
        && if [ -z "$IS_OSX" ]; then \
        LIBS=-ldl ./configure $flags; else \
        ./configure $flags; fi\
318
319
320
321
322
        && make \
        && make install)
    touch curl-stamp
}

Matthew Brett's avatar
Matthew Brett committed
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
function check_sha256sum {
    local fname=$1
    if [ -z "$fname" ]; then echo "Need path"; exit 1; fi
    local sha256=$2
    if [ -z "$sha256" ]; then echo "Need SHA256 hash"; exit 1; fi
    echo "${sha256}  ${fname}" > ${fname}.sha256
    if [ -n "$IS_OSX" ]; then
        shasum -a 256 -c ${fname}.sha256
    else
        sha256sum -c ${fname}.sha256
    fi
    rm ${fname}.sha256
}

function build_openssl {
    if [ -e openssl-stamp ]; then return; fi
    fetch_unpack ${OPENSSL_DOWNLOAD_URL}/${OPENSSL_ROOT}.tar.gz
    check_sha256sum $ARCHIVE_SDIR/${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
    (cd ${OPENSSL_ROOT} \
        && ./config no-ssl2 no-shared -fPIC --prefix=$BUILD_PREFIX \
        && make \
        && make install)
    touch openssl-stamp
}

348
349
350
351
352
353
function build_netcdf {
    if [ -e netcdf-stamp ]; then return; fi
    build_hdf5
    build_curl
    fetch_unpack https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz
    (cd netcdf-c-${NETCDF_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
354
        && ./configure --prefix=$BUILD_PREFIX --enable-dap \
355
356
357
358
        && make \
        && make install)
    touch netcdf-stamp
}
xoviat's avatar
xoviat committed
359

360
361
function build_pcre {
    build_simple pcre $PCRE_VERSION https://ftp.pcre.org/pub/pcre/
xoviat's avatar
xoviat committed
362
363
}

xoviat's avatar
xoviat committed
364
365
366
367
function build_swig {
    if [ -n "$IS_OSX" ]; then
        brew install swig > /dev/null
    else
368
369
        build_pcre
        build_simple swig $SWIG_VERSION http://prdownloads.sourceforge.net/swig/
xoviat's avatar
xoviat committed
370
371
372
    fi
}

xoviat's avatar
xoviat committed
373
function build_suitesparse {
xoviat's avatar
xoviat committed
374
    if [ -n "$IS_OSX" ]; then
xoviat's avatar
xoviat committed
375
        brew install suite-sparse > /dev/null
xoviat's avatar
xoviat committed
376
    else
xoviat's avatar
xoviat committed
377
        yum install -y suitesparse-devel > /dev/null
xoviat's avatar
xoviat committed
378
379
    fi
}
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397

function build_libtool {
    build_simple libtool $LIBTOOL_VERSION https://ftp.gnu.org/gnu/libtool/
}

function build_ragel {
    build_simple ragel $RAGEL_VERSION http://www.colm.net/files/ragel/
}

function build_bison {
    build_simple bison $BISON_VERSION https://ftp.gnu.org/gnu/bison/
}

function build_flex {
    # the flex repository's git tags have a 'v' prefix
    build_simple flex $FLEX_VERSION \
        https://github.com/westes/flex/releases/download/v$FLEX_VERSION
}
xoviat's avatar
xoviat committed
398
399
400
401
402
403
404

function build_netcdf {
    build_zlib
    build_hdf5
    build_curl
    build_simple netcdf $NETCDF_VERSION ftp://ftp.unidata.ucar.edu/pub/netcdf
}