library_builders.sh 10.9 KB
Newer Older
1
# Find, load common utilties
2
# Defines IS_OSX, fetch_unpack
3
4
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
source $MULTIBUILD_DIR/common_utils.sh
xoviat's avatar
xoviat committed
5
6
7
8
9
source $MULTIBUILD_DIR/_gfortran_utils.sh

# For OpenBLAS
PLAT="${PLAT:x86_64}"
GF_LIB_URL="https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com"
10

11
# Recipes for building some libaries
Matthew Brett's avatar
Matthew Brett committed
12
OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.2.18}"
13
# We use system zlib by default - see build_new_zlib
Matthew Brett's avatar
Matthew Brett committed
14
ZLIB_VERSION="${ZLIB_VERSION:-1.2.10}"
15
16
17
LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.21}"
BZIP2_VERSION="${BZIP2_VERSION:-1.0.6}"
FREETYPE_VERSION="${FREETYPE_VERSION:-2.6.3}"
Matthew Brett's avatar
Matthew Brett committed
18
TIFF_VERSION="${TIFF_VERSION:-4.0.6}"
19
JPEG_VERSION="${JPEG_VERSION:-9b}"
20
21
22
23
24
25
OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}"
LCMS2_VERSION="${LCMS2_VERSION:-2.7}"
GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}"
LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}"
XZ_VERSION="${XZ_VERSION:-5.2.2}"
LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}"
Matthew Brett's avatar
Matthew Brett committed
26
SZIP_VERSION="${SZIP_VERSION:-2.1}"
Andrea Bedini's avatar
Andrea Bedini committed
27
HDF5_VERSION="${HDF5_VERSION:-1.8.18}"
28
LIBAEC_VERSION="${LIBAEC_VERSION:-0.3.3}"
29
LZO_VERSION=${LZO_VERSION:-2.09}
Rolando Espinoza's avatar
Rolando Espinoza committed
30
LZF_VERSION="${LZF_VERSION:-3.6}"
31
BLOSC_VERSION=${BLOSC_VERSION:-1.10.2}
Rolando Espinoza's avatar
Rolando Espinoza committed
32
SNAPPY_VERSION="${SNAPPY_VERSION:-1.1.3}"
33
CURL_VERSION=${CURL_VERSION:-7.49.1}
34
NETCDF_VERSION=${NETCDF_VERSION:-4.4.1.1}
xoviat's avatar
xoviat committed
35
SWIG_VERSION=${SWIG_VERSION:-3.0.12}
36
PCRE_VERSION=${PCRE_VERSION:-8.38}
37
SUITESPARSE_VERSION=${SUITESPARSE_VERSION:-4.5.6}
Matthew Brett's avatar
Matthew Brett committed
38
39
40
OPENSSL_ROOT=openssl-1.0.2l
# Hash from https://www.openssl.org/source/openssl-1.0.2?.tar.gz.sha256
OPENSSL_HASH=ce07195b659e75f4e1db43552860070061f156a98bb37b672b101ba6e3ddf30c
41
OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source
42

Rolando Espinoza's avatar
Rolando Espinoza committed
43

Matthew Brett's avatar
Matthew Brett committed
44
BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}"
45
ARCHIVE_SDIR=${ARCHIVE_DIR:-archives}
46

47
48
49
# Set default library compilation flags for OSX
# IS_OSX defined in common_utils.sh
if [ -n "$IS_OSX" ]; then
50
    # Dual arch build by default
51
    ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"}
52
53
54
    # Only set CFLAGS, FFLAGS if they are not already defined.  Build functions
    # can override the arch flags by setting CFLAGS, FFLAGS
    export CFLAGS="${CFLAGS:-$ARCH_FLAGS}"
55
    export CXXFLAGS="${CXXFLAGS:-$ARCH_FLAGS}"
56
    export FFLAGS="${FFLAGS:-$ARCH_FLAGS}"
57
fi
58
59

function build_simple {
xoviat's avatar
xoviat committed
60
61
62
    # Example: build_simple libpng $LIBPNG_VERSION \
    #               http://download.sourceforge.net/libpng tar.gz \
    #               --additional --configure --arguments
63
64
65
    local name=$1
    local version=$2
    local url=$3
66
    local ext=${4:-tar.gz}
67
    local configure_args=${@:5}
68
69
70
71
    if [ -e "${name}-stamp" ]; then
        return
    fi
    local name_version="${name}-${version}"
72
73
    local archive=${name_version}.${ext}
    fetch_unpack $url/$archive
Matthew Brett's avatar
Matthew Brett committed
74
    (cd $name_version \
75
        && ./configure --prefix=$BUILD_PREFIX $configure_args \
Matthew Brett's avatar
Matthew Brett committed
76
77
        && make \
        && make install)
78
79
80
    touch "${name}-stamp"
}

81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
function build_github {
    # Example: build_github fredrik-johansson/arb 2.11.1
    local path=$1
    local version=$2
    local configure_args=${@:3}
    local name=`basename "$path"`
    local name_version="${name}-${version}"
    if [ -e "${name}-stamp" ]; then
        return
    fi
    fetch_unpack "https://github.com/${path}/archive/${version}.tar.gz"
    (cd $name_version \
        && ./configure --prefix=$BUILD_PREFIX $configure_args \
        && make \
        && make install)
    touch "${name}-stamp"
}

99
100
function build_openblas {
    if [ -e openblas-stamp ]; then return; fi
101
102
103
    if [ -n "$IS_OSX" ]; then
        brew install openblas
        brew link --force openblas
104
    else
105
        mkdir -p $ARCHIVE_SDIR
xoviat's avatar
xoviat committed
106
107
108
        local plat=${1:-$PLAT}
        local tar_path=$(abspath $(_mb_get_gf_lib "openblas-${OPENBLAS_VERSION}" "$plat"))
        (cd / && tar zxf $tar_path)
109
    fi
110
111
112
113
114
    touch openblas-stamp
}

function build_zlib {
    # Gives an old but safe version
115
    if [ -n "$IS_OSX" ]; then return; fi  # OSX has zlib already
116
    if [ -e zlib-stamp ]; then return; fi
117
    yum install -y zlib-devel
118
119
120
121
122
    touch zlib-stamp
}

function build_new_zlib {
    # Careful, this one may cause yum to segfault
123
124
    # Fossils directory should also contain latest
    build_simple zlib $ZLIB_VERSION http://zlib.net/fossils
125
126
127
128
}

function build_jpeg {
    if [ -e jpeg-stamp ]; then return; fi
129
130
    fetch_unpack http://ijg.org/files/jpegsrc.v${JPEG_VERSION}.tar.gz
    (cd jpeg-${JPEG_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
131
132
133
        && ./configure --prefix=$BUILD_PREFIX \
        && make \
        && make install)
134
135
136
137
138
139
140
141
142
    touch jpeg-stamp
}

function build_libpng {
    build_zlib
    build_simple libpng $LIBPNG_VERSION http://download.sourceforge.net/libpng
}

function build_bzip2 {
143
    if [ -n "$IS_OSX" ]; then return; fi  # OSX has bzip2 libs already
144
    if [ -e bzip2-stamp ]; then return; fi
145
    fetch_unpack http://bzip.org/${BZIP2_VERSION}/bzip2-${BZIP2_VERSION}.tar.gz
Matthew Brett's avatar
Matthew Brett committed
146
147
148
    (cd bzip2-${BZIP2_VERSION} \
        && make -f Makefile-libbz2_so \
        && make install PREFIX=$BUILD_PREFIX)
149
150
151
152
153
154
155
    touch bzip2-stamp
}

function build_tiff {
    build_zlib
    build_jpeg
    build_xz
Matthew Brett's avatar
Matthew Brett committed
156
    build_simple tiff $TIFF_VERSION http://download.osgeo.org/libtiff
157
158
}

159
function get_cmake {
Matthew Brett's avatar
Matthew Brett committed
160
    local cmake=cmake
161
    if [ -n "$IS_OSX" ]; then
162
        brew install cmake > /dev/null
163
    else
164
        yum install -y cmake28 > /dev/null
Matthew Brett's avatar
Matthew Brett committed
165
        cmake=cmake28
166
    fi
167
168
169
170
171
    echo $cmake
}

function build_openjpeg {
    if [ -e openjpeg-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
172
173
174
175
    build_zlib
    build_libpng
    build_tiff
    build_lcms2
176
    local cmake=$(get_cmake)
177
    local archive_prefix="v"
178
    local directory_prefix="openjpeg-"
179
180
    if [ $(lex_ver $OPENJPEG_VERSION) -lt $(lex_ver 2.1.1) ]; then
        archive_prefix="version."
181
        directory_prefix="openjpeg-version."
182
183
    fi
    fetch_unpack https://github.com/uclouvain/openjpeg/archive/${archive_prefix}${OPENJPEG_VERSION}.tar.gz
184
    (cd ${directory_prefix}${OPENJPEG_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
185
186
        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \
        && make install)
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
    touch openjpeg-stamp
}

function build_lcms2 {
    build_tiff
    build_simple lcms2 $LCMS2_VERSION http://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION
}

function build_giflib {
    build_simple giflib $GIFLIB_VERSION http://downloads.sourceforge.net/project/giflib
}

function build_xz {
    build_simple xz $XZ_VERSION http://tukaani.org/xz
}

function build_libwebp {
    build_libpng
    build_tiff
    build_giflib
207
    build_simple libwebp $LIBWEBP_VERSION \
xoviat's avatar
xoviat committed
208
        https://storage.googleapis.com/downloads.webmproject.org/releases/webp/ tar.gz \
209
        --enable-libwebpmux --enable-libwebpdemux 
210
211
212
213
214
215
216
217
218
219
220
}

function build_freetype {
    build_libpng
    build_bzip2
    build_simple freetype $FREETYPE_VERSION http://download.savannah.gnu.org/releases/freetype
}

function build_libyaml {
    build_simple yaml $LIBYAML_VERSION http://pyyaml.org/download/libyaml
}
Matthew Brett's avatar
Matthew Brett committed
221
222
223
224

function build_szip {
    # Build szip without encoding (patent restrictions)
    build_zlib
225
    build_simple szip $SZIP_VERSION \
xoviat's avatar
xoviat committed
226
        https://www.hdfgroup.org/ftp/lib-external/szip/ tar.gz \
227
        --enable-encoding=no
Matthew Brett's avatar
Matthew Brett committed
228
229
230
231
}

function build_hdf5 {
    if [ -e hdf5-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
232
    build_zlib
233
234
    # libaec is a drop-in replacement for szip
    build_libaec
Matthew Brett's avatar
Matthew Brett committed
235
    local hdf5_url=https://www.hdfgroup.org/ftp/HDF5/releases
236
237
    local short=$(echo $HDF5_VERSION | awk -F "." '{printf "%d.%d", $1, $2}')
    fetch_unpack $hdf5_url/hdf5-$short/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz
Matthew Brett's avatar
Matthew Brett committed
238
    (cd hdf5-$HDF5_VERSION \
Matthew Brett's avatar
Matthew Brett committed
239
        && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \
Matthew Brett's avatar
Matthew Brett committed
240
241
242
243
        && make \
        && make install)
    touch hdf5-stamp
}
244
245
246
247
248
249

function build_libaec {
    if [ -e libaec-stamp ]; then return; fi
    local root_name=libaec-0.3.3
    local tar_name=${root_name}.tar.gz
    # Note URL will change for each version
250
    fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/48398bd5b7bc05a3edb3325abfeac864/${tar_name}
251
    (cd $root_name \
Matthew Brett's avatar
Matthew Brett committed
252
        && ./configure --prefix=$BUILD_PREFIX \
253
254
255
256
        && make \
        && make install)
    touch libaec-stamp
}
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273

function build_blosc {
    if [ -e blosc-stamp ]; then return; fi
    local cmake=$(get_cmake)
    fetch_unpack https://github.com/Blosc/c-blosc/archive/v${BLOSC_VERSION}.tar.gz
    (cd c-blosc-${BLOSC_VERSION} \
        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \
        && make install)
    if [ -n "$IS_OSX" ]; then
        # Fix blosc library id bug
        for lib in $(ls ${BUILD_PREFIX}/lib/libblosc*.dylib); do
            install_name_tool -id $lib $lib
        done
    fi
    touch blosc-stamp
}

Rolando Espinoza's avatar
Rolando Espinoza committed
274
275
276
277
function build_snappy {
    build_simple snappy $SNAPPY_VERSION https://github.com/google/snappy/releases/download/$SNAPPY_VERSION
}

278
279
280
281
282
283
284
285
286
function build_lzo {
    if [ -e lzo-stamp ]; then return; fi
    fetch_unpack http://www.oberhumer.com/opensource/lzo/download/lzo-${LZO_VERSION}.tar.gz
    (cd lzo-${LZO_VERSION} \
        && ./configure --prefix=$BUILD_PREFIX --enable-shared \
        && make \
        && make install)
    touch lzo-stamp
}
287

Rolando Espinoza's avatar
Rolando Espinoza committed
288
289
290
291
function build_lzf {
    build_simple liblzf $LZF_VERSION http://dist.schmorp.de/liblzf
}

292
293
function build_curl {
    if [ -e curl-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
294
    local flags="--prefix=$BUILD_PREFIX"
Matthew Brett's avatar
Matthew Brett committed
295
    if [ -n "$IS_OSX" ]; then
296
        flags="$flags --with-darwinssl"
297
    else  # manylinux
298
        flags="$flags --with-ssl"
Matthew Brett's avatar
Matthew Brett committed
299
300
        build_openssl
    fi
301
302
    fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz
    (cd curl-${CURL_VERSION} \
303
304
305
        && if [ -z "$IS_OSX" ]; then \
        LIBS=-ldl ./configure $flags; else \
        ./configure $flags; fi\
306
307
308
309
310
        && make \
        && make install)
    touch curl-stamp
}

Matthew Brett's avatar
Matthew Brett committed
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
function check_sha256sum {
    local fname=$1
    if [ -z "$fname" ]; then echo "Need path"; exit 1; fi
    local sha256=$2
    if [ -z "$sha256" ]; then echo "Need SHA256 hash"; exit 1; fi
    echo "${sha256}  ${fname}" > ${fname}.sha256
    if [ -n "$IS_OSX" ]; then
        shasum -a 256 -c ${fname}.sha256
    else
        sha256sum -c ${fname}.sha256
    fi
    rm ${fname}.sha256
}

function build_openssl {
    if [ -e openssl-stamp ]; then return; fi
    fetch_unpack ${OPENSSL_DOWNLOAD_URL}/${OPENSSL_ROOT}.tar.gz
    check_sha256sum $ARCHIVE_SDIR/${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
    (cd ${OPENSSL_ROOT} \
        && ./config no-ssl2 no-shared -fPIC --prefix=$BUILD_PREFIX \
        && make \
        && make install)
    touch openssl-stamp
}

336
337
338
339
340
341
function build_netcdf {
    if [ -e netcdf-stamp ]; then return; fi
    build_hdf5
    build_curl
    fetch_unpack https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz
    (cd netcdf-c-${NETCDF_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
342
        && ./configure --prefix=$BUILD_PREFIX --enable-dap \
343
344
345
346
        && make \
        && make install)
    touch netcdf-stamp
}
xoviat's avatar
xoviat committed
347

348
349
function build_pcre {
    build_simple pcre $PCRE_VERSION https://ftp.pcre.org/pub/pcre/
xoviat's avatar
xoviat committed
350
351
}

xoviat's avatar
xoviat committed
352
353
354
355
function build_swig {
    if [ -n "$IS_OSX" ]; then
        brew install swig > /dev/null
    else
356
357
        build_pcre
        build_simple swig $SWIG_VERSION http://prdownloads.sourceforge.net/swig/
xoviat's avatar
xoviat committed
358
359
360
    fi
}

xoviat's avatar
xoviat committed
361
function build_suitesparse {
xoviat's avatar
xoviat committed
362
    if [ -n "$IS_OSX" ]; then
xoviat's avatar
xoviat committed
363
        brew install suite-sparse > /dev/null
xoviat's avatar
xoviat committed
364
    else
xoviat's avatar
xoviat committed
365
        yum install -y suitesparse-devel > /dev/null
xoviat's avatar
xoviat committed
366
367
    fi
}