library_builders.sh 9.66 KB
Newer Older
1
# Find, load common utilties
2
# Defines IS_OSX, fetch_unpack
3
4
5
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
source $MULTIBUILD_DIR/common_utils.sh

6
# Recipes for building some libaries
Matthew Brett's avatar
Matthew Brett committed
7
OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.2.18}"
8
# We use system zlib by default - see build_new_zlib
Matthew Brett's avatar
Matthew Brett committed
9
ZLIB_VERSION="${ZLIB_VERSION:-1.2.10}"
10
11
12
LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.21}"
BZIP2_VERSION="${BZIP2_VERSION:-1.0.6}"
FREETYPE_VERSION="${FREETYPE_VERSION:-2.6.3}"
Matthew Brett's avatar
Matthew Brett committed
13
TIFF_VERSION="${TIFF_VERSION:-4.0.6}"
14
JPEG_VERSION="${JPEG_VERSION:-9b}"
15
16
17
18
19
20
OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}"
LCMS2_VERSION="${LCMS2_VERSION:-2.7}"
GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}"
LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}"
XZ_VERSION="${XZ_VERSION:-5.2.2}"
LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}"
Matthew Brett's avatar
Matthew Brett committed
21
SZIP_VERSION="${SZIP_VERSION:-2.1}"
Andrea Bedini's avatar
Andrea Bedini committed
22
HDF5_VERSION="${HDF5_VERSION:-1.8.18}"
23
LIBAEC_VERSION="${LIBAEC_VERSION:-0.3.3}"
24
LZO_VERSION=${LZO_VERSION:-2.09}
Rolando Espinoza's avatar
Rolando Espinoza committed
25
LZF_VERSION="${LZF_VERSION:-3.6}"
26
BLOSC_VERSION=${BLOSC_VERSION:-1.10.2}
Rolando Espinoza's avatar
Rolando Espinoza committed
27
SNAPPY_VERSION="${SNAPPY_VERSION:-1.1.3}"
28
CURL_VERSION=${CURL_VERSION:-7.49.1}
29
NETCDF_VERSION=${NETCDF_VERSION:-4.4.1.1}
Matthew Brett's avatar
Matthew Brett committed
30
31
32
OPENSSL_ROOT=openssl-1.0.2l
# Hash from https://www.openssl.org/source/openssl-1.0.2?.tar.gz.sha256
OPENSSL_HASH=ce07195b659e75f4e1db43552860070061f156a98bb37b672b101ba6e3ddf30c
33
OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source
34

Rolando Espinoza's avatar
Rolando Espinoza committed
35

Matthew Brett's avatar
Matthew Brett committed
36
BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}"
37
ARCHIVE_SDIR=${ARCHIVE_DIR:-archives}
38

39
40
41
# Set default library compilation flags for OSX
# IS_OSX defined in common_utils.sh
if [ -n "$IS_OSX" ]; then
42
    # Dual arch build by default
43
    ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"}
44
45
46
    # Only set CFLAGS, FFLAGS if they are not already defined.  Build functions
    # can override the arch flags by setting CFLAGS, FFLAGS
    export CFLAGS="${CFLAGS:-$ARCH_FLAGS}"
47
    export CXXFLAGS="${CXXFLAGS:-$ARCH_FLAGS}"
48
    export FFLAGS="${FFLAGS:-$ARCH_FLAGS}"
49
fi
50
51
52
53
54

function build_simple {
    local name=$1
    local version=$2
    local url=$3
55
    local configure_args=${@:4}
56
57
58
59
60
    if [ -e "${name}-stamp" ]; then
        return
    fi
    local name_version="${name}-${version}"
    local targz=${name_version}.tar.gz
61
    fetch_unpack $url/$targz
Matthew Brett's avatar
Matthew Brett committed
62
    (cd $name_version \
63
        && ./configure --prefix=$BUILD_PREFIX $configure_args \
Matthew Brett's avatar
Matthew Brett committed
64
65
        && make \
        && make install)
66
67
68
69
70
    touch "${name}-stamp"
}

function build_openblas {
    if [ -e openblas-stamp ]; then return; fi
71
72
73
74
75
    if [ -d "OpenBLAS" ]; then
        (cd OpenBLAS && git clean -fxd && git reset --hard)
    else
        git clone https://github.com/xianyi/OpenBLAS
    fi
76
77
78
    (cd OpenBLAS \
        && git checkout "v${OPENBLAS_VERSION}" \
        && make DYNAMIC_ARCH=1 USE_OPENMP=0 NUM_THREADS=64 > /dev/null \
Matthew Brett's avatar
Matthew Brett committed
79
        && make PREFIX=$BUILD_PREFIX install)
80
81
82
83
84
    touch openblas-stamp
}

function build_zlib {
    # Gives an old but safe version
85
    if [ -n "$IS_OSX" ]; then return; fi  # OSX has zlib already
86
    if [ -e zlib-stamp ]; then return; fi
87
    yum install -y zlib-devel
88
89
90
91
92
    touch zlib-stamp
}

function build_new_zlib {
    # Careful, this one may cause yum to segfault
93
94
    # Fossils directory should also contain latest
    build_simple zlib $ZLIB_VERSION http://zlib.net/fossils
95
96
97
98
}

function build_jpeg {
    if [ -e jpeg-stamp ]; then return; fi
99
100
    fetch_unpack http://ijg.org/files/jpegsrc.v${JPEG_VERSION}.tar.gz
    (cd jpeg-${JPEG_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
101
102
103
        && ./configure --prefix=$BUILD_PREFIX \
        && make \
        && make install)
104
105
106
107
108
109
110
111
112
    touch jpeg-stamp
}

function build_libpng {
    build_zlib
    build_simple libpng $LIBPNG_VERSION http://download.sourceforge.net/libpng
}

function build_bzip2 {
113
    if [ -n "$IS_OSX" ]; then return; fi  # OSX has bzip2 libs already
114
    if [ -e bzip2-stamp ]; then return; fi
115
    fetch_unpack http://bzip.org/${BZIP2_VERSION}/bzip2-${BZIP2_VERSION}.tar.gz
Matthew Brett's avatar
Matthew Brett committed
116
117
118
    (cd bzip2-${BZIP2_VERSION} \
        && make -f Makefile-libbz2_so \
        && make install PREFIX=$BUILD_PREFIX)
119
120
121
122
123
124
125
    touch bzip2-stamp
}

function build_tiff {
    build_zlib
    build_jpeg
    build_xz
Matthew Brett's avatar
Matthew Brett committed
126
    build_simple tiff $TIFF_VERSION http://download.osgeo.org/libtiff
127
128
}

129
function get_cmake {
Matthew Brett's avatar
Matthew Brett committed
130
    local cmake=cmake
131
    if [ -n "$IS_OSX" ]; then
132
        brew install cmake > /dev/null
133
    else
134
        yum install -y cmake28 > /dev/null
Matthew Brett's avatar
Matthew Brett committed
135
        cmake=cmake28
136
    fi
137
138
139
140
141
    echo $cmake
}

function build_openjpeg {
    if [ -e openjpeg-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
142
143
144
145
    build_zlib
    build_libpng
    build_tiff
    build_lcms2
146
    local cmake=$(get_cmake)
147
    fetch_unpack https://github.com/uclouvain/openjpeg/archive/version.${OPENJPEG_VERSION}.tar.gz
Matthew Brett's avatar
Matthew Brett committed
148
149
150
    (cd openjpeg-version.${OPENJPEG_VERSION} \
        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \
        && make install)
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
    touch openjpeg-stamp
}

function build_lcms2 {
    build_tiff
    build_simple lcms2 $LCMS2_VERSION http://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION
}

function build_giflib {
    build_simple giflib $GIFLIB_VERSION http://downloads.sourceforge.net/project/giflib
}

function build_xz {
    build_simple xz $XZ_VERSION http://tukaani.org/xz
}

function build_libwebp {
    if [ -e libwebp-stamp ]; then return; fi
    build_libpng
    build_tiff
    build_giflib
172
    fetch_unpack https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${LIBWEBP_VERSION}.tar.gz
173
    (cd libwebp-${LIBWEBP_VERSION} && \
Matthew Brett's avatar
Matthew Brett committed
174
175
176
        ./configure --enable-libwebpmux --enable-libwebpdemux --prefix=$BUILD_PREFIX \
        && make \
        && make install)
177
178
179
180
181
182
183
184
185
186
187
188
    touch libwebp-stamp
}

function build_freetype {
    build_libpng
    build_bzip2
    build_simple freetype $FREETYPE_VERSION http://download.savannah.gnu.org/releases/freetype
}

function build_libyaml {
    build_simple yaml $LIBYAML_VERSION http://pyyaml.org/download/libyaml
}
Matthew Brett's avatar
Matthew Brett committed
189
190
191
192
193
194

function build_szip {
    # Build szip without encoding (patent restrictions)
    if [ -e szip-stamp ]; then return; fi
    build_zlib
    local szip_url=https://www.hdfgroup.org/ftp/lib-external/szip/
195
    fetch_unpack ${szip_url}/$SZIP_VERSION/src/szip-$SZIP_VERSION.tar.gz
Matthew Brett's avatar
Matthew Brett committed
196
    (cd szip-$SZIP_VERSION \
Matthew Brett's avatar
Matthew Brett committed
197
        && ./configure --enable-encoding=no --prefix=$BUILD_PREFIX \
Matthew Brett's avatar
Matthew Brett committed
198
199
200
201
202
203
204
        && make \
        && make install)
    touch szip-stamp
}

function build_hdf5 {
    if [ -e hdf5-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
205
    build_zlib
206
207
    # libaec is a drop-in replacement for szip
    build_libaec
Matthew Brett's avatar
Matthew Brett committed
208
    local hdf5_url=https://www.hdfgroup.org/ftp/HDF5/releases
209
210
    local short=$(echo $HDF5_VERSION | awk -F "." '{printf "%d.%d", $1, $2}')
    fetch_unpack $hdf5_url/hdf5-$short/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz
Matthew Brett's avatar
Matthew Brett committed
211
    (cd hdf5-$HDF5_VERSION \
Matthew Brett's avatar
Matthew Brett committed
212
        && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \
Matthew Brett's avatar
Matthew Brett committed
213
214
215
216
        && make \
        && make install)
    touch hdf5-stamp
}
217
218
219
220
221
222

function build_libaec {
    if [ -e libaec-stamp ]; then return; fi
    local root_name=libaec-0.3.3
    local tar_name=${root_name}.tar.gz
    # Note URL will change for each version
223
    fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/48398bd5b7bc05a3edb3325abfeac864/${tar_name}
224
    (cd $root_name \
Matthew Brett's avatar
Matthew Brett committed
225
        && ./configure --prefix=$BUILD_PREFIX \
226
227
228
229
        && make \
        && make install)
    touch libaec-stamp
}
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246

function build_blosc {
    if [ -e blosc-stamp ]; then return; fi
    local cmake=$(get_cmake)
    fetch_unpack https://github.com/Blosc/c-blosc/archive/v${BLOSC_VERSION}.tar.gz
    (cd c-blosc-${BLOSC_VERSION} \
        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \
        && make install)
    if [ -n "$IS_OSX" ]; then
        # Fix blosc library id bug
        for lib in $(ls ${BUILD_PREFIX}/lib/libblosc*.dylib); do
            install_name_tool -id $lib $lib
        done
    fi
    touch blosc-stamp
}

Rolando Espinoza's avatar
Rolando Espinoza committed
247
248
249
250
function build_snappy {
    build_simple snappy $SNAPPY_VERSION https://github.com/google/snappy/releases/download/$SNAPPY_VERSION
}

251
252
253
254
255
256
257
258
259
function build_lzo {
    if [ -e lzo-stamp ]; then return; fi
    fetch_unpack http://www.oberhumer.com/opensource/lzo/download/lzo-${LZO_VERSION}.tar.gz
    (cd lzo-${LZO_VERSION} \
        && ./configure --prefix=$BUILD_PREFIX --enable-shared \
        && make \
        && make install)
    touch lzo-stamp
}
260

Rolando Espinoza's avatar
Rolando Espinoza committed
261
262
263
264
function build_lzf {
    build_simple liblzf $LZF_VERSION http://dist.schmorp.de/liblzf
}

265
266
function build_curl {
    if [ -e curl-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
267
    local flags="--prefix=$BUILD_PREFIX"
Matthew Brett's avatar
Matthew Brett committed
268
    if [ -n "$IS_OSX" ]; then
269
        flags="$flags --with-darwinssl"
270
    else  # manylinux
271
        flags="$flags --with-ssl"
Matthew Brett's avatar
Matthew Brett committed
272
273
        build_openssl
    fi
274
275
    fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz
    (cd curl-${CURL_VERSION} \
276
277
278
        && if [ -z "$IS_OSX" ]; then \
        LIBS=-ldl ./configure $flags; else \
        ./configure $flags; fi\
279
280
281
282
283
        && make \
        && make install)
    touch curl-stamp
}

Matthew Brett's avatar
Matthew Brett committed
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
function check_sha256sum {
    local fname=$1
    if [ -z "$fname" ]; then echo "Need path"; exit 1; fi
    local sha256=$2
    if [ -z "$sha256" ]; then echo "Need SHA256 hash"; exit 1; fi
    echo "${sha256}  ${fname}" > ${fname}.sha256
    if [ -n "$IS_OSX" ]; then
        shasum -a 256 -c ${fname}.sha256
    else
        sha256sum -c ${fname}.sha256
    fi
    rm ${fname}.sha256
}

function build_openssl {
    if [ -e openssl-stamp ]; then return; fi
    fetch_unpack ${OPENSSL_DOWNLOAD_URL}/${OPENSSL_ROOT}.tar.gz
    check_sha256sum $ARCHIVE_SDIR/${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
    (cd ${OPENSSL_ROOT} \
        && ./config no-ssl2 no-shared -fPIC --prefix=$BUILD_PREFIX \
        && make \
        && make install)
    touch openssl-stamp
}

309
310
311
312
313
314
function build_netcdf {
    if [ -e netcdf-stamp ]; then return; fi
    build_hdf5
    build_curl
    fetch_unpack https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz
    (cd netcdf-c-${NETCDF_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
315
        && ./configure --prefix=$BUILD_PREFIX --enable-dap \
316
317
318
319
        && make \
        && make install)
    touch netcdf-stamp
}