library_builders.sh 9.02 KB
Newer Older
1
# Find, load common utilties
2
# Defines IS_OSX, fetch_unpack
3
4
5
MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}")
source $MULTIBUILD_DIR/common_utils.sh

6
# Recipes for building some libaries
Matthew Brett's avatar
Matthew Brett committed
7
OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.2.18}"
8
9
10
11
12
# We use system zlib by default - see build_new_zlib
ZLIB_VERSION="${ZLIB_VERSION:-1.2.8}"
LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.21}"
BZIP2_VERSION="${BZIP2_VERSION:-1.0.6}"
FREETYPE_VERSION="${FREETYPE_VERSION:-2.6.3}"
Matthew Brett's avatar
Matthew Brett committed
13
TIFF_VERSION="${TIFF_VERSION:-4.0.6}"
14
JPEG_VERSION="${JPEG_VERSION:-9b}"
15
16
17
18
19
20
OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}"
LCMS2_VERSION="${LCMS2_VERSION:-2.7}"
GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}"
LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}"
XZ_VERSION="${XZ_VERSION:-5.2.2}"
LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}"
Matthew Brett's avatar
Matthew Brett committed
21
22
SZIP_VERSION="${SZIP_VERSION:-2.1}"
HDF5_VERSION="${HDF5_VERSION:-1.8.17}"
23
LIBAEC_VERSION="${LIBAEC_VERSION:-0.3.3}"
24
25
LZO_VERSION=${LZO_VERSION:-2.09}
BLOSC_VERSION=${BLOSC_VERSION:-1.10.2}
26
27
CURL_VERSION=${CURL_VERSION:-7.43.0}
NETCDF_VERSION=${NETCDF_VERSION:-4.4.1}
Matthew Brett's avatar
Matthew Brett committed
28
29
30
OPENSSL_ROOT=openssl-1.0.2j
OPENSSL_HASH=e7aff292be21c259c6af26469c7a9b3ba26e9abaaffd325e3dccc9785256c431
OPENSSL_DOWNLOAD_URL=ftp://ftp.openssl.org/source
31

Matthew Brett's avatar
Matthew Brett committed
32
BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}"
33
ARCHIVE_SDIR=${ARCHIVE_DIR:-archives}
34

35
36
37
# Set default library compilation flags for OSX
# IS_OSX defined in common_utils.sh
if [ -n "$IS_OSX" ]; then
38
    # Dual arch build by default
39
    ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"}
40
41
42
    # Only set CFLAGS, FFLAGS if they are not already defined.  Build functions
    # can override the arch flags by setting CFLAGS, FFLAGS
    export CFLAGS="${CFLAGS:-$ARCH_FLAGS}"
43
    export CXXFLAGS="${CXXFLAGS:-$ARCH_FLAGS}"
44
    export FFLAGS="${FFLAGS:-$ARCH_FLAGS}"
45
fi
46
47
48
49
50
51
52
53
54
55

function build_simple {
    local name=$1
    local version=$2
    local url=$3
    if [ -e "${name}-stamp" ]; then
        return
    fi
    local name_version="${name}-${version}"
    local targz=${name_version}.tar.gz
56
    fetch_unpack $url/$targz
Matthew Brett's avatar
Matthew Brett committed
57
58
59
60
    (cd $name_version \
        && ./configure --prefix=$BUILD_PREFIX \
        && make \
        && make install)
61
62
63
64
65
    touch "${name}-stamp"
}

function build_openblas {
    if [ -e openblas-stamp ]; then return; fi
66
67
68
69
70
    if [ -d "OpenBLAS" ]; then
        (cd OpenBLAS && git clean -fxd && git reset --hard)
    else
        git clone https://github.com/xianyi/OpenBLAS
    fi
71
72
73
    (cd OpenBLAS \
        && git checkout "v${OPENBLAS_VERSION}" \
        && make DYNAMIC_ARCH=1 USE_OPENMP=0 NUM_THREADS=64 > /dev/null \
Matthew Brett's avatar
Matthew Brett committed
74
        && make PREFIX=$BUILD_PREFIX install)
75
76
77
78
79
    touch openblas-stamp
}

function build_zlib {
    # Gives an old but safe version
80
    if [ -n "$IS_OSX" ]; then return; fi  # OSX has zlib already
81
    if [ -e zlib-stamp ]; then return; fi
82
    yum install -y zlib-devel
83
84
85
86
87
88
89
90
91
92
    touch zlib-stamp
}

function build_new_zlib {
    # Careful, this one may cause yum to segfault
    build_simple zlib $ZLIB_VERSION http://zlib.net
}

function build_jpeg {
    if [ -e jpeg-stamp ]; then return; fi
93
94
    fetch_unpack http://ijg.org/files/jpegsrc.v${JPEG_VERSION}.tar.gz
    (cd jpeg-${JPEG_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
95
96
97
        && ./configure --prefix=$BUILD_PREFIX \
        && make \
        && make install)
98
99
100
101
102
103
104
105
106
    touch jpeg-stamp
}

function build_libpng {
    build_zlib
    build_simple libpng $LIBPNG_VERSION http://download.sourceforge.net/libpng
}

function build_bzip2 {
107
    if [ -n "$IS_OSX" ]; then return; fi  # OSX has bzip2 libs already
108
    if [ -e bzip2-stamp ]; then return; fi
109
    fetch_unpack http://bzip.org/${BZIP2_VERSION}/bzip2-${BZIP2_VERSION}.tar.gz
Matthew Brett's avatar
Matthew Brett committed
110
111
112
    (cd bzip2-${BZIP2_VERSION} \
        && make -f Makefile-libbz2_so \
        && make install PREFIX=$BUILD_PREFIX)
113
114
115
116
117
118
119
    touch bzip2-stamp
}

function build_tiff {
    build_zlib
    build_jpeg
    build_xz
Matthew Brett's avatar
Matthew Brett committed
120
    build_simple tiff $TIFF_VERSION http://download.osgeo.org/libtiff
121
122
}

123
function get_cmake {
Matthew Brett's avatar
Matthew Brett committed
124
    local cmake=cmake
125
    if [ -n "$IS_OSX" ]; then
126
        brew install cmake > /dev/null
127
    else
128
        yum install -y cmake28 > /dev/null
Matthew Brett's avatar
Matthew Brett committed
129
        cmake=cmake28
130
    fi
131
132
133
134
135
    echo $cmake
}

function build_openjpeg {
    if [ -e openjpeg-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
136
137
138
139
    build_zlib
    build_libpng
    build_tiff
    build_lcms2
140
    local cmake=$(get_cmake)
141
    fetch_unpack https://github.com/uclouvain/openjpeg/archive/version.${OPENJPEG_VERSION}.tar.gz
Matthew Brett's avatar
Matthew Brett committed
142
143
144
    (cd openjpeg-version.${OPENJPEG_VERSION} \
        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \
        && make install)
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
    touch openjpeg-stamp
}

function build_lcms2 {
    build_tiff
    build_simple lcms2 $LCMS2_VERSION http://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION
}

function build_giflib {
    build_simple giflib $GIFLIB_VERSION http://downloads.sourceforge.net/project/giflib
}

function build_xz {
    build_simple xz $XZ_VERSION http://tukaani.org/xz
}

function build_libwebp {
    if [ -e libwebp-stamp ]; then return; fi
    build_libpng
    build_tiff
    build_giflib
166
    fetch_unpack https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${LIBWEBP_VERSION}.tar.gz
167
    (cd libwebp-${LIBWEBP_VERSION} && \
Matthew Brett's avatar
Matthew Brett committed
168
169
170
        ./configure --enable-libwebpmux --enable-libwebpdemux --prefix=$BUILD_PREFIX \
        && make \
        && make install)
171
172
173
174
175
176
177
178
179
180
181
182
    touch libwebp-stamp
}

function build_freetype {
    build_libpng
    build_bzip2
    build_simple freetype $FREETYPE_VERSION http://download.savannah.gnu.org/releases/freetype
}

function build_libyaml {
    build_simple yaml $LIBYAML_VERSION http://pyyaml.org/download/libyaml
}
Matthew Brett's avatar
Matthew Brett committed
183
184
185
186
187
188

function build_szip {
    # Build szip without encoding (patent restrictions)
    if [ -e szip-stamp ]; then return; fi
    build_zlib
    local szip_url=https://www.hdfgroup.org/ftp/lib-external/szip/
189
    fetch_unpack ${szip_url}/$SZIP_VERSION/src/szip-$SZIP_VERSION.tar.gz
Matthew Brett's avatar
Matthew Brett committed
190
    (cd szip-$SZIP_VERSION \
Matthew Brett's avatar
Matthew Brett committed
191
        && ./configure --enable-encoding=no --prefix=$BUILD_PREFIX \
Matthew Brett's avatar
Matthew Brett committed
192
193
194
195
196
197
198
        && make \
        && make install)
    touch szip-stamp
}

function build_hdf5 {
    if [ -e hdf5-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
199
    build_zlib
200
201
    # libaec is a drop-in replacement for szip
    build_libaec
Matthew Brett's avatar
Matthew Brett committed
202
    local hdf5_url=https://www.hdfgroup.org/ftp/HDF5/releases
203
    fetch_unpack $hdf5_url/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz
Matthew Brett's avatar
Matthew Brett committed
204
    (cd hdf5-$HDF5_VERSION \
Matthew Brett's avatar
Matthew Brett committed
205
        && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \
Matthew Brett's avatar
Matthew Brett committed
206
207
208
209
        && make \
        && make install)
    touch hdf5-stamp
}
210
211
212
213
214
215

function build_libaec {
    if [ -e libaec-stamp ]; then return; fi
    local root_name=libaec-0.3.3
    local tar_name=${root_name}.tar.gz
    # Note URL will change for each version
216
    fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/48398bd5b7bc05a3edb3325abfeac864/${tar_name}
217
    (cd $root_name \
Matthew Brett's avatar
Matthew Brett committed
218
        && ./configure --prefix=$BUILD_PREFIX \
219
220
221
222
        && make \
        && make install)
    touch libaec-stamp
}
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248

function build_blosc {
    if [ -e blosc-stamp ]; then return; fi
    local cmake=$(get_cmake)
    fetch_unpack https://github.com/Blosc/c-blosc/archive/v${BLOSC_VERSION}.tar.gz
    (cd c-blosc-${BLOSC_VERSION} \
        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \
        && make install)
    if [ -n "$IS_OSX" ]; then
        # Fix blosc library id bug
        for lib in $(ls ${BUILD_PREFIX}/lib/libblosc*.dylib); do
            install_name_tool -id $lib $lib
        done
    fi
    touch blosc-stamp
}

function build_lzo {
    if [ -e lzo-stamp ]; then return; fi
    fetch_unpack http://www.oberhumer.com/opensource/lzo/download/lzo-${LZO_VERSION}.tar.gz
    (cd lzo-${LZO_VERSION} \
        && ./configure --prefix=$BUILD_PREFIX --enable-shared \
        && make \
        && make install)
    touch lzo-stamp
}
249
250
251

function build_curl {
    if [ -e curl-stamp ]; then return; fi
Matthew Brett's avatar
Matthew Brett committed
252
253
    if [ -n "$IS_OSX" ]; then
        flags="--with-darwinssl"
254
255
256
    else  # manylinux
        prefix="LIBS=-ldl"
        flags="--with-ssl"
Matthew Brett's avatar
Matthew Brett committed
257
258
        build_openssl
    fi
259
260
    fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz
    (cd curl-${CURL_VERSION} \
261
        && $prefix ./configure --prefix=$BUILD_PREFIX $flags \
262
263
264
265
266
        && make \
        && make install)
    touch curl-stamp
}

Matthew Brett's avatar
Matthew Brett committed
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
function check_sha256sum {
    local fname=$1
    if [ -z "$fname" ]; then echo "Need path"; exit 1; fi
    local sha256=$2
    if [ -z "$sha256" ]; then echo "Need SHA256 hash"; exit 1; fi
    echo "${sha256}  ${fname}" > ${fname}.sha256
    if [ -n "$IS_OSX" ]; then
        shasum -a 256 -c ${fname}.sha256
    else
        sha256sum -c ${fname}.sha256
    fi
    rm ${fname}.sha256
}

function build_openssl {
    if [ -e openssl-stamp ]; then return; fi
    fetch_unpack ${OPENSSL_DOWNLOAD_URL}/${OPENSSL_ROOT}.tar.gz
    check_sha256sum $ARCHIVE_SDIR/${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
    (cd ${OPENSSL_ROOT} \
        && ./config no-ssl2 no-shared -fPIC --prefix=$BUILD_PREFIX \
        && make \
        && make install)
    touch openssl-stamp
}

292
293
294
295
296
297
function build_netcdf {
    if [ -e netcdf-stamp ]; then return; fi
    build_hdf5
    build_curl
    fetch_unpack https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz
    (cd netcdf-c-${NETCDF_VERSION} \
Matthew Brett's avatar
Matthew Brett committed
298
        && ./configure --prefix=$BUILD_PREFIX --enable-dap \
299
300
301
302
        && make \
        && make install)
    touch netcdf-stamp
}