Merge pull request #169 from matthew-brett/refactor-suppress

MRG: fixes and tests for builds

* Added Andrew Murray's changes to the URLs, http -> https.
* Fixed the `suppress` utility to work correctly with set -e; it was previously
  swallowing errors, concealing real build failures, in the tests.
* Fixed some build recipe failures revealed by the fixes to `suppress`.
* Added BUILD_PREFIX to pkg-config PATH, to promote built packages before
  system packages.
* Disabled the lzo build test, pending fixes to clang build by author.
* Updated lzo version to latest.

.travis.yml

@@ -6,23 +6,35 @@ cache:
   directories:
    - $HOME/.ccache
 
+env:
+  global:
+    - PLAT=x86_64
+
 matrix:
   include:
     - os: linux
       dist: precise
+    # 64-bit builds
+    - os: linux
+      dist: trusty
       env:
         - TEST_BUILDS=1
+      sudo: required
+    # 32-bit builds
     - os: linux
       dist: trusty
       env:
         - TEST_BUILDS=1
+        - PLAT=i686
       sudo: required
+    # Builds with caching
     - os: linux
       dist: trusty
       env:
         - TEST_BUILDS=1
         - USE_CCACHE=1
       sudo: required
+    # OSX builds
     - os: osx
       osx_image: xcode6.4
       env:

common_utils.sh

@@ -95,12 +95,32 @@ function gh-clone {
     git clone https://github.com/$1
 }
 
+function set_opts {
+    # Set options from input options string (in $- format).
+    local opts=$1
+    local chars="exhimBH"
+    for (( i=0; i<${#chars}; i++ )); do
+        char=${chars:$i:1}
+        [ -n "${opts//[^${char}]/}" ] && set -$char || set +$char
+    done
+}
+
 function suppress {
-    # Suppress the output of a bash command unless it fails
-    out=$( ( $@ ) 2>&1 )
-    ret=$?
-    [ "$ret" -eq 0 ] || >&2 echo "$out" # if $? (the return of the last run command) is not zero, cat the temp file
-    return "$ret" # return the exit status of the command
+    # Run a command, show output only if return code not 0.
+    # Takes into account state of -e option.
+    # Compare
+    # https://unix.stackexchange.com/questions/256120/how-can-i-suppress-output-only-if-the-command-succeeds#256122
+    # Set -e stuff agonized over in
+    # https://unix.stackexchange.com/questions/296526/set-e-in-a-subshell
+    local tmp=$(mktemp tmp.XXXXXXXXX) || return
+    local opts=$-
+    echo "Running $@"
+    set +e
+    ( set_opts $opts ; $@  > "$tmp" 2>&1 ) ; ret=$?
+    [ "$ret" -eq 0 ] || cat "$tmp"
+    rm -f "$tmp"
+    set_opts $opts
+    return "$ret"
 }
 
 function rm_mkdir {
@@ -137,6 +157,8 @@ function fetch_unpack {
     #    url - URL from which to fetch archive
     #    archive_fname (optional) archive name
     #
+    # Echos unpacked directory and file names.
+    #
     # If `archive_fname` not specified then use basename from `url`
     # If `archive_fname` already present at download location, use that instead.
     local url=$1
@@ -150,10 +172,14 @@ function fetch_unpack {
     if [ ! -f "$out_archive" ]; then
         curl -L $url > $out_archive
     fi
-    # Unpack archive, refreshing contents
+    # Unpack archive, refreshing contents, echoing dir and file
+    # names.
     rm_mkdir arch_tmp
     install_rsync
-    (cd arch_tmp && untar ../$out_archive && rsync --delete -ah * ..)
+    (cd arch_tmp && \
+        untar ../$out_archive && \
+        ls -1d * &&
+        rsync --delete -ah * ..)
 }
 
 function clean_code {

configure_build.sh

@@ -37,3 +37,4 @@ fi
 # Promote BUILD_PREFIX on search path to any newly built libs
 export CPPFLAGS="-L$BUILD_PREFIX/include $CPPFLAGS"
 export LIBRARY_PATH="$BUILD_PREFIX/lib:$LIBRARY_PATH"
+export PKG_CONFIG_PATH="$BUILD_PREFIX/lib/pkgconfig/:$PKG_CONFIG_PATH"

library_builders.sh

@@ -23,10 +23,10 @@ GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}"
 LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}"
 XZ_VERSION="${XZ_VERSION:-5.2.2}"
 LIBYAML_VERSION="${LIBYAML_VERSION:-0.1.5}"
-SZIP_VERSION="${SZIP_VERSION:-2.1}"
+SZIP_VERSION="${SZIP_VERSION:-2.1.1}"
 HDF5_VERSION="${HDF5_VERSION:-1.10.2}"
 LIBAEC_VERSION="${LIBAEC_VERSION:-0.3.3}"
-LZO_VERSION=${LZO_VERSION:-2.09}
+LZO_VERSION=${LZO_VERSION:-2.10}
 LZF_VERSION="${LZF_VERSION:-3.6}"
 BLOSC_VERSION=${BLOSC_VERSION:-1.10.2}
 SNAPPY_VERSION="${SNAPPY_VERSION:-1.1.3}"
@@ -51,7 +51,7 @@ ARCHIVE_SDIR=${ARCHIVE_DIR:-archives}
 
 function build_simple {
     # Example: build_simple libpng $LIBPNG_VERSION \
-    #               http://download.sourceforge.net/libpng tar.gz \
+    #               https://download.sourceforge.net/libpng tar.gz \
     #               --additional --configure --arguments
     local name=$1
     local version=$2
@@ -74,21 +74,14 @@ function build_simple {
 function build_github {
     # Example: build_github fredrik-johansson/arb 2.11.1
     local path=$1
-    local version=$2
+    local tag_name=$2
     local configure_args=${@:3}
     local name=`basename "$path"`
-    # This is tricky. If the version name starts with a "v",
-    # then the archive name will not start with a "v"
-    if [[ $version == v* ]]; then
-        local name_version="${name}-${version:1}"
-    else
-        local name_version="${name}-${version}"
-    fi
     if [ -e "${name}-stamp" ]; then
         return
     fi
-    fetch_unpack "https://github.com/${path}/archive/${version}.tar.gz"
-    (cd $name_version \
+    local out_dir=$(fetch_unpack "https://github.com/${path}/archive/${tag_name}.tar.gz")
+    (cd $out_dir \
         && ./configure --prefix=$BUILD_PREFIX $configure_args \
         && make -j4 \
         && make install)
@@ -98,11 +91,13 @@ function build_github {
 function build_openblas {
     if [ -e openblas-stamp ]; then return; fi
     if [ -n "$IS_OSX" ]; then
+        # https://github.com/travis-ci/travis-ci/issues/8826
+        brew cask uninstall oclint || echo "no oclint"
         brew install openblas
         brew link --force openblas
     else
         mkdir -p $ARCHIVE_SDIR
-        local plat=${1:-$PLAT}
+        local plat=${1:-${PLAT:-x86_64}}
         local tar_path=$(abspath $(_mb_get_gf_lib "openblas-${OPENBLAS_VERSION}" "$plat"))
         (cd / && tar zxf $tar_path)
     fi
@@ -120,7 +115,7 @@ function build_zlib {
 function build_new_zlib {
     # Careful, this one may cause yum to segfault
     # Fossils directory should also contain latest
-    build_simple zlib $ZLIB_VERSION http://zlib.net/fossils
+    build_simple zlib $ZLIB_VERSION https://zlib.net/fossils
 }
 
 function build_jpeg {
@@ -135,7 +130,7 @@ function build_jpeg {
 
 function build_libpng {
     build_zlib
-    build_simple libpng $LIBPNG_VERSION http://download.sourceforge.net/libpng
+    build_simple libpng $LIBPNG_VERSION https://download.sourceforge.net/libpng
 }
 
 function build_bzip2 {
@@ -152,7 +147,7 @@ function build_tiff {
     build_zlib
     build_jpeg
     build_xz
-    build_simple tiff $TIFF_VERSION http://download.osgeo.org/libtiff
+    build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff
 }
 
 function get_cmake {
@@ -188,15 +183,15 @@ function build_openjpeg {
 
 function build_lcms2 {
     build_tiff
-    build_simple lcms2 $LCMS2_VERSION http://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION
+    build_simple lcms2 $LCMS2_VERSION https://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION
 }
 
 function build_giflib {
-    build_simple giflib $GIFLIB_VERSION http://downloads.sourceforge.net/project/giflib
+    build_simple giflib $GIFLIB_VERSION https://downloads.sourceforge.net/project/giflib
 }
 
 function build_xz {
-    build_simple xz $XZ_VERSION http://tukaani.org/xz
+    build_simple xz $XZ_VERSION https://tukaani.org/xz
 }
 
 function build_libwebp {
@@ -211,18 +206,18 @@ function build_libwebp {
 function build_freetype {
     build_libpng
     build_bzip2
-    build_simple freetype $FREETYPE_VERSION http://download.savannah.gnu.org/releases/freetype
+    build_simple freetype $FREETYPE_VERSION https://download.savannah.gnu.org/releases/freetype
 }
 
 function build_libyaml {
-    build_simple yaml $LIBYAML_VERSION http://pyyaml.org/download/libyaml
+    build_simple yaml $LIBYAML_VERSION https://pyyaml.org/download/libyaml
 }
 
 function build_szip {
     # Build szip without encoding (patent restrictions)
     build_zlib
     build_simple szip $SZIP_VERSION \
-        https://www.hdfgroup.org/ftp/lib-external/szip tar.gz \
+        https://support.hdfgroup.org/ftp/lib-external/szip/$SZIP_VERSION/src tar.gz \
         --enable-encoding=no
 }
 
@@ -276,7 +271,7 @@ function build_snappy {
 
 function build_lzo {
     if [ -e lzo-stamp ]; then return; fi
-    fetch_unpack http://www.oberhumer.com/opensource/lzo/download/lzo-${LZO_VERSION}.tar.gz
+    fetch_unpack https://www.oberhumer.com/opensource/lzo/download/lzo-${LZO_VERSION}.tar.gz
     (cd lzo-${LZO_VERSION} \
         && ./configure --prefix=$BUILD_PREFIX --enable-shared \
         && make \
@@ -353,7 +348,7 @@ function build_swig {
         brew install swig > /dev/null
     else
         build_pcre
-        build_simple swig $SWIG_VERSION http://prdownloads.sourceforge.net/swig
+        build_simple swig $SWIG_VERSION https://prdownloads.sourceforge.net/swig
     fi
 }
 
@@ -370,7 +365,7 @@ function build_libtool {
 }
 
 function build_ragel {
-    build_simple ragel $RAGEL_VERSION http://www.colm.net/files/ragel
+    build_simple ragel $RAGEL_VERSION https://www.colm.net/files/ragel
 }
 
 function build_bison {

tests/test_common_utils.sh

@@ -44,5 +44,47 @@ rm_mkdir tmp_dir
 [ -e tmp_dir/afile ] && ingest "tmp_dir/afile should have been deleted"
 rmdir tmp_dir
 
+# Test suppress command
+function bad_cmd {
+    echo bad
+    return 1
+}
+
+function bad_mid_cmd {
+    # Command returns 0, but errors in the middle
+    echo ok for now
+    false
+    echo should be bad now
+    return 0
+}
+
+function good_cmd {
+    echo good
+    return 0
+}
+
+# Store state of options including -e, -x
+# https://stackoverflow.com/questions/14564746/in-bash-how-to-get-the-current-status-of-set-x?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa
+ORIG_OPTS=$-
+set +ex
+[ "$(suppress bad_cmd)" == "$(printf "Running bad_cmd\nbad")" ] \
+    || ingest "suppress bad_cmd"
+[ "$(suppress good_cmd)" == "Running good_cmd" ] \
+    || ingest "suppress good_cmd"
+[ "$(suppress bad_mid_cmd)" == "Running bad_mid_cmd" ] \
+    || ingest "suppress bad_mid_cmd"
+# Can't use pipes here, because of the effect on set -e behavior.
+expected="$(printf "Running bad_cmd\nbad")"
+actual="$(set -e; suppress bad_cmd)"
+[ "$actual" == "$expected" ] || ingest "suppress bad_cmd set -e"
+expected="$(printf "Running good_cmd")"
+actual="$(set -e; suppress good_cmd)"
+[ "$actual" == "$expected" ] || ingest "suppress good_cmd set -e"
+expected="$(printf "Running bad_mid_cmd\nok for now")"
+actual="$(set -e; suppress bad_mid_cmd)"
+[ "$actual" == "$expected" ] || ingest "suppress bad_mid_cmd set -e"
+# Reset options
+set_opts $ORIG_OPTS
+
 # On Linux docker containers in travis, can only be x86_64 or i686
 [ "$(get_platform)" == x86_64 ] || [ "$(get_platform)" == i686 ] || exit 1

tests/test_library_builders.sh

@@ -12,8 +12,29 @@ suppress build_libpng
 suppress build_libwebp
 suppress build_szip
 suppress build_swig
-suppress build_github fredrik-johansson/arb 2.13.0
+# We need to find a failable test for build_github
+# It needs a standalone C library with ./configure script.
+# E.g. arb (below) requires a couple of other libraries.
+# Run here just for the output, even though they fail.
+(set +e ;
+    build_github fredrik-johansson/arb 2.13.0 ;
+    build_github glennrp/libpng v1.6.34 ;
+    build_github wbhart/mpir mpir-3.0.0
+    )
 suppress build_flex
 suppress build_openblas
+suppress build_tiff
+suppress build_lcms2
+suppress build_xz
+suppress build_freetype
+suppress build_libyaml
+if [ -z "$IS_OSX" ]; then
+    # Gives compiler conformance error on macOS Sierra:
+    # https://gist.github.com/5e20e137ea51fa8ca9fc443191f9d463
+    # https://gist.github.com/ad86c474f3c0b7ec74290bb13f9414af
+    suppress build_lzo
+fi
+suppress build_ragel
+suppress build_new_zlib
 
 stop_spinner

tests/test_multibuild.sh

@@ -18,12 +18,14 @@ else
     source tests/test_manylinux_utils.sh
 fi
 if [ -n "$TEST_BUILDS" ]; then
-    if [ -n "$IS_OSX" ] || [ ! -x "$(command -v docker)" ]; then
-        source tests/test_library_builders.sh        
+    if [ -n "$IS_OSX" ]; then
+        source tests/test_library_builders.sh
+    elif [ ! -x "$(command -v docker)" ]; then
+        echo "Skipping build tests; no docker available"
     else
         touch config.sh
         source travis_linux_steps.sh
-        build_multilinux i686 "source tests/test_library_builders.sh"
+        build_multilinux $PLAT "source tests/test_library_builders.sh"
     fi
 fi