Use cmake for third party (#753)

* Use cmake for third party

* Apply patch to libmad

* Update gitignore

* Update docker test image

.circleci/config.yml

@@ -38,6 +38,17 @@ commands:
               our_upload_channel=test
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
+  install_cmake_macos:
+    description: "installs cmake on macOS. Use binary distribution as brew is slow"
+    steps:
+      - run:
+          name: Install cmake
+          command: |
+            curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
+            mkdir cmake
+            tar -xf cmake.tar.gz --strip 3 -C cmake
+            rm cmake.tar.gz
+            echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
 
 binary_common: &binary_common
   parameters:
@@ -82,7 +93,7 @@ jobs:
 
   download_third_parties_nix:
     docker:
-      - image: "centos:8"
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
     resource_class: small
     steps:
       - checkout
@@ -90,20 +101,22 @@ jobs:
       - restore_cache:
 
           keys:
-            - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+            - tp-nix-v2-{{ checksum ".cachekey" }}
 
       - run:
-          command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
+          command: |
+              mkdir -p third_party/archives/
+              wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
       - save_cache:
 
-          key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+          key: tp-nix-v2-{{ checksum ".cachekey" }}
 
           paths:
-            - third_party/tmp
+            - third_party/archives
       - persist_to_workspace:
           root: third_party
           paths:
-            - tmp
+            - archives
 
   binary_linux_wheel:
     <<: *binary_common
@@ -145,6 +158,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -169,6 +183,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -383,19 +398,21 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -423,19 +440,21 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -542,6 +561,8 @@ jobs:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh

.circleci/config.yml.in

@@ -38,6 +38,17 @@ commands:
               our_upload_channel=test
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
+  install_cmake_macos:
+    description: "installs cmake on macOS. Use binary distribution as brew is slow"
+    steps:
+      - run:
+          name: Install cmake
+          command: |
+            curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
+            mkdir cmake
+            tar -xf cmake.tar.gz --strip 3 -C cmake
+            rm cmake.tar.gz
+            echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
 
 binary_common: &binary_common
   parameters:
@@ -82,7 +93,7 @@ jobs:
 
   download_third_parties_nix:
     docker:
-      - image: "centos:8"
+      - image: "pytorch/torchaudio_unittest_base:manylinux"
     resource_class: small
     steps:
       - checkout
@@ -90,20 +101,22 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+            - tp-nix-v2-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
-          command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
+          command: |
+              mkdir -p third_party/archives/
+              wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
       - save_cache:
           {% raw %}
-          key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
+          key: tp-nix-v2-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
-            - third_party/tmp
+            - third_party/archives
       - persist_to_workspace:
           root: third_party
           paths:
-            - tmp
+            - archives
 
   binary_linux_wheel:
     <<: *binary_common
@@ -145,6 +158,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -169,6 +183,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
+      - install_cmake_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -383,19 +398,21 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -423,19 +440,21 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Install torchaudio
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -542,6 +561,8 @@ jobs:
             - conda
             - env
             - third_party/build
+            - third_party/install
+            - third_party/src
       - run:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh

.circleci/unittest/linux/docker/Dockerfile

@@ -58,6 +58,7 @@ RUN apt update && apt install -y \
         sox \
         libsox-dev \
         libsox-fmt-all \
+        cmake \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /kaldi /kaldi
 COPY --from=builder /third_party /third_party

.circleci/unittest/linux/scripts/run_test.sh

@@ -6,5 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)"
 conda activate ./env
 
 python -m torch.utils.collect_env
-export PATH="${PWD}/third_party/build/bin/:${PATH}"
+export PATH="${PWD}/third_party/install/bin/:${PATH}"
 pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test

.circleci/unittest/linux/scripts/setup_env.sh

@@ -33,5 +33,10 @@ conda activate "${env_dir}"
 printf "* Installing dependencies (except PyTorch)\n"
 conda env update --file "${this_dir}/environment.yml" --prune
 
-# 4. Build codecs
-build_tools/setup_helpers/build_third_party.sh
+# 4. Buld codecs
+mkdir -p third_party/build
+(
+    cd third_party/build
+    cmake ..
+    cmake --build .
+)

.gitignore

@@ -121,3 +121,9 @@ gen.yml
 examples/interactive_asr/data/*.txt
 examples/interactive_asr/data/*.model
 examples/interactive_asr/data/*.pt
+
+# third parties
+third_party/archives/
+third_party/install/
+third_party/src/
+third_party/tmp/

build_tools/setup_helpers/build_third_party.sh

-#!/bin/bash
-# Build third party libraries (SoX, lame, libmad, and flac)
-# Usage: ./build_thid_parth.sh [prefix] [download_only?=false]
-
-set -e
-
-this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-root_dir="${this_dir}/../.."
-
-prefix="${1:-}"
-if [ -z "${prefix}" ]; then
-    prefix="${root_dir}"
-fi
-download_only="${2:-false}"
-
-tp_dir="${prefix}/third_party"
-tmp_dir="${tp_dir}/tmp"
-build_dir="${tp_dir}/build"
-
-mkdir -p "${tmp_dir}" "${build_dir}"
-
-. "${this_dir}/build_third_party_helper.sh"
-
-if ! found_ogg "${build_dir}" ; then
-    get_ogg "${tmp_dir}"
-    if [ "${download_only}" = "false" ]; then
-        build_ogg "${tmp_dir}" "${build_dir}"
-    fi
-fi
-
-if ! found_vorbis "${build_dir}" ; then
-    get_vorbis "${tmp_dir}"
-    if [ "${download_only}" = "false" ]; then
-        build_vorbis "${tmp_dir}" "${build_dir}"
-    fi
-fi
-
-if ! found_lame "${build_dir}" ; then
-    get_lame "${tmp_dir}"
-    if [ "${download_only}" = "false" ]; then
-        build_lame "${tmp_dir}" "${build_dir}"
-    fi
-fi
-
-if ! found_flac "${build_dir}" ; then
-   get_flac "${tmp_dir}"
-   if [ "${download_only}" = "false" ]; then
-       build_flac "${tmp_dir}" "${build_dir}"
-   fi
-fi
-
-if ! found_mad "${build_dir}" ; then
-   get_mad "${tmp_dir}"
-   if [ "${download_only}" = "false" ]; then
-       build_mad "${tmp_dir}" "${build_dir}"
-   fi
-fi
-
-if ! found_sox "${build_dir}" ; then
-   get_sox "${tmp_dir}"
-   if [ "${download_only}" = "false" ]; then
-       build_sox "${tmp_dir}" "${build_dir}"
-   fi
-fi

build_tools/setup_helpers/build_third_party_helper.sh

-#!/usr/bin/env bash
-
-set -euo pipefail
-
-# Global options
-CURL_OPTS="-L --retry 10 --connect-timeout 5 --max-time 180"
-MAKE_OPTS="-j"
-CONFIG_OPTS=""
-
-if [ -z ${DEBUG+x} ]; then
-    CURL_OPTS="${CURL_OPTS} --silent --show-error"
-    MAKE_OPTS="${MAKE_OPTS} --quiet"
-    CONFIG_OPTS="${CONFIG_OPTS} --quiet"
-fi
-
-all_found() {
-    dir="$1"
-    shift
-    while [ "$#" -gt 0 ]; do
-        if [ ! -f "${dir}/$1" ]; then
-            return 1
-        fi
-        shift
-    done
-}
-
-found_ogg() {
-    all_found "$1" 'include/ogg/ogg.h' 'lib/libogg.a'
-}
-
-found_vorbis() {
-    all_found "$1" \
-              'include/vorbis/vorbisenc.h' \
-              'include/vorbis/vorbisfile.h' \
-              'lib/libvorbis.a' \
-              'lib/libvorbisenc.a' \
-              'lib/libvorbisfile.a'
-}
-
-found_lame() {
-    all_found "$1" 'include/lame/lame.h' 'lib/libmp3lame.a'
-}
-
-found_flac() {
-    all_found "$1" \
-              'include/FLAC/format.h' \
-              'include/FLAC/stream_decoder.h' \
-              'include/FLAC/export.h' \
-              'include/FLAC/ordinals.h' \
-              'include/FLAC/all.h' \
-              'include/FLAC/assert.h' \
-              'include/FLAC/callback.h' \
-              'include/FLAC/metadata.h' \
-              'include/FLAC/stream_encoder.h' \
-              'include/FLAC++/export.h' \
-              'include/FLAC++/decoder.h' \
-              'include/FLAC++/all.h' \
-              'include/FLAC++/metadata.h' \
-              'include/FLAC++/encoder.h' \
-              'lib/libFLAC++.a' \
-              'lib/libFLAC.a'
-}
-
-found_mad() {
-    all_found "$1" 'include/mad.h' 'lib/libmad.a'
-}
-
-found_sox() {
-    all_found "$1" 'include/sox.h' 'lib/libsox.a'
-}
-
-# libogg 1.3.4 has bug on mac OS.
-# https://trac.macports.org/ticket/58924
-OGG="libogg-1.3.3"
-OGG_ARCHIVE="${OGG}.tar.gz"
-
-get_ogg() {
-    work_dir="$1"
-    url="https://ftp.osuosl.org/pub/xiph/releases/ogg/${OGG_ARCHIVE}"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${OGG}" ]; then
-            if [ ! -f "${OGG_ARCHIVE}" ]; then
-                printf "Fetching libogg from %s\n" "${url}"
-                curl $CURL_OPTS -O "${url}"
-            fi
-        fi
-    )
-}
-
-build_ogg() {
-    work_dir="$1"
-    install_dir="$2"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${OGG}" ]; then
-            tar xfp "${OGG_ARCHIVE}"
-        fi
-        cd "${OGG}"
-        printf "Building libogg\n"
-        if [ ! -f Makefile ]; then
-            ./configure ${CONFIG_OPTS} \
-                        --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
-                        --with-pic --disable-dependency-tracking
-        fi
-        make ${MAKE_OPTS} > make.log 2>&1
-        make install
-    )
-}
-
-VORBIS="libvorbis-1.3.6"
-VORBIS_ARCHIVE="${VORBIS}.tar.gz"
-
-get_vorbis() {
-    work_dir="$1"
-    url="https://ftp.osuosl.org/pub/xiph/releases/vorbis/${VORBIS_ARCHIVE}"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${VORBIS}" ]; then
-            if [ ! -f "${VORBIS_ARCHIVE}" ]; then
-                printf "Fetching libvorbis from %s\n" "${url}"
-                curl $CURL_OPTS -O "${url}"
-            fi
-        fi
-    )
-}
-
-build_vorbis() {
-    work_dir="$1"
-    install_dir="$2"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${VORBIS}" ]; then
-            tar xfp "${VORBIS_ARCHIVE}"
-        fi
-        cd "${VORBIS}"
-        printf "Building libvorbis\n"
-        if [ ! -f Makefile ]; then
-            ./configure ${CONFIG_OPTS} \
-                        --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
-                        --with-pic --disable-dependency-tracking
-        fi
-        make ${MAKE_OPTS} > make.log 2>&1
-        make install
-    )
-}
-
-LAME="lame-3.99.5"
-LAME_ARCHIVE="${LAME}.tar.gz"
-
-get_lame() {
-    work_dir="$1"
-    url="https://downloads.sourceforge.net/project/lame/lame/3.99/${LAME_ARCHIVE}"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${LAME}" ]; then
-            if [ ! -f "${LAME_ARCHIVE}" ]; then
-                printf "Fetching liblame from %s\n" "${url}"
-                curl $CURL_OPTS -O "${url}"
-            fi
-        fi
-    )
-}
-
-build_lame() {
-    work_dir="$1"
-    install_dir="$2"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${LAME}" ]; then
-            tar xfp "${LAME_ARCHIVE}"
-        fi
-        cd "${LAME}"
-        # build statically
-        printf "Building liblame\n"
-        if [ ! -f Makefile ]; then
-            ./configure ${CONFIG_OPTS} \
-                        --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
-                        --with-pic --disable-debug --disable-dependency-tracking --enable-nasm
-        fi
-        make ${MAKE_OPTS} > make.log 2>&1
-        make ${MAKE_OPTS} install
-    )
-}
-
-FLAC="flac-1.3.2"
-FLAC_ARCHIVE="${FLAC}.tar.xz"
-
-get_flac() {
-    work_dir="$1"
-    url="https://downloads.sourceforge.net/project/flac/flac-src/${FLAC_ARCHIVE}"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${FLAC}" ]; then
-            if [ ! -f "${FLAC_ARCHIVE}" ]; then
-                printf "Fetching flac from %s\n" "${url}"
-                curl $CURL_OPTS -O "${url}"
-            fi
-        fi
-    )
-}
-
-build_flac() {
-    work_dir="$1"
-    install_dir="$2"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${FLAC}" ]; then
-            tar xfp "${FLAC_ARCHIVE}"
-        fi
-        cd "${FLAC}"
-        # build statically
-        printf "Building flac\n"
-        if [ ! -f Makefile ]; then
-            ./configure ${CONFIG_OPTS} \
-                        --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
-                        --with-pic --with-ogg="${install_dir}" --disable-debug --disable-dependency-tracking
-        fi
-        make ${MAKE_OPTS} > make.log 2>&1
-        make ${MAKE_OPTS} install
-    )
-}
-
-LIBMAD="libmad-0.15.1b"
-LIBMAD_ARCHIVE="${LIBMAD}.tar.gz"
-
-get_mad() {
-    work_dir="$1"
-    url="https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/${LIBMAD_ARCHIVE}"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${LIBMAD}" ]; then
-            if [ ! -f "${LIBMAD_ARCHIVE}" ]; then
-                printf "Fetching mad from %s\n" "${url}"
-                curl $CURL_OPTS -O "${url}"
-            fi
-        fi
-    )
-}
-
-build_mad() {
-    work_dir="$1"
-    install_dir="$2"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${LIBMAD}" ]; then
-            tar xfp "${LIBMAD_ARCHIVE}"
-        fi
-        cd "${LIBMAD}"
-        # build statically
-        printf "Building mad\n"
-        if [ ! -f Makefile ]; then
-            # See https://stackoverflow.com/a/12864879/23845
-            sed -i.bak 's/-march=i486//' configure
-            ./configure ${CONFIG_OPTS} \
-                        --disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
-                        --with-pic --disable-debug --disable-dependency-tracking
-        fi
-        make ${MAKE_OPTS} > make.log 2>&1
-        make ${MAKE_OPTS} install
-    )
-}
-
-SOX="sox-14.4.2"
-SOX_ARCHIVE="${SOX}.tar.bz2"
-
-get_sox() {
-    work_dir="$1"
-    url="https://downloads.sourceforge.net/project/sox/sox/14.4.2/${SOX_ARCHIVE}"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${SOX}" ]; then
-            if [ ! -f "${SOX_ARCHIVE}" ]; then
-                printf "Fetching SoX from %s\n" "${url}"
-                curl $CURL_OPTS -O "${url}"
-            fi
-        fi
-    )
-}
-
-build_sox() {
-    work_dir="$1"
-    install_dir="$2"
-    (
-        cd "${work_dir}"
-        if [ ! -d "${SOX}" ]; then
-            tar xfp "${SOX_ARCHIVE}"
-        fi
-        cd "${SOX}"
-        # build statically
-        printf "Building SoX\n"
-        if [ ! -f Makefile ]; then
-            # --without-png makes OS X build less hazardous; somehow the build
-            # finds png and enables it.  We don't want it; we'd need to package
-            # it statically if we do.
-            ./configure ${CONFIG_OPTS} --disable-shared --enable-static --prefix="${install_dir}" \
-                        LDFLAGS="-L${install_dir}/lib" CPPFLAGS="-I${install_dir}/include" \
-                        --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio \
-                        --without-png --without-oss --without-sndfile \
-                        CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking
-        fi
-        make ${MAKE_OPTS} > make.log 2>&1
-        make ${MAKE_OPTS} install
-    )
-}

build_tools/setup_helpers/extension.py

@@ -17,7 +17,7 @@ _THIS_DIR = Path(__file__).parent.resolve()
 _ROOT_DIR = _THIS_DIR.parent.parent.resolve()
 _CSRC_DIR = _ROOT_DIR / 'torchaudio' / 'csrc'
 _TP_BASE_DIR = _ROOT_DIR / 'third_party'
-_TP_INSTALL_DIR = _TP_BASE_DIR / 'build'
+_TP_INSTALL_DIR = _TP_BASE_DIR / 'install'
 
 
 def _get_build_sox():
@@ -97,15 +97,19 @@ def _get_libraries():
     return [] if _BUILD_SOX else ['sox']
 
 
-def _build_codecs():
+def _build_third_party():
+    build_dir = str(_TP_BASE_DIR / 'build')
+    os.makedirs(build_dir, exist_ok=True)
     subprocess.run(
-        args=[str(_THIS_DIR / 'build_third_party.sh')],
+        args=['cmake', '..'],
+        cwd=build_dir,
+        check=True,
+    )
+    subprocess.run(
+        args=['cmake', '--build', '.'],
+        cwd=build_dir,
         check=True,
     )
-
-
-def _configure_third_party():
-    _build_codecs()
 
 
 _EXT_NAME = 'torchaudio._torchaudio'
@@ -130,5 +134,5 @@ def get_ext_modules(debug=False):
 class BuildExtension(TorchBuildExtension):
     def build_extension(self, ext):
         if ext.name == _EXT_NAME and _BUILD_SOX:
-            _configure_third_party()
+            _build_third_party()
         super().build_extension(ext)

third_party/CMakeLists.txt

+cmake_minimum_required(VERSION 3.1)
+
+project(torchaudio_third_parties)
+include(ExternalProject)
+
+set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/install)
+set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/archives)
+set(COMMON_ARGS --quiet --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc)
+
+ExternalProject_Add(libmad
+  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz
+  URL_HASH SHA256=bbfac3ed6bfbc2823d3775ebb931087371e142bb0e9bb1bee51a76a6e0078690
+  PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/patch/libmad.patch
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmad/configure ${COMMON_ARGS}
+)
+
+ExternalProject_Add(libmp3lame
+  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
+  URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmp3lame/configure ${COMMON_ARGS} --enable-nasm
+)
+
+ExternalProject_Add(libogg
+  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
+  URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libogg/configure ${COMMON_ARGS}
+)
+
+ExternalProject_Add(libflac
+  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS libogg
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
+  URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libflac/configure ${COMMON_ARGS} --with-ogg=${INSTALL_DIR}
+)
+
+ExternalProject_Add(libvorbis
+  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS libogg
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
+  URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libvorbis/configure ${COMMON_ARGS} --with-ogg=${INSTALL_DIR}
+)
+
+ExternalProject_Add(libsox
+  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS libogg libflac libvorbis libmp3lame libmad
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
+  URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} LDFLAGS=-L${INSTALL_DIR}/lib CPPFLAGS=-I${INSTALL_DIR}/include --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile
+)

third_party/patch/libmad.patch

+See the followings for the origin of this patch
+http://www.linuxfromscratch.org/blfs/view/svn/multimedia/libmad.html
+http://www.linuxfromscratch.org/patches/blfs/svn/libmad-0.15.1b-fixes-1.patch
+--- src/libmad/configure	2004-02-05 09:34:07.000000000 +0000
++++ src/libmad/configure.new	2020-06-30 21:10:28.528018931 +0000
+@@ -19083,71 +19083,7 @@
+ 
+ if test "$GCC" = yes
+ then
+-    if test -z "$arch"
+-    then
+-	case "$host" in
+-	    i386-*)           ;;
+-	    i?86-*)           arch="-march=i486" ;;
+-	    arm*-empeg-*)     arch="-march=armv4 -mtune=strongarm1100" ;;
+-	    armv4*-*)         arch="-march=armv4 -mtune=strongarm" ;;
+-	    powerpc-*)        ;;
+-	    mips*-agenda-*)   arch="-mcpu=vr4100" ;;
+-	    mips*-luxsonor-*) arch="-mips1 -mcpu=r3000 -Wa,-m4010" ;;
+-	esac
+-    fi
+-
+-    case "$optimize" in
+-	-O|"-O "*)
+-	    optimize="-O"
+-	    optimize="$optimize -fforce-mem"
+-	    optimize="$optimize -fforce-addr"
+-	    : #x optimize="$optimize -finline-functions"
+-	    : #- optimize="$optimize -fstrength-reduce"
+-	    optimize="$optimize -fthread-jumps"
+-	    optimize="$optimize -fcse-follow-jumps"
+-	    optimize="$optimize -fcse-skip-blocks"
+-	    : #x optimize="$optimize -frerun-cse-after-loop"
+-	    : #x optimize="$optimize -frerun-loop-opt"
+-	    : #x optimize="$optimize -fgcse"
+-	    optimize="$optimize -fexpensive-optimizations"
+-	    optimize="$optimize -fregmove"
+-	    : #* optimize="$optimize -fdelayed-branch"
+-	    : #x optimize="$optimize -fschedule-insns"
+-	    optimize="$optimize -fschedule-insns2"
+-	    : #? optimize="$optimize -ffunction-sections"
+-	    : #? optimize="$optimize -fcaller-saves"
+-	    : #> optimize="$optimize -funroll-loops"
+-	    : #> optimize="$optimize -funroll-all-loops"
+-	    : #x optimize="$optimize -fmove-all-movables"
+-	    : #x optimize="$optimize -freduce-all-givs"
+-	    : #? optimize="$optimize -fstrict-aliasing"
+-	    : #* optimize="$optimize -fstructure-noalias"
+-
+-	    case "$host" in
+-		arm*-*)
+-		    optimize="$optimize -fstrength-reduce"
+-		    ;;
+-		mips*-*)
+-		    optimize="$optimize -fstrength-reduce"
+-		    optimize="$optimize -finline-functions"
+-		    ;;
+-		i?86-*)
+-		    optimize="$optimize -fstrength-reduce"
+-		    ;;
+-		powerpc-apple-*)
+-		    # this triggers an internal compiler error with gcc2
+-		    : #optimize="$optimize -fstrength-reduce"
+-
+-		    # this is really only beneficial with gcc3
+-		    : #optimize="$optimize -finline-functions"
+-		    ;;
+-		*)
+-		    # this sometimes provokes bugs in gcc 2.95.2
+-		    : #optimize="$optimize -fstrength-reduce"
+-		    ;;
+-	    esac
+-	    ;;
+-    esac
++    optimize="-O2"
+ fi
+ 
+ case "$host" in
+@@ -21497,6 +21433,7 @@
+ then
+     case "$host" in
+ 	i?86-*)     FPM="INTEL"  ;;
++	x86_64*)    FPM="64BIT"  ;;
+ 	arm*-*)     FPM="ARM"    ;;
+ 	mips*-*)    FPM="MIPS"   ;;
+ 	sparc*-*)   FPM="SPARC"  ;;