Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "159fcfeb63610af84aee93a981e45504a7c19dd7"
Unverified Commit ea42513f authored by moto's avatar moto Committed by GitHub
Browse files

Use cmake for third party (#753) 

* Use cmake for third party

* Apply patch to libmad

* Update gitignore

* Update docker test image
parent d71661aa
Show whitespace changes
Inline Side-by-side
Showing with 235 additions and 400 deletions
.circleci/config.yml
View file @ ea42513f
......@@ -38,6 +38,17 @@ commands:
our_upload_channel=test
fi
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
install_cmake_macos:
description: "installs cmake on macOS. Use binary distribution as brew is slow"
steps:
- run:
name: Install cmake
command: |
curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
mkdir cmake
tar -xf cmake.tar.gz --strip 3 -C cmake
rm cmake.tar.gz
echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
binary_common: &binary_common
parameters:
......@@ -82,7 +93,7 @@ jobs:
download_third_parties_nix:
docker:
- image: "centos:8"
- image: "pytorch/torchaudio_unittest_base:manylinux"
resource_class: small
steps:
- checkout
......@@ -90,20 +101,22 @@ jobs:
- restore_cache:
keys:
- tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
- tp-nix-v2-{{ checksum ".cachekey" }}
- run:
command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
command: |
mkdir -p third_party/archives/
wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
- save_cache:
key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
key: tp-nix-v2-{{ checksum ".cachekey" }}
paths:
- third_party/tmp
- third_party/archives
- persist_to_workspace:
root: third_party
paths:
- tmp
- archives
binary_linux_wheel:
<<: *binary_common
......@@ -145,6 +158,7 @@ jobs:
xcode: "9.0"
steps:
- checkout
- install_cmake_macos
- attach_workspace:
at: third_party
- run:
......@@ -169,6 +183,7 @@ jobs:
xcode: "9.0"
steps:
- checkout
- install_cmake_macos
- attach_workspace:
at: third_party
- run:
......@@ -383,19 +398,21 @@ jobs:
- restore_cache:
keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- run:
name: Setup
command: .circleci/unittest/linux/scripts/setup_env.sh
- save_cache:
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
paths:
- conda
- env
- third_party/build
- third_party/install
- third_party/src
- run:
name: Install torchaudio
command: .circleci/unittest/linux/scripts/install.sh
......@@ -423,19 +440,21 @@ jobs:
- restore_cache:
keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- run:
name: Setup
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
- save_cache:
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
paths:
- conda
- env
- third_party/build
- third_party/install
- third_party/src
- run:
name: Install torchaudio
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
......@@ -542,6 +561,8 @@ jobs:
- conda
- env
- third_party/build
- third_party/install
- third_party/src
- run:
name: Run style check
command: .circleci/unittest/linux/scripts/run_style_checks.sh
......
.circleci/config.yml.in
View file @ ea42513f
......@@ -38,6 +38,17 @@ commands:
our_upload_channel=test
fi
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
install_cmake_macos:
description: "installs cmake on macOS. Use binary distribution as brew is slow"
steps:
- run:
name: Install cmake
command: |
curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
mkdir cmake
tar -xf cmake.tar.gz --strip 3 -C cmake
rm cmake.tar.gz
echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
binary_common: &binary_common
parameters:
......@@ -82,7 +93,7 @@ jobs:
download_third_parties_nix:
docker:
- image: "centos:8"
- image: "pytorch/torchaudio_unittest_base:manylinux"
resource_class: small
steps:
- checkout
......@@ -90,20 +101,22 @@ jobs:
- restore_cache:
{% raw %}
keys:
- tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
- tp-nix-v2-{{ checksum ".cachekey" }}
{% endraw %}
- run:
command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
command: |
mkdir -p third_party/archives/
wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
- save_cache:
{% raw %}
key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
key: tp-nix-v2-{{ checksum ".cachekey" }}
{% endraw %}
paths:
- third_party/tmp
- third_party/archives
- persist_to_workspace:
root: third_party
paths:
- tmp
- archives
binary_linux_wheel:
<<: *binary_common
......@@ -145,6 +158,7 @@ jobs:
xcode: "9.0"
steps:
- checkout
- install_cmake_macos
- attach_workspace:
at: third_party
- run:
......@@ -169,6 +183,7 @@ jobs:
xcode: "9.0"
steps:
- checkout
- install_cmake_macos
- attach_workspace:
at: third_party
- run:
......@@ -383,19 +398,21 @@ jobs:
- restore_cache:
{% raw %}
keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %}
- run:
name: Setup
command: .circleci/unittest/linux/scripts/setup_env.sh
- save_cache:
{% raw %}
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %}
paths:
- conda
- env
- third_party/build
- third_party/install
- third_party/src
- run:
name: Install torchaudio
command: .circleci/unittest/linux/scripts/install.sh
......@@ -423,19 +440,21 @@ jobs:
- restore_cache:
{% raw %}
keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %}
- run:
name: Setup
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
- save_cache:
{% raw %}
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %}
paths:
- conda
- env
- third_party/build
- third_party/install
- third_party/src
- run:
name: Install torchaudio
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
......@@ -542,6 +561,8 @@ jobs:
- conda
- env
- third_party/build
- third_party/install
- third_party/src
- run:
name: Run style check
command: .circleci/unittest/linux/scripts/run_style_checks.sh
......
.circleci/unittest/linux/docker/Dockerfile
View file @ ea42513f
......@@ -58,6 +58,7 @@ RUN apt update && apt install -y \
sox \
libsox-dev \
libsox-fmt-all \
cmake \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /kaldi /kaldi
COPY --from=builder /third_party /third_party
......
.circleci/unittest/linux/scripts/run_test.sh
View file @ ea42513f
......@@ -6,5 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)"
conda activate ./env
python -m torch.utils.collect_env
export PATH="${PWD}/third_party/build/bin/:${PATH}"
export PATH="${PWD}/third_party/install/bin/:${PATH}"
pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test
.circleci/unittest/linux/scripts/setup_env.sh
View file @ ea42513f
......@@ -33,5 +33,10 @@ conda activate "${env_dir}"
printf "* Installing dependencies (except PyTorch)\n"
conda env update --file "${this_dir}/environment.yml" --prune
# 4. Build codecs
build_tools/setup_helpers/build_third_party.sh
# 4. Buld codecs
mkdir -p third_party/build
(
cd third_party/build
cmake ..
cmake --build .
)
.gitignore
View file @ ea42513f
......@@ -121,3 +121,9 @@ gen.yml
examples/interactive_asr/data/*.txt
examples/interactive_asr/data/*.model
examples/interactive_asr/data/*.pt
# third parties
third_party/archives/
third_party/install/
third_party/src/
third_party/tmp/
build_tools/setup_helpers/build_third_party.sh deleted 100755 → 0
View file @ d71661aa
#!/bin/bash
# Build third party libraries (SoX, lame, libmad, and flac)
# Usage: ./build_thid_parth.sh [prefix] [download_only?=false]
set -e
this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
root_dir="${this_dir}/../.."
prefix="${1:-}"
if [ -z "${prefix}" ]; then
prefix="${root_dir}"
fi
download_only="${2:-false}"
tp_dir="${prefix}/third_party"
tmp_dir="${tp_dir}/tmp"
build_dir="${tp_dir}/build"
mkdir -p "${tmp_dir}" "${build_dir}"
. "${this_dir}/build_third_party_helper.sh"
if ! found_ogg "${build_dir}" ; then
get_ogg "${tmp_dir}"
if [ "${download_only}" = "false" ]; then
build_ogg "${tmp_dir}" "${build_dir}"
fi
fi
if ! found_vorbis "${build_dir}" ; then
get_vorbis "${tmp_dir}"
if [ "${download_only}" = "false" ]; then
build_vorbis "${tmp_dir}" "${build_dir}"
fi
fi
if ! found_lame "${build_dir}" ; then
get_lame "${tmp_dir}"
if [ "${download_only}" = "false" ]; then
build_lame "${tmp_dir}" "${build_dir}"
fi
fi
if ! found_flac "${build_dir}" ; then
get_flac "${tmp_dir}"
if [ "${download_only}" = "false" ]; then
build_flac "${tmp_dir}" "${build_dir}"
fi
fi
if ! found_mad "${build_dir}" ; then
get_mad "${tmp_dir}"
if [ "${download_only}" = "false" ]; then
build_mad "${tmp_dir}" "${build_dir}"
fi
fi
if ! found_sox "${build_dir}" ; then
get_sox "${tmp_dir}"
if [ "${download_only}" = "false" ]; then
build_sox "${tmp_dir}" "${build_dir}"
fi
fi
build_tools/setup_helpers/build_third_party_helper.sh deleted 100644 → 0
View file @ d71661aa
#!/usr/bin/env bash
set -euo pipefail
# Global options
CURL_OPTS="-L --retry 10 --connect-timeout 5 --max-time 180"
MAKE_OPTS="-j"
CONFIG_OPTS=""
if [ -z ${DEBUG+x} ]; then
CURL_OPTS="${CURL_OPTS} --silent --show-error"
MAKE_OPTS="${MAKE_OPTS} --quiet"
CONFIG_OPTS="${CONFIG_OPTS} --quiet"
fi
all_found() {
dir="$1"
shift
while [ "$#" -gt 0 ]; do
if [ ! -f "${dir}/$1" ]; then
return 1
fi
shift
done
}
found_ogg() {
all_found "$1" 'include/ogg/ogg.h' 'lib/libogg.a'
}
found_vorbis() {
all_found "$1" \
'include/vorbis/vorbisenc.h' \
'include/vorbis/vorbisfile.h' \
'lib/libvorbis.a' \
'lib/libvorbisenc.a' \
'lib/libvorbisfile.a'
}
found_lame() {
all_found "$1" 'include/lame/lame.h' 'lib/libmp3lame.a'
}
found_flac() {
all_found "$1" \
'include/FLAC/format.h' \
'include/FLAC/stream_decoder.h' \
'include/FLAC/export.h' \
'include/FLAC/ordinals.h' \
'include/FLAC/all.h' \
'include/FLAC/assert.h' \
'include/FLAC/callback.h' \
'include/FLAC/metadata.h' \
'include/FLAC/stream_encoder.h' \
'include/FLAC++/export.h' \
'include/FLAC++/decoder.h' \
'include/FLAC++/all.h' \
'include/FLAC++/metadata.h' \
'include/FLAC++/encoder.h' \
'lib/libFLAC++.a' \
'lib/libFLAC.a'
}
found_mad() {
all_found "$1" 'include/mad.h' 'lib/libmad.a'
}
found_sox() {
all_found "$1" 'include/sox.h' 'lib/libsox.a'
}
# libogg 1.3.4 has bug on mac OS.
# https://trac.macports.org/ticket/58924
OGG="libogg-1.3.3"
OGG_ARCHIVE="${OGG}.tar.gz"
get_ogg() {
work_dir="$1"
url="https://ftp.osuosl.org/pub/xiph/releases/ogg/${OGG_ARCHIVE}"
(
cd "${work_dir}"
if [ ! -d "${OGG}" ]; then
if [ ! -f "${OGG_ARCHIVE}" ]; then
printf "Fetching libogg from %s\n" "${url}"
curl $CURL_OPTS -O "${url}"
fi
fi
)
}
build_ogg() {
work_dir="$1"
install_dir="$2"
(
cd "${work_dir}"
if [ ! -d "${OGG}" ]; then
tar xfp "${OGG_ARCHIVE}"
fi
cd "${OGG}"
printf "Building libogg\n"
if [ ! -f Makefile ]; then
./configure ${CONFIG_OPTS} \
--disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
--with-pic --disable-dependency-tracking
fi
make ${MAKE_OPTS} > make.log 2>&1
make install
)
}
VORBIS="libvorbis-1.3.6"
VORBIS_ARCHIVE="${VORBIS}.tar.gz"
get_vorbis() {
work_dir="$1"
url="https://ftp.osuosl.org/pub/xiph/releases/vorbis/${VORBIS_ARCHIVE}"
(
cd "${work_dir}"
if [ ! -d "${VORBIS}" ]; then
if [ ! -f "${VORBIS_ARCHIVE}" ]; then
printf "Fetching libvorbis from %s\n" "${url}"
curl $CURL_OPTS -O "${url}"
fi
fi
)
}
build_vorbis() {
work_dir="$1"
install_dir="$2"
(
cd "${work_dir}"
if [ ! -d "${VORBIS}" ]; then
tar xfp "${VORBIS_ARCHIVE}"
fi
cd "${VORBIS}"
printf "Building libvorbis\n"
if [ ! -f Makefile ]; then
./configure ${CONFIG_OPTS} \
--disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
--with-pic --disable-dependency-tracking
fi
make ${MAKE_OPTS} > make.log 2>&1
make install
)
}
LAME="lame-3.99.5"
LAME_ARCHIVE="${LAME}.tar.gz"
get_lame() {
work_dir="$1"
url="https://downloads.sourceforge.net/project/lame/lame/3.99/${LAME_ARCHIVE}"
(
cd "${work_dir}"
if [ ! -d "${LAME}" ]; then
if [ ! -f "${LAME_ARCHIVE}" ]; then
printf "Fetching liblame from %s\n" "${url}"
curl $CURL_OPTS -O "${url}"
fi
fi
)
}
build_lame() {
work_dir="$1"
install_dir="$2"
(
cd "${work_dir}"
if [ ! -d "${LAME}" ]; then
tar xfp "${LAME_ARCHIVE}"
fi
cd "${LAME}"
# build statically
printf "Building liblame\n"
if [ ! -f Makefile ]; then
./configure ${CONFIG_OPTS} \
--disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
--with-pic --disable-debug --disable-dependency-tracking --enable-nasm
fi
make ${MAKE_OPTS} > make.log 2>&1
make ${MAKE_OPTS} install
)
}
FLAC="flac-1.3.2"
FLAC_ARCHIVE="${FLAC}.tar.xz"
get_flac() {
work_dir="$1"
url="https://downloads.sourceforge.net/project/flac/flac-src/${FLAC_ARCHIVE}"
(
cd "${work_dir}"
if [ ! -d "${FLAC}" ]; then
if [ ! -f "${FLAC_ARCHIVE}" ]; then
printf "Fetching flac from %s\n" "${url}"
curl $CURL_OPTS -O "${url}"
fi
fi
)
}
build_flac() {
work_dir="$1"
install_dir="$2"
(
cd "${work_dir}"
if [ ! -d "${FLAC}" ]; then
tar xfp "${FLAC_ARCHIVE}"
fi
cd "${FLAC}"
# build statically
printf "Building flac\n"
if [ ! -f Makefile ]; then
./configure ${CONFIG_OPTS} \
--disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
--with-pic --with-ogg="${install_dir}" --disable-debug --disable-dependency-tracking
fi
make ${MAKE_OPTS} > make.log 2>&1
make ${MAKE_OPTS} install
)
}
LIBMAD="libmad-0.15.1b"
LIBMAD_ARCHIVE="${LIBMAD}.tar.gz"
get_mad() {
work_dir="$1"
url="https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/${LIBMAD_ARCHIVE}"
(
cd "${work_dir}"
if [ ! -d "${LIBMAD}" ]; then
if [ ! -f "${LIBMAD_ARCHIVE}" ]; then
printf "Fetching mad from %s\n" "${url}"
curl $CURL_OPTS -O "${url}"
fi
fi
)
}
build_mad() {
work_dir="$1"
install_dir="$2"
(
cd "${work_dir}"
if [ ! -d "${LIBMAD}" ]; then
tar xfp "${LIBMAD_ARCHIVE}"
fi
cd "${LIBMAD}"
# build statically
printf "Building mad\n"
if [ ! -f Makefile ]; then
# See https://stackoverflow.com/a/12864879/23845
sed -i.bak 's/-march=i486//' configure
./configure ${CONFIG_OPTS} \
--disable-shared --enable-static --prefix="${install_dir}" CFLAGS=-fPIC CXXFLAGS=-fPIC \
--with-pic --disable-debug --disable-dependency-tracking
fi
make ${MAKE_OPTS} > make.log 2>&1
make ${MAKE_OPTS} install
)
}
SOX="sox-14.4.2"
SOX_ARCHIVE="${SOX}.tar.bz2"
get_sox() {
work_dir="$1"
url="https://downloads.sourceforge.net/project/sox/sox/14.4.2/${SOX_ARCHIVE}"
(
cd "${work_dir}"
if [ ! -d "${SOX}" ]; then
if [ ! -f "${SOX_ARCHIVE}" ]; then
printf "Fetching SoX from %s\n" "${url}"
curl $CURL_OPTS -O "${url}"
fi
fi
)
}
build_sox() {
work_dir="$1"
install_dir="$2"
(
cd "${work_dir}"
if [ ! -d "${SOX}" ]; then
tar xfp "${SOX_ARCHIVE}"
fi
cd "${SOX}"
# build statically
printf "Building SoX\n"
if [ ! -f Makefile ]; then
# --without-png makes OS X build less hazardous; somehow the build
# finds png and enables it. We don't want it; we'd need to package
# it statically if we do.
./configure ${CONFIG_OPTS} --disable-shared --enable-static --prefix="${install_dir}" \
LDFLAGS="-L${install_dir}/lib" CPPFLAGS="-I${install_dir}/include" \
--with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio \
--without-png --without-oss --without-sndfile \
CFLAGS=-fPIC CXXFLAGS=-fPIC --with-pic --disable-debug --disable-dependency-tracking
fi
make ${MAKE_OPTS} > make.log 2>&1
make ${MAKE_OPTS} install
)
}
build_tools/setup_helpers/extension.py
View file @ ea42513f
......@@ -17,7 +17,7 @@ _THIS_DIR = Path(__file__).parent.resolve()
_ROOT_DIR = _THIS_DIR.parent.parent.resolve()
_CSRC_DIR = _ROOT_DIR / 'torchaudio' / 'csrc'
_TP_BASE_DIR = _ROOT_DIR / 'third_party'
_TP_INSTALL_DIR = _TP_BASE_DIR / 'build'
_TP_INSTALL_DIR = _TP_BASE_DIR / 'install'
def _get_build_sox():
......@@ -97,15 +97,19 @@ def _get_libraries():
return [] if _BUILD_SOX else ['sox']
def _build_codecs():
def _build_third_party():
build_dir = str(_TP_BASE_DIR / 'build')
os.makedirs(build_dir, exist_ok=True)
subprocess.run(
args=[str(_THIS_DIR / 'build_third_party.sh')],
args=['cmake', '..'],
cwd=build_dir,
check=True,
)
subprocess.run(
args=['cmake', '--build', '.'],
cwd=build_dir,
check=True,
)
def _configure_third_party():
_build_codecs()
_EXT_NAME = 'torchaudio._torchaudio'
......@@ -130,5 +134,5 @@ def get_ext_modules(debug=False):
class BuildExtension(TorchBuildExtension):
def build_extension(self, ext):
if ext.name == _EXT_NAME and _BUILD_SOX:
_configure_third_party()
_build_third_party()
super().build_extension(ext)
third_party/CMakeLists.txt 0 → 100644
View file @ ea42513f
cmake_minimum_required(VERSION 3.1)
project(torchaudio_third_parties)
include(ExternalProject)
set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/install)
set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/archives)
set(COMMON_ARGS --quiet --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc)
ExternalProject_Add(libmad
PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz
URL_HASH SHA256=bbfac3ed6bfbc2823d3775ebb931087371e142bb0e9bb1bee51a76a6e0078690
PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/patch/libmad.patch
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmad/configure ${COMMON_ARGS}
)
ExternalProject_Add(libmp3lame
PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmp3lame/configure ${COMMON_ARGS} --enable-nasm
)
ExternalProject_Add(libogg
PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libogg/configure ${COMMON_ARGS}
)
ExternalProject_Add(libflac
PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
DEPENDS libogg
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libflac/configure ${COMMON_ARGS} --with-ogg=${INSTALL_DIR}
)
ExternalProject_Add(libvorbis
PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
DEPENDS libogg
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libvorbis/configure ${COMMON_ARGS} --with-ogg=${INSTALL_DIR}
)
ExternalProject_Add(libsox
PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
DEPENDS libogg libflac libvorbis libmp3lame libmad
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} LDFLAGS=-L${INSTALL_DIR}/lib CPPFLAGS=-I${INSTALL_DIR}/include --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile
)
third_party/patch/libmad.patch 0 → 100644
View file @ ea42513f
See the followings for the origin of this patch
http://www.linuxfromscratch.org/blfs/view/svn/multimedia/libmad.html
http://www.linuxfromscratch.org/patches/blfs/svn/libmad-0.15.1b-fixes-1.patch
--- src/libmad/configure 2004-02-05 09:34:07.000000000 +0000
+++ src/libmad/configure.new 2020-06-30 21:10:28.528018931 +0000
@@ -19083,71 +19083,7 @@
if test "$GCC" = yes
then
- if test -z "$arch"
- then
- case "$host" in
- i386-*) ;;
- i?86-*) arch="-march=i486" ;;
- arm*-empeg-*) arch="-march=armv4 -mtune=strongarm1100" ;;
- armv4*-*) arch="-march=armv4 -mtune=strongarm" ;;
- powerpc-*) ;;
- mips*-agenda-*) arch="-mcpu=vr4100" ;;
- mips*-luxsonor-*) arch="-mips1 -mcpu=r3000 -Wa,-m4010" ;;
- esac
- fi
-
- case "$optimize" in
- -O|"-O "*)
- optimize="-O"
- optimize="$optimize -fforce-mem"
- optimize="$optimize -fforce-addr"
- : #x optimize="$optimize -finline-functions"
- : #- optimize="$optimize -fstrength-reduce"
- optimize="$optimize -fthread-jumps"
- optimize="$optimize -fcse-follow-jumps"
- optimize="$optimize -fcse-skip-blocks"
- : #x optimize="$optimize -frerun-cse-after-loop"
- : #x optimize="$optimize -frerun-loop-opt"
- : #x optimize="$optimize -fgcse"
- optimize="$optimize -fexpensive-optimizations"
- optimize="$optimize -fregmove"
- : #* optimize="$optimize -fdelayed-branch"
- : #x optimize="$optimize -fschedule-insns"
- optimize="$optimize -fschedule-insns2"
- : #? optimize="$optimize -ffunction-sections"
- : #? optimize="$optimize -fcaller-saves"
- : #> optimize="$optimize -funroll-loops"
- : #> optimize="$optimize -funroll-all-loops"
- : #x optimize="$optimize -fmove-all-movables"
- : #x optimize="$optimize -freduce-all-givs"
- : #? optimize="$optimize -fstrict-aliasing"
- : #* optimize="$optimize -fstructure-noalias"
-
- case "$host" in
- arm*-*)
- optimize="$optimize -fstrength-reduce"
- ;;
- mips*-*)
- optimize="$optimize -fstrength-reduce"
- optimize="$optimize -finline-functions"
- ;;
- i?86-*)
- optimize="$optimize -fstrength-reduce"
- ;;
- powerpc-apple-*)
- # this triggers an internal compiler error with gcc2
- : #optimize="$optimize -fstrength-reduce"
-
- # this is really only beneficial with gcc3
- : #optimize="$optimize -finline-functions"
- ;;
- *)
- # this sometimes provokes bugs in gcc 2.95.2
- : #optimize="$optimize -fstrength-reduce"
- ;;
- esac
- ;;
- esac
+ optimize="-O2"
fi
case "$host" in
@@ -21497,6 +21433,7 @@
then
case "$host" in
i?86-*) FPM="INTEL" ;;
+ x86_64*) FPM="64BIT" ;;
arm*-*) FPM="ARM" ;;
mips*-*) FPM="MIPS" ;;
sparc*-*) FPM="SPARC" ;;
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment