Skip to content

GitLab

Commit 565f8d41 authored by moto's avatar moto Committed by Facebook GitHub Bot
Browse files

Enable build ffmpeg-features in all related jobs (#2140) 

Summary:
This commit enables ffmpeg-feature build in tests and
binary builds of all platforms.
(Linux/macOS/Windows x conda/wheel)

It also moves the definition of BUILD_FFMPEG env vars to the
top level `config.yml`.

 ---
Manual checking if all the build log contains `libtorchaudio_ffmpeg`.
### binary build
- [x] `binary_linux_conda_py3.7_cpu`
- [x] `binary_linux_conda_py3.7_cu102`
- [x] `binary_linux_wheel_py3.7_cpu`
- [x] `binary_linux_wheel_py3.7_cu102`
- [x] `binary_macos_conda_py3.7_cpu`
- [x] `binary_macos_wheel_py3.7_cpu`
- [x] `binary_windows_conda_py3.7_cpu`
- [x] `binary_windows_conda_py3.7_cu113`
- [x] `binary_windows_wheel_py3.7_cpu`
- [x] `binary_windows_wheel_py3.7_cu113`

### test
- [x] `unittest_linux_cpu_py3.7`
- [x] `unittest_linux_gpu_py3.7`
- [x] `unittest_macos_cpu_py3.7`
- [x] `unittest_windows_cpu_py3.7`
- [x] `unittest_windows_gpu_py3.7`
- [x] `integration test`

Pull Request resolved: https://github.com/pytorch/audio/pull/2140

Reviewed By: hwangjeff

Differential Revision: D33464430

Pulled By: mthrok

fbshipit-source-id: 2c5b72be75d49019bf1599036180d4e56074e46b
parent 1ccd33ec
Hide whitespace changes
Inline Side-by-side
Showing with 54 additions and 11 deletions
.circleci/config.yml
View file @ 565f8d41
......@@ -164,6 +164,8 @@ jobs:
command: |
./tools/bootstrap_ffmpeg.sh
packaging/build_wheel.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: dist
- persist_to_workspace:
......@@ -181,7 +183,10 @@ jobs:
- load_conda_channel_flags
- attach_workspace:
at: third_party
- run: packaging/build_conda.sh
- run:
command: packaging/build_conda.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: /opt/conda/conda-bld/linux-64
- persist_to_workspace:
......@@ -232,6 +237,8 @@ jobs:
source $HOME/miniconda3/bin/activate
conda install -yq conda-build
packaging/build_conda.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: /Users/distiller/miniconda3/conda-bld/osx-64
- persist_to_workspace:
......@@ -254,6 +261,8 @@ jobs:
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
bash packaging/build_wheel.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: dist
- persist_to_workspace:
......@@ -282,6 +291,8 @@ jobs:
export CONDA_CHANNEL_FLAGS="-c conda-forge"
fi
bash packaging/build_conda.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: C:/tools/miniconda3/conda-bld/win-64
- persist_to_workspace:
......@@ -457,6 +468,8 @@ jobs:
- run:
name: Install torchaudio
command: .circleci/unittest/linux/scripts/install.sh
environment:
BUILD_FFMPEG: true
- run:
name: Run tests
command: .circleci/unittest/linux/scripts/run_test.sh
......@@ -488,7 +501,7 @@ jobs:
command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
- run:
name: Install torchaudio
command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -e BUILD_FFMPEG=1 -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
- run:
name: Run tests
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "CI=${CI}" "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
......@@ -511,6 +524,8 @@ jobs:
- run:
name: Install torchaudio
command: .circleci/unittest/windows/scripts/install.sh
environment:
BUILD_FFMPEG: true
- run:
name: Run tests
command: .circleci/unittest/windows/scripts/run_test.sh
......@@ -553,6 +568,8 @@ jobs:
- run:
name: Install torchaudio
command: .circleci/unittest/windows/scripts/install.sh
environment:
BUILD_FFMPEG: true
- run:
name: Run tests
command: .circleci/unittest/windows/scripts/run_test.sh
......@@ -588,6 +605,8 @@ jobs:
- run:
name: Install torchaudio
command: .circleci/unittest/linux/scripts/install.sh
environment:
BUILD_FFMPEG: true
- run:
name: Run tests
command: .circleci/unittest/linux/scripts/run_test.sh
......
.circleci/config.yml.in
View file @ 565f8d41
......@@ -164,6 +164,8 @@ jobs:
command: |
./tools/bootstrap_ffmpeg.sh
packaging/build_wheel.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: dist
- persist_to_workspace:
......@@ -181,7 +183,10 @@ jobs:
- load_conda_channel_flags
- attach_workspace:
at: third_party
- run: packaging/build_conda.sh
- run:
command: packaging/build_conda.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: /opt/conda/conda-bld/linux-64
- persist_to_workspace:
......@@ -232,6 +237,8 @@ jobs:
source $HOME/miniconda3/bin/activate
conda install -yq conda-build
packaging/build_conda.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: /Users/distiller/miniconda3/conda-bld/osx-64
- persist_to_workspace:
......@@ -254,6 +261,8 @@ jobs:
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
bash packaging/build_wheel.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: dist
- persist_to_workspace:
......@@ -282,6 +291,8 @@ jobs:
export CONDA_CHANNEL_FLAGS="-c conda-forge"
fi
bash packaging/build_conda.sh
environment:
BUILD_FFMPEG: true
- store_artifacts:
path: C:/tools/miniconda3/conda-bld/win-64
- persist_to_workspace:
......@@ -457,6 +468,8 @@ jobs:
- run:
name: Install torchaudio
command: .circleci/unittest/linux/scripts/install.sh
environment:
BUILD_FFMPEG: true
- run:
name: Run tests
command: .circleci/unittest/linux/scripts/run_test.sh
......@@ -488,7 +501,7 @@ jobs:
command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
- run:
name: Install torchaudio
command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -e BUILD_FFMPEG=1 -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
- run:
name: Run tests
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "CI=${CI}" "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
......@@ -511,6 +524,8 @@ jobs:
- run:
name: Install torchaudio
command: .circleci/unittest/windows/scripts/install.sh
environment:
BUILD_FFMPEG: true
- run:
name: Run tests
command: .circleci/unittest/windows/scripts/run_test.sh
......@@ -553,6 +568,8 @@ jobs:
- run:
name: Install torchaudio
command: .circleci/unittest/windows/scripts/install.sh
environment:
BUILD_FFMPEG: true
- run:
name: Run tests
command: .circleci/unittest/windows/scripts/run_test.sh
......@@ -588,6 +605,8 @@ jobs:
- run:
name: Install torchaudio
command: .circleci/unittest/linux/scripts/install.sh
environment:
BUILD_FFMPEG: true
- run:
name: Run tests
command: .circleci/unittest/linux/scripts/run_test.sh
......
.circleci/unittest/linux/scripts/install.sh
View file @ 565f8d41
......@@ -52,7 +52,7 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}"
# 2. Install torchaudio
printf "* Installing torchaudio\n"
BUILD_FFMPEG=1 python setup.py install
python setup.py install
# 3. Install Test tools
printf "* Installing test tools\n"
......
.circleci/unittest/windows/scripts/setup_env.sh
View file @ 565f8d41
......@@ -39,3 +39,4 @@ conda activate "${env_dir}"
# 3. Install minimal build tools
pip --quiet install cmake ninja
conda install --quiet -y 'ffmpeg>=4.1'
.github/workflows/integration-test.yml
View file @ 565f8d41
......@@ -9,7 +9,7 @@ on:
jobs:
build:
runs-on: ubuntu-latest
runs-on: ubuntu-18.04
strategy:
fail-fast: false
matrix:
......@@ -21,12 +21,18 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
sudo apt install -y -qq pkg-config libavfilter-dev libavdevice-dev
- name: Install packages
run: |
python -m pip install --quiet --upgrade pip
python -m pip install --quiet --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
python -m pip install --quiet pytest requests cmake ninja deep-phonemizer
python setup.py install
env:
BUILD_FFMPEG: true
- name: Run integration test
run: |
cd test && pytest integration_tests -v --use-tmp-hub-dir
packaging/pkg_helpers.bash
View file @ 565f8d41
......@@ -182,9 +182,7 @@ setup_wheel_python() {
conda env remove -n "env$PYTHON_VERSION" || true
conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
conda activate "env$PYTHON_VERSION"
if [[ "$(uname)" == Darwin ]]; then
conda install --quiet -y pkg-config "ffmpeg>=4.1"
fi
conda install --quiet -y pkg-config 'ffmpeg>=4.1'
else
case "$PYTHON_VERSION" in
2.7)
......
packaging/torchaudio/build.sh
View file @ 565f8d41
......@@ -14,5 +14,4 @@ if [ "${USE_CUDA}" == "1" ] ; then
fi
fi
shopt -u nocasematch
export BUILD_FFMPEG=1
python setup.py install --single-version-externally-managed --record=record.txt
packaging/torchaudio/meta.yaml
View file @ 565f8d41
......@@ -22,7 +22,7 @@ requirements:
{{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT', 'pytorch') }}
{{ environ.get('CONDA_EXTRA_BUILD_CONSTRAINT', '') }}
{{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT', '') }}
- ffmpeg >=4.1 # [not win]
- ffmpeg >=4.1
run:
- python
......@@ -45,6 +45,7 @@ build:
- BUILD_VERSION
- USE_CUDA
- TORCH_CUDA_ARCH_LIST
- BUILD_FFMPEG
test:
imports:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment