Enable build ffmpeg-features in all related jobs (#2140)

Summary:
This commit enables ffmpeg-feature build in tests and
binary builds of all platforms.
(Linux/macOS/Windows x conda/wheel)

It also moves the definition of BUILD_FFMPEG env vars to the
top level `config.yml`.

 ---
Manual checking if all the build log contains `libtorchaudio_ffmpeg`.
### binary build
- [x] `binary_linux_conda_py3.7_cpu`
- [x] `binary_linux_conda_py3.7_cu102`
- [x] `binary_linux_wheel_py3.7_cpu`
- [x] `binary_linux_wheel_py3.7_cu102`
- [x] `binary_macos_conda_py3.7_cpu`
- [x] `binary_macos_wheel_py3.7_cpu`
- [x] `binary_windows_conda_py3.7_cpu`
- [x] `binary_windows_conda_py3.7_cu113`
- [x] `binary_windows_wheel_py3.7_cpu`
- [x] `binary_windows_wheel_py3.7_cu113`

### test
- [x] `unittest_linux_cpu_py3.7`
- [x] `unittest_linux_gpu_py3.7`
- [x] `unittest_macos_cpu_py3.7`
- [x] `unittest_windows_cpu_py3.7`
- [x] `unittest_windows_gpu_py3.7`
- [x] `integration test`

Pull Request resolved: https://github.com/pytorch/audio/pull/2140

Reviewed By: hwangjeff

Differential Revision: D33464430

Pulled By: mthrok

fbshipit-source-id: 2c5b72be75d49019bf1599036180d4e56074e46b

.circleci/config.yml

@@ -164,6 +164,8 @@ jobs:
           command: |
             ./tools/bootstrap_ffmpeg.sh
             packaging/build_wheel.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: dist
       - persist_to_workspace:
@@ -181,7 +183,10 @@ jobs:
       - load_conda_channel_flags
       - attach_workspace:
           at: third_party
-      - run: packaging/build_conda.sh
+      - run:
+          command: packaging/build_conda.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: /opt/conda/conda-bld/linux-64
       - persist_to_workspace:
@@ -232,6 +237,8 @@ jobs:
             source $HOME/miniconda3/bin/activate
             conda install -yq conda-build
             packaging/build_conda.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: /Users/distiller/miniconda3/conda-bld/osx-64
       - persist_to_workspace:
@@ -254,6 +261,8 @@ jobs:
             eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
             conda activate base
             bash packaging/build_wheel.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: dist
       - persist_to_workspace:
@@ -282,6 +291,8 @@ jobs:
               export CONDA_CHANNEL_FLAGS="-c conda-forge"
             fi
             bash packaging/build_conda.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: C:/tools/miniconda3/conda-bld/win-64
       - persist_to_workspace:
@@ -457,6 +468,8 @@ jobs:
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
+          environment:
+              BUILD_FFMPEG: true
       - run:
           name: Run tests
           command: .circleci/unittest/linux/scripts/run_test.sh
@@ -488,7 +501,7 @@ jobs:
           command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - run:
           name: Install torchaudio
-          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
+          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -e BUILD_FFMPEG=1 -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "CI=${CI}" "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
@@ -511,6 +524,8 @@ jobs:
       - run:
           name: Install torchaudio
           command: .circleci/unittest/windows/scripts/install.sh
+          environment:
+              BUILD_FFMPEG: true
       - run:
           name: Run tests
           command: .circleci/unittest/windows/scripts/run_test.sh
@@ -553,6 +568,8 @@ jobs:
       - run:
           name: Install torchaudio
           command: .circleci/unittest/windows/scripts/install.sh
+          environment:
+              BUILD_FFMPEG: true
       - run:
           name: Run tests
           command: .circleci/unittest/windows/scripts/run_test.sh
@@ -588,6 +605,8 @@ jobs:
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
+          environment:
+              BUILD_FFMPEG: true
       - run:
           name: Run tests
           command: .circleci/unittest/linux/scripts/run_test.sh

.circleci/config.yml.in

@@ -164,6 +164,8 @@ jobs:
           command: |
             ./tools/bootstrap_ffmpeg.sh
             packaging/build_wheel.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: dist
       - persist_to_workspace:
@@ -181,7 +183,10 @@ jobs:
       - load_conda_channel_flags
       - attach_workspace:
           at: third_party
-      - run: packaging/build_conda.sh
+      - run:
+          command: packaging/build_conda.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: /opt/conda/conda-bld/linux-64
       - persist_to_workspace:
@@ -232,6 +237,8 @@ jobs:
             source $HOME/miniconda3/bin/activate
             conda install -yq conda-build
             packaging/build_conda.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: /Users/distiller/miniconda3/conda-bld/osx-64
       - persist_to_workspace:
@@ -254,6 +261,8 @@ jobs:
             eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
             conda activate base
             bash packaging/build_wheel.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: dist
       - persist_to_workspace:
@@ -282,6 +291,8 @@ jobs:
               export CONDA_CHANNEL_FLAGS="-c conda-forge"
             fi
             bash packaging/build_conda.sh
+          environment:
+            BUILD_FFMPEG: true
       - store_artifacts:
           path: C:/tools/miniconda3/conda-bld/win-64
       - persist_to_workspace:
@@ -457,6 +468,8 @@ jobs:
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
+          environment:
+              BUILD_FFMPEG: true
       - run:
           name: Run tests
           command: .circleci/unittest/linux/scripts/run_test.sh
@@ -488,7 +501,7 @@ jobs:
           command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - run:
           name: Install torchaudio
-          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
+          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -e BUILD_FFMPEG=1 -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "CI=${CI}" "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
@@ -511,6 +524,8 @@ jobs:
       - run:
           name: Install torchaudio
           command: .circleci/unittest/windows/scripts/install.sh
+          environment:
+              BUILD_FFMPEG: true
       - run:
           name: Run tests
           command: .circleci/unittest/windows/scripts/run_test.sh
@@ -553,6 +568,8 @@ jobs:
       - run:
           name: Install torchaudio
           command: .circleci/unittest/windows/scripts/install.sh
+          environment:
+              BUILD_FFMPEG: true
       - run:
           name: Run tests
           command: .circleci/unittest/windows/scripts/run_test.sh
@@ -588,6 +605,8 @@ jobs:
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
+          environment:
+              BUILD_FFMPEG: true
       - run:
           name: Run tests
           command: .circleci/unittest/linux/scripts/run_test.sh

.circleci/unittest/linux/scripts/install.sh

@@ -52,7 +52,7 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}"
 
 # 2. Install torchaudio
 printf "* Installing torchaudio\n"
-BUILD_FFMPEG=1 python setup.py install
+python setup.py install
 
 # 3. Install Test tools
 printf "* Installing test tools\n"

.circleci/unittest/windows/scripts/setup_env.sh

@@ -39,3 +39,4 @@ conda activate "${env_dir}"
 
 # 3. Install minimal build tools
 pip --quiet install cmake ninja
+conda install --quiet -y 'ffmpeg>=4.1'

.github/workflows/integration-test.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     strategy:
       fail-fast: false
       matrix:
@@ -21,12 +21,18 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
+        sudo apt install -y -qq pkg-config libavfilter-dev libavdevice-dev
     - name: Install packages
       run: |
         python -m pip install --quiet --upgrade pip
         python -m pip install --quiet --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
         python -m pip install --quiet pytest requests cmake ninja deep-phonemizer
         python setup.py install
+      env:
+        BUILD_FFMPEG: true
     - name: Run integration test
       run: |
         cd test && pytest integration_tests -v --use-tmp-hub-dir

packaging/pkg_helpers.bash

@@ -182,9 +182,7 @@ setup_wheel_python() {
     conda env remove -n "env$PYTHON_VERSION" || true
     conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
     conda activate "env$PYTHON_VERSION"
-    if [[ "$(uname)" == Darwin ]]; then
-        conda install --quiet -y pkg-config "ffmpeg>=4.1"
-    fi
+    conda install --quiet -y pkg-config 'ffmpeg>=4.1'
   else
     case "$PYTHON_VERSION" in
       2.7)

packaging/torchaudio/build.sh

@@ -14,5 +14,4 @@ if [ "${USE_CUDA}" == "1" ] ; then
     fi
 fi
 shopt -u nocasematch
-export BUILD_FFMPEG=1
 python setup.py install --single-version-externally-managed --record=record.txt

packaging/torchaudio/meta.yaml

@@ -22,7 +22,7 @@ requirements:
     {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT', 'pytorch') }}
     {{ environ.get('CONDA_EXTRA_BUILD_CONSTRAINT', '') }}
     {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT', '') }}
-    - ffmpeg >=4.1  # [not win]
+    - ffmpeg >=4.1
 
   run:
     - python
@@ -45,6 +45,7 @@ build:
     - BUILD_VERSION
     - USE_CUDA
     - TORCH_CUDA_ARCH_LIST
+    - BUILD_FFMPEG
 
 test:
   imports: