Use custom FFmpeg libraries for torchaudio binary distributions (#2355)

Summary:
This commit changes the way torchaudio binary distributions are built.

* For all the binary distributions (conda/pip on Linux/macOS/Windnows), build custom FFmpeg libraries.
* The custom FFmpeg libraries do not use `--use-gpl` nor `--use-nonfree`, so that they stay LGPL.
* The custom FFmpeg libraries employ rpath so that the torchaudio binary distributions look for the corresponding FFmpeg libraries installed in the runtime environment.
* The torchaudio binary build process will use them to bootstrap its build process.
* The custom FFmpeg libraries are NOT shipped.

This commit also add disclaimer about FFmpeg in README.

Pull Request resolved: https://github.com/pytorch/audio/pull/2355

Reviewed By: nateanl

Differential Revision: D36202087

Pulled By: mthrok

fbshipit-source-id: c30e5222ba190106c897e42f567cac9152dbd8ef

.circleci/config.yml.in

@@ -124,7 +124,7 @@ jobs:
           when: always
           command: git --no-pager diff --color=always
 
-  download_third_parties_nix:
+  download_third_parties:
     docker:
       - image: "pytorch/torchaudio_unittest_base:manylinux"
     resource_class: small
@@ -151,6 +151,95 @@ jobs:
           paths:
             - archives
 
+  build_ffmpeg_linux:
+    <<: *binary_common
+    docker:
+      - image: << parameters.wheel_docker_image >>
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - generate_cache_key
+      - restore_cache:
+          {% raw %}
+          keys:
+            - ffmpeg-linux-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run:
+          command: |
+            export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
+            if [[ ! -d ${FFMPEG_ROOT} ]]; then
+                packaging/ffmpeg/build.sh
+            fi
+      - save_cache:
+          {% raw %}
+          key: ffmpeg-linux-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - third_party/ffmpeg
+      - persist_to_workspace:
+          root: third_party
+          paths:
+            - ffmpeg
+
+  build_ffmpeg_macos:
+    <<: *binary_common
+    macos:
+      xcode: "12.0"
+    steps:
+      - checkout
+      - generate_cache_key
+      - restore_cache:
+          {% raw %}
+          keys:
+            - ffmpeg-macos-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run:
+          command: |
+            export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
+            if [[ ! -d ${FFMPEG_ROOT} ]]; then
+                packaging/ffmpeg/build.sh
+            fi
+      - save_cache:
+          {% raw %}
+          key: ffmpeg-macos-v0-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - third_party/ffmpeg
+      - persist_to_workspace:
+          root: third_party
+          paths:
+            - ffmpeg
+
+  build_ffmpeg_windows:
+    <<: *binary_common
+    machine:
+      resource_class: windows.xlarge
+      image: windows-server-2019-vs2019:stable
+      # Note:
+      # Unlike other Windows job, this job uses cmd.exe as shell because
+      # we need to invoke bash.exe from msys2 in ffmpeg build process, and doing so
+      # from different installation of bash.exe (the one from the VM) cause issue
+      shell: cmd.exe
+    steps:
+      - checkout
+      - run: date /t > .cachekey
+      - restore_cache:
+          {% raw %}
+          keys:
+            - ffmpeg-windows-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run: packaging\ffmpeg\build.bat
+      - save_cache:
+          {% raw %}
+          key: ffmpeg-windows-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - third_party/ffmpeg
+      - persist_to_workspace:
+          root: third_party
+          paths:
+            - ffmpeg
+
   binary_linux_wheel:
     <<: *binary_common
     docker:
@@ -162,7 +251,7 @@ jobs:
           at: third_party
       - run:
           command: |
-            ./tools/bootstrap_ffmpeg.sh
+            export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
             packaging/build_wheel.sh
           environment:
             BUILD_FFMPEG: true
@@ -184,7 +273,9 @@ jobs:
       - attach_workspace:
           at: third_party
       - run:
-          command: packaging/build_conda.sh
+          command: |
+            export FFMPEG_ROOT=${PWD}/third_party/ffmpeg
+            packaging/build_conda.sh
           environment:
             BUILD_FFMPEG: true
       - store_artifacts:
@@ -211,6 +302,7 @@ jobs:
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
             sh conda.sh -b
             source $HOME/miniconda3/bin/activate
+            export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
             packaging/build_wheel.sh
           environment:
             BUILD_FFMPEG: true
@@ -236,6 +328,7 @@ jobs:
             sh conda.sh -b
             source $HOME/miniconda3/bin/activate
             conda install -yq conda-build
+            export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
             packaging/build_conda.sh
           environment:
             BUILD_FFMPEG: true
@@ -254,12 +347,15 @@ jobs:
       - checkout
       - load_conda_channel_flags
       - windows_install_cuda
+      - attach_workspace:
+          at: third_party
       - run:
           name: Build wheel packages
           command: |
             set -ex
             eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
             conda activate base
+            export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
             bash packaging/build_wheel.sh
           environment:
             BUILD_FFMPEG: true
@@ -278,6 +374,8 @@ jobs:
       - checkout
       - load_conda_channel_flags
       - windows_install_cuda
+      - attach_workspace:
+          at: third_party
       - run:
           name: Build conda packages
           no_output_timeout: 20m
@@ -290,6 +388,7 @@ jobs:
             if [[ "${CU_VERSION}" =~ cu11.* ]]; then
               export CONDA_CHANNEL_FLAGS="-c conda-forge"
             fi
+            export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
             bash packaging/build_conda.sh
           environment:
             BUILD_FFMPEG: true

.circleci/regenerate.py

@@ -35,6 +35,8 @@ DOC_VERSION = ("linux", "3.8")
 def build_workflows(prefix="", upload=False, filter_branch=None, indentation=6):
     w = []
     w += build_download_job(filter_branch)
+    for os_type in ["linux", "macos", "windows"]:
+        w += build_ffmpeg_job(os_type, filter_branch)
     for btype in ["wheel", "conda"]:
         for os_type in ["linux", "macos", "windows"]:
             for python_version in PYTHON_VERSIONS:
@@ -60,12 +62,24 @@ def build_workflows(prefix="", upload=False, filter_branch=None, indentation=6):
 
 def build_download_job(filter_branch):
     job = {
-        "name": "download_third_parties_nix",
+        "name": "download_third_parties",
     }
 
     if filter_branch:
         job["filters"] = gen_filter_branch_tree(filter_branch)
-    return [{"download_third_parties_nix": job}]
+    return [{"download_third_parties": job}]
+
+
+def build_ffmpeg_job(os_type, filter_branch):
+    job = {
+        "name": f"build_ffmpeg_{os_type}",
+        "requires": ["download_third_parties"],
+    }
+
+    if filter_branch:
+        job["filters"] = gen_filter_branch_tree(filter_branch)
+    job["python_version"] = "foo"
+    return [{f"build_ffmpeg_{os_type}": job}]
 
 
 def build_workflow_pair(btype, os_type, python_version, cu_version, filter_branch, prefix="", upload=False):
@@ -138,10 +152,9 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version, filte
         "name": base_workflow_name,
         "python_version": python_version,
         "cuda_version": cu_version,
+        "requires": [f"build_ffmpeg_{os_type}"],
     }
 
-    if os_type in ["linux", "macos"]:
-        d["requires"] = ["download_third_parties_nix"]
     if btype == "conda":
         d["conda_docker_image"] = f'pytorch/conda-builder:{cu_version.replace("cu1","cuda1")}'
     elif cu_version.startswith("cu"):
@@ -228,11 +241,9 @@ def unittest_workflows(indentation=6):
                     "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                     "python_version": python_version,
                     "cuda_version": "cpu" if device_type == "cpu" else "cu113",
+                    "requires": ["download_third_parties"],
                 }
 
-                if os_type != "windows":
-                    job["requires"] = ["download_third_parties_nix"]
-
                 jobs.append({f"unittest_{os_type}_{device_type}": job})
 
                 if i == 0 and os_type == "linux" and device_type == "cpu":

README.md

@@ -67,6 +67,8 @@ Please refer to https://pytorch.org/get-started/locally/ for the details.
 
 **Note** <ins>LTS versions are distributed through a different channel than the other versioned releases. Please refer to the above page for details.</ins>
 
+**Note** This software was compiled against an unmodified copy of FFmpeg (licensed under [the LGPLv2.1](https://github.com/FFmpeg/FFmpeg/blob/a5d2008e2a2360d351798e9abe883d603e231442/COPYING.LGPLv2.1)), with the specific rpath removed so as to enable the use of system libraries. The LGPL source can be downloaded [here](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.1.8).
+
 ### From Source
 
 On non-Windows platforms, the build process builds libsox and codecs that torchaudio need to link to. It will fetch and build libmad, lame, flac, vorbis, opus, and libsox before building extension. This process requires `cmake` and `pkg-config`. libsox-based features can be disabled with `BUILD_SOX=0`.

packaging/build_conda.sh

 #!/bin/bash
 set -ex
 
+echo FFMPEG_ROOT=${FFMPEG_ROOT}
+
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 . "$script_dir/pkg_helpers.bash"
 

packaging/build_wheel.sh

 #!/bin/bash
 set -ex
 
+echo FFMPEG_ROOT=${FFMPEG_ROOT}
+
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 . "$script_dir/pkg_helpers.bash"
 

packaging/ffmpeg/build.bat

+@echo off
+
+if exist "third_party\ffmpeg\" goto end
+
+choco install -y --no-progress msys2 --package-parameters "/NoUpdate"
+C:\tools\msys64\usr\bin\env MSYSTEM=MINGW64 /bin/bash -l -c "pacman -S --noconfirm --needed base-devel mingw-w64-x86_64-toolchain diffutils"
+C:\tools\msys64\usr\bin\env MSYSTEM=MINGW64 /bin/bash -l -c "cd /c/Users/circleci/project && export FFMPEG_ROOT=${PWD}/third_party/ffmpeg && packaging/vc_env_helper.bat bash ./packaging/ffmpeg/build.sh"
+
+:end

packaging/ffmpeg/build.sh

+#!/usr/bin/env bash
+
+# This script builds FFmpeg libraries without any functional features.
+#
+# IMPORTANT:
+# The resulting library files have to be LGPL version of FFmpeg libraries.
+# - Do not enable `--enable-nonfree` and `--enable-gpl`.
+# - Do not enable third party library integrations like x264.
+#
+# This script is not meant to build useful FFmpeg libraries, but to build
+# a skeleton of FFmpeg libraries that are use only during the build process of
+# torchaudio.
+#
+# The resulting FFmpeg libraries should not be shipped either.
+
+set -eux
+
+prefix="${FFMPEG_ROOT}"
+args=""
+if [[ "$OSTYPE" == "msys" ]]; then
+   args="--toolchain=msvc"
+fi
+
+build_dir=$(mktemp -d -t ffmpeg-build.XXXXXXXXXX)
+cleanup() {
+    rm -rf "${build_dir}"
+}
+trap 'cleanup $?' EXIT
+
+(
+cd "${build_dir}"
+# NOTE:
+# When changing the version of FFmpeg, update the README so that the link to the source points
+# the same version.
+curl -LsS -o ffmpeg.tar.gz https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n4.1.8.tar.gz
+tar -xf ffmpeg.tar.gz --strip-components 1
+./configure \
+    --prefix="${prefix}" \
+    --disable-all \
+    --disable-everything \
+    --disable-programs \
+    --disable-doc \
+    --disable-debug \
+    --disable-autodetect \
+    --disable-x86asm \
+    --disable-iconv \
+    --disable-encoders \
+    --disable-decoders \
+    --disable-hwaccels \
+    --disable-muxers \
+    --disable-demuxers \
+    --disable-parsers \
+    --disable-bsfs \
+    --disable-protocols \
+    --disable-devices \
+    --disable-filters \
+    --disable-asm \
+    --disable-static \
+    --enable-shared \
+    --enable-rpath \
+    --enable-pic \
+    --enable-avcodec \
+    --enable-avdevice \
+    --enable-avfilter \
+    --enable-avformat \
+    --enable-avutil ${args}
+
+make -j install
+ls ${prefix}/*
+)
+
+# macOS: Fix rpath so that the libraries are searched dynamically in user environment.
+# In Linux, this is handled by `--enable-rpath` flag.
+if [[ "$(uname)" == Darwin ]]; then
+    avcodec=libavcodec.58
+    avdevice=libavdevice.58
+    avfilter=libavfilter.7
+    avformat=libavformat.58
+    avutil=libavutil.56
+
+    # list up the paths to fix
+    for lib in ${avcodec} ${avdevice} ${avfilter} ${avformat} ${avutil}; do
+        otool -l ${prefix}/lib/${lib}.dylib | grep -B2 ${prefix}
+    done
+
+    # Replace the hardcoded paths to @rpath
+    install_name_tool \
+        -change ${prefix}/lib/${avutil}.dylib @rpath/${avutil}.dylib \
+        -delete_rpath ${prefix}/lib \
+        -id @rpath/${avcodec}.dylib \
+        ${prefix}/lib/${avcodec}.dylib
+    otool -l ${prefix}/lib/${avcodec}.dylib | grep -B2 ${prefix}
+
+    install_name_tool \
+        -change ${prefix}/lib/${avformat}.dylib @rpath/${avformat}.dylib \
+        -change ${prefix}/lib/${avcodec}.dylib @rpath/${avcodec}.dylib \
+        -change ${prefix}/lib/${avutil}.dylib @rpath/${avutil}.dylib \
+        -delete_rpath ${prefix}/lib \
+        -id @rpath/${avdevice}.dylib \
+        ${prefix}/lib/${avdevice}.dylib
+    otool -l ${prefix}/lib/${avdevice}.dylib | grep -B2 ${prefix}
+
+    install_name_tool \
+        -change ${prefix}/lib/${avutil}.dylib @rpath/${avutil}.dylib \
+        -delete_rpath ${prefix}/lib \
+        -id @rpath/${avfilter}.dylib \
+        ${prefix}/lib/${avfilter}.dylib
+    otool -l ${prefix}/lib/${avfilter}.dylib | grep -B2 ${prefix}
+
+    install_name_tool \
+        -change ${prefix}/lib/${avcodec}.dylib @rpath/${avcodec}.dylib \
+        -change ${prefix}/lib/${avutil}.dylib @rpath/${avutil}.dylib \
+        -delete_rpath ${prefix}/lib \
+        -id @rpath/${avformat}.dylib \
+        ${prefix}/lib/${avformat}.dylib
+    otool -l ${prefix}/lib/${avformat}.dylib | grep -B2 ${prefix}
+
+    install_name_tool \
+        -delete_rpath ${prefix}/lib \
+        -id @rpath/${avutil}.dylib \
+        ${prefix}/lib/${avutil}.dylib
+    otool -l ${prefix}/lib/${avutil}.dylib | grep -B2 ${prefix}
+fi

packaging/pkg_helpers.bash

@@ -190,7 +190,7 @@ setup_wheel_python() {
     conda env remove -n "env$PYTHON_VERSION" || true
     conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
     conda activate "env$PYTHON_VERSION"
-    conda install --quiet -y pkg-config 'ffmpeg>=4.1'
+    conda install --quiet -y pkg-config
   else
     case "$PYTHON_VERSION" in
       2.7)

packaging/torchaudio/meta.yaml

@@ -23,7 +23,6 @@ requirements:
     {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT', 'pytorch') }}
     {{ environ.get('CONDA_EXTRA_BUILD_CONSTRAINT', '') }}
     {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT', '') }}
-    - ffmpeg >=4.1
 
   run:
     - python
@@ -32,6 +31,7 @@ requirements:
     - pytorch-mutex 1.0 {{ build_variant }}  # [not osx ]
     {{ environ.get('CONDA_PYTORCH_CONSTRAINT', 'pytorch') }}
     {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT', '') }}
+    - ffmpeg>=4.1,<5
 
   {% if build_variant == 'cpu' %}
   run_constrained:
@@ -48,6 +48,7 @@ build:
     - USE_CUDA
     - TORCH_CUDA_ARCH_LIST
     - BUILD_FFMPEG
+    - FFMPEG_ROOT
 
 test:
   imports:
@@ -56,6 +57,7 @@ test:
     - torchaudio.kaldi_io
     - torchaudio.sox_effects
     - torchaudio.transforms
+    - torchaudio.prototype.io
 
   source_files:
     - test

tools/bootstrap_ffmpeg.sh

-#!/usr/bin/env bash
-
-# Helper script to install MINIMUM ffmpeg in centos:7.
-# The goal of this script is to allow bootstrapping the ffmpeg-feature build
-# for Linux/wheel build process which happens in centos-based Docker.
-# It is not intended to build the useful feature subset of ffmpegs
-
-set -eux
-
-build_dir=$(mktemp -d -t ffmpeg-build-XXXXXXXXXX)
-cleanup() {
-    echo rm -rf "${build_dir}"
-}
-trap cleanup EXIT
-
-cd "${build_dir}"
-
-wget --quiet -O ffmpeg.tar.gz https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n4.1.8.tar.gz
-tar -xf ffmpeg.tar.gz --strip-components 1
-./configure \
-    --disable-all \
-    --disable-static \
-    --enable-shared \
-    --enable-pic \
-    --disable-debug \
-    --disable-doc \
-    --disable-autodetect \
-    --disable-x86asm \
-    --enable-avcodec \
-    --enable-avdevice \
-    --enable-avfilter \
-    --enable-avformat \
-    --enable-avutil
-
-make -j install

torchaudio/csrc/CMakeLists.txt

@@ -182,6 +182,7 @@ if(BUILD_FFMPEG)
     ffmpeg/stream_processor.cpp
     ffmpeg/streamer.cpp
     )
+  message(STATUS "FFMPEG_ROOT=$ENV{FFMPEG_ROOT}")
   find_package(FFMPEG 4.1 REQUIRED COMPONENTS avdevice avfilter avformat avcodec avutil)
   define_library(
     libtorchaudio_ffmpeg