Add opus support (#755)

.circleci/config.yml

@@ -38,17 +38,13 @@ commands:
               our_upload_channel=test
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
-  install_cmake_macos:
-    description: "installs cmake on macOS. Use binary distribution as brew is slow"
+  install_build_tools_macos:
+    description: "installs tools required to build torchaudio"
     steps:
       - run:
-          name: Install cmake
-          command: |
-            curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
-            mkdir cmake
-            tar -xf cmake.tar.gz --strip 3 -C cmake
-            rm cmake.tar.gz
-            echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
+          name: Install cmake and pkg-config
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config
+          # Disable brew auto update which is very slow
 
 binary_common: &binary_common
   parameters:
@@ -158,7 +154,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
-      - install_cmake_macos
+      - install_build_tools_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -183,7 +179,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
-      - install_cmake_macos
+      - install_build_tools_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -398,14 +394,14 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda
@@ -440,14 +436,14 @@ jobs:
       - restore_cache:
 
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
       - run:
           name: Setup
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
 
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
 
           paths:
             - conda

.circleci/config.yml.in

@@ -38,17 +38,13 @@ commands:
               our_upload_channel=test
             fi
             echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
-  install_cmake_macos:
-    description: "installs cmake on macOS. Use binary distribution as brew is slow"
+  install_build_tools_macos:
+    description: "installs tools required to build torchaudio"
     steps:
       - run:
-          name: Install cmake
-          command: |
-            curl -L -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5-Darwin-x86_64.tar.gz
-            mkdir cmake
-            tar -xf cmake.tar.gz --strip 3 -C cmake
-            rm cmake.tar.gz
-            echo 'export PATH='"${PWD}/cmake/bin"':${PATH}' >> ${BASH_ENV}
+          name: Install cmake and pkg-config
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config
+          # Disable brew auto update which is very slow
 
 binary_common: &binary_common
   parameters:
@@ -158,7 +154,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
-      - install_cmake_macos
+      - install_build_tools_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -183,7 +179,7 @@ jobs:
       xcode: "9.0"
     steps:
       - checkout
-      - install_cmake_macos
+      - install_build_tools_macos
       - attach_workspace:
           at: third_party
       - run:
@@ -398,14 +394,14 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda
@@ -440,14 +436,14 @@ jobs:
       - restore_cache:
           {% raw %}
           keys:
-            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
       - run:
           name: Setup
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
       - save_cache:
           {% raw %}
-          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
             - conda

.circleci/unittest/linux/docker/Dockerfile

@@ -59,6 +59,7 @@ RUN apt update && apt install -y \
         libsox-dev \
         libsox-fmt-all \
         cmake \
+        pkg-config \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /kaldi /kaldi
 COPY --from=builder /third_party /third_party

README.md

@@ -113,8 +113,8 @@ python setup.py install
 MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ python setup.py install
 ```
 
-Alternatively, the build process can build SoX (and codecs such as libmad, lame and flac) statically and torchaudio can link them, by setting environment variable `BUILD_SOX=1`.
-The build process will fetch and build SoX, liblame, libmad, flac before building extension.
+Alternatively, the build process can build libsox and some optional codecs statically and torchaudio can link them, by setting environment variable `BUILD_SOX=1`.
+The build process will fetch and build libmad, lame, flac, vorbis, opus, and libsox before building extension. This process requires `cmake` and `pkg-config`.
 
 ```bash
 # Linux

build_tools/setup_helpers/extension.py

@@ -83,6 +83,8 @@ def _get_extra_objects():
             'libmad.a',
             'libFLAC.a',
             'libmp3lame.a',
+            'libopusfile.a',
+            'libopus.a',
             'libvorbisenc.a',
             'libvorbisfile.a',
             'libvorbis.a',

test/assets/io/generate_opus.py

+"""Generate opus file for testing load functions"""
+
+import argparse
+import subprocess
+
+import scipy.io.wavfile
+import torch
+
+
+def _parse_args():
+    parser = argparse.ArgumentParser(
+        description='Generate opus files for test'
+    )
+    parser.add_argument('--num-channels', required=True, type=int)
+    parser.add_argument('--compression-level', required=True, type=int, choices=list(range(11)))
+    parser.add_argument('--bitrate', default='96k')
+    return parser.parse_args()
+
+
+def convert_to_opus(
+        src_path, dst_path,
+        *, bitrate, compression_level):
+    """Convert audio file with `ffmpeg` command."""
+    command = ['ffmpeg', '-y', '-i', src_path, '-c:a', 'libopus', '-b:a', bitrate]
+    if compression_level is not None:
+        command += ['-compression_level', str(compression_level)]
+    command += [dst_path]
+    print(' '.join(command))
+    subprocess.run(command, check=True)
+
+
+def _generate(num_channels, compression_level, bitrate):
+    org_path = 'original.wav'
+    ops_path = f'{bitrate}_{compression_level}_{num_channels}ch.opus'
+
+    # Note: ffmpeg forces sample rate 48k Hz for opus https://stackoverflow.com/a/39186779
+    # 1. generate original wav
+    data = torch.linspace(-32768, 32767, 32768, dtype=torch.int16).repeat([num_channels, 1]).t()
+    scipy.io.wavfile.write(org_path, 48000, data.numpy())
+    # 2. convert to opus
+    convert_to_opus(org_path, ops_path, bitrate=bitrate, compression_level=compression_level)
+
+
+def _main():
+    args = _parse_args()
+    _generate(args.num_channels, args.compression_level, args.bitrate)
+
+
+if __name__ == '__main__':
+    _main()

test/sox_io_backend/test_info.py

@@ -8,9 +8,10 @@ from ..common_utils import (
     PytorchTestCase,
     skipIfNoExec,
     skipIfNoExtension,
-    sox_utils,
+    get_asset_path,
     get_wav_data,
     save_wav,
+    sox_utils,
 )
 from .common import (
     name_func,
@@ -106,3 +107,19 @@ class TestInfo(TempDirMixin, PytorchTestCase):
         assert info.get_sample_rate() == sample_rate
         assert info.get_num_frames() == sample_rate * duration
         assert info.get_num_channels() == num_channels
+
+
+@skipIfNoExtension
+class TestInfoOpus(PytorchTestCase):
+    @parameterized.expand(list(itertools.product(
+        ['96k'],
+        [1, 2],
+        [0, 5, 10],
+    )), name_func=name_func)
+    def test_opus(self, bitrate, num_channels, compression_level):
+        """`sox_io_backend.info` can check opus file correcty"""
+        path = get_asset_path('io', f'{bitrate}_{compression_level}_{num_channels}ch.opus')
+        info = sox_io_backend.info(path)
+        assert info.get_sample_rate() == 48000
+        assert info.get_num_frames() == 32768
+        assert info.get_num_channels() == num_channels

test/sox_io_backend/test_load.py

@@ -8,6 +8,7 @@ from ..common_utils import (
     PytorchTestCase,
     skipIfNoExec,
     skipIfNoExtension,
+    get_asset_path,
     get_wav_data,
     load_wav,
     save_wav,
@@ -212,6 +213,23 @@ class TestLoad(LoadTestBase):
         two_hours = 2 * 60 * 60
         self.assert_vorbis(sample_rate, num_channels, quality_level, two_hours)
 
+    @parameterized.expand(list(itertools.product(
+        ['96k'],
+        [1, 2],
+        [0, 5, 10],
+    )), name_func=name_func)
+    def test_opus(self, bitrate, num_channels, compression_level):
+        """`sox_io_backend.load` can load opus file correctly."""
+        ops_path = get_asset_path('io', f'{bitrate}_{compression_level}_{num_channels}ch.opus')
+        wav_path = self.get_temp_path(f'{bitrate}_{compression_level}_{num_channels}ch.opus.wav')
+        sox_utils.convert_audio_file(ops_path, wav_path)
+
+        expected, sample_rate = load_wav(wav_path)
+        found, sr = sox_io_backend.load(ops_path)
+
+        assert sample_rate == sr
+        self.assertEqual(expected, found)
+
 
 @skipIfNoExec('sox')
 @skipIfNoExtension

third_party/CMakeLists.txt

@@ -38,7 +38,7 @@ ExternalProject_Add(libflac
   DOWNLOAD_DIR ${ARCHIVE_DIR}
   URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
   URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libflac/configure ${COMMON_ARGS} --with-ogg=${INSTALL_DIR}
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libflac/configure ${COMMON_ARGS} --with-ogg
 )
 
 ExternalProject_Add(libvorbis
@@ -47,14 +47,34 @@ ExternalProject_Add(libvorbis
   DOWNLOAD_DIR ${ARCHIVE_DIR}
   URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
   URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libvorbis/configure ${COMMON_ARGS} --with-ogg=${INSTALL_DIR}
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libvorbis/configure ${COMMON_ARGS} --with-ogg
+)
+
+ExternalProject_Add(libopus
+  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS libogg
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
+  URL_HASH SHA256=65b58e1e25b2a114157014736a3d9dfeaad8d41be1c8179866f144a2fb44ff9d
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libopus/configure ${COMMON_ARGS} --with-ogg
+)
+
+ExternalProject_Add(opusfile
+  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS libopus
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  STAMP_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile-stamp
+  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile
+  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
+  URL_HASH SHA256=118d8601c12dd6a44f52423e68ca9083cc9f2bfe72da7a8c1acb22a80ae3550b
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile/configure ${COMMON_ARGS} --disable-http
 )
 
 ExternalProject_Add(libsox
   PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libogg libflac libvorbis libmp3lame libmad
+  DEPENDS libogg libflac libvorbis opusfile libmp3lame libmad
   DOWNLOAD_DIR ${ARCHIVE_DIR}
   URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
   URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} LDFLAGS=-L${INSTALL_DIR}/lib CPPFLAGS=-I${INSTALL_DIR}/include --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile
+  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile --with-opus
 )

third_party/build_codec_helper.sh

+#!/usr/bin/env bash
+# Helper script for building codecs depending on libogg, such as libopus and opus.
+# It is difficult to set environment variable inside of ExternalProject_Add,
+# so this script sets necessary environment variables before running the given command
+
+this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+install_dir="${this_dir}/install"
+
+export PKG_CONFIG_PATH="${install_dir}/lib/pkgconfig"
+export LDFLAGS="-L${install_dir}/lib ${LDFLAGS}"
+export CPPFLAGS="-I${install_dir}/include ${CPPFLAGS}"
+
+$@