Remove obsolete third party dependencies of CTC decoder (#3339)

Summary:
TorchAudio has migrated CTC decoder to flashlight-text, and code related CTC decoder was removed in https://github.com/pytorch/audio/issues/3236.

This commit cleans up the residual, removes the third party libraries used for CTC decoder, and mention to environment variable for CTC decoder.

Pull Request resolved: https://github.com/pytorch/audio/pull/3339

Reviewed By: nateanl

Differential Revision: D45920878

Pulled By: mthrok

fbshipit-source-id: 8d93e64138697781570e5b0b1c9f86e1a7923a89

CONTRIBUTING.md

@@ -76,7 +76,6 @@ Some environmnet variables that change the build behavior
 - `USE_CUDA`: Determines whether build the custom CUDA kernel. Default to the availability of CUDA-compatible GPUs.
 - `BUILD_KALDI`: Determines whether build Kaldi extension. This is required for `kaldi_pitch` function. Default value is 1 on Linux/macOS and 0 on Windows.
 - `BUILD_RNNT`: Determines whether build RNN-T loss function. Default value is 1.
-- `BUILD_CTC_DECODER`: Determines whether build decoder features based on KenLM and FlashLight CTC decoder. Default value is 1.
 - `BUILD_CUDA_CTC_DECODER`: Determines whether build decoder features based on CUDA CTC decoder. Default value is 1. (`USE_CUDA` has to be 1.)
 
 Please check the [./tools/setup_helpers/extension.py](./tools/setup_helpers/extension.py) for the up-to-date detail.

docs/source/build.rst

@@ -33,7 +33,6 @@ They can be enabled by passing ``1`` and disabled by ``0``.
 - ``BUILD_SOX``: Enable/disable I/O features based on libsox.
 - ``BUILD_KALDI``: Enable/disable feature extraction based on Kaldi.
 - ``BUILD_RNNT``: Enable/disable custom RNN-T loss function.
-- ``BUILD_CTC_DECODER``: Enable/disable CTC decoder based on `Flashlight Text <https://github.com/flashlight/text>`_.
 - ``USE_FFMPEG``: Enable/disable I/O features based on FFmpeg libraries.
 - ``USE_ROCM``: Enable/disable AMD ROCm support.
 - ``USE_CUDA``: Enable/disable CUDA support.

setup.py

@@ -106,7 +106,7 @@ def _parse_url(path):
 
 def _parse_sources():
     third_party_dir = ROOT_DIR / "third_party"
-    libs = ["zlib", "bzip2", "lzma", "sox"]
+    libs = ["sox"]
     archive_dir = third_party_dir / "archives"
     archive_dir.mkdir(exist_ok=True)
     for lib in libs:

third_party/CMakeLists.txt

@@ -16,16 +16,3 @@ endif()
 if (BUILD_KALDI)
   add_subdirectory(kaldi)
 endif()
-
-################################################################################
-# Decoder
-################################################################################
-if (BUILD_CTC_DECODER)
-  add_subdirectory(zlib)
-  add_subdirectory(bzip2)
-  if (NOT MSVC)
-    add_subdirectory(lzma)
-  endif()
-  add_subdirectory(kenlm)
-  add_subdirectory(flashlight-text)
-endif()

third_party/bzip2/CMakeLists.txt

-include(ExternalProject)
-
-set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install)
-set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives)
-
-if (MSVC)
-  set(
-    BZIP_LIBRARIES
-    ${INSTALL_DIR}/lib/libbz2.lib
-  )
-  ExternalProject_Add(bzip2-
-    PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-    DOWNLOAD_DIR ${ARCHIVE_DIR}
-    URL https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
-    URL_HASH SHA256=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
-    BUILD_BYPRODUCTS ${BZIP_LIBRARIES}
-    BUILD_IN_SOURCE 1
-    CONFIGURE_COMMAND nmake -f makefile.msc
-    BUILD_COMMAND cp bzlib.h ${INSTALL_DIR}/include
-    INSTALL_COMMAND cp libbz2.* ${INSTALL_DIR}/lib
-    DOWNLOAD_NO_PROGRESS ON
-    LOG_DOWNLOAD ON
-    LOG_UPDATE ON
-    LOG_CONFIGURE ON
-    LOG_BUILD ON
-    LOG_INSTALL ON
-    LOG_MERGED_STDOUTERR ON
-    LOG_OUTPUT_ON_FAILURE ON
-  )
-else()
-  set(
-    BZIP_LIBRARIES
-    ${INSTALL_DIR}/lib/libbz2.a
-  )
-  ExternalProject_Add(bzip2-
-    PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-    DOWNLOAD_DIR ${ARCHIVE_DIR}
-    URL https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
-    URL_HASH SHA256=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
-    BUILD_BYPRODUCTS ${BZIP_LIBRARIES}
-    BUILD_IN_SOURCE 1
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND make VERBOSE=1 "CFLAGS=-fPIC -fvisibility=hidden -Wall -Winline -O2 -g -D_FILE_OFFSET_BITS=64"
-    INSTALL_COMMAND make install PREFIX=${INSTALL_DIR}
-    DOWNLOAD_NO_PROGRESS ON
-    LOG_DOWNLOAD ON
-    LOG_UPDATE ON
-    LOG_CONFIGURE ON
-    LOG_BUILD ON
-    LOG_INSTALL ON
-    LOG_MERGED_STDOUTERR ON
-    LOG_OUTPUT_ON_FAILURE ON
-  )
-endif(MSVC)
-
-add_library(bzip2 INTERFACE)
-add_dependencies(bzip2 bzip2-)
-target_include_directories(bzip2 INTERFACE ${INSTALL_DIR}/include)
-target_link_libraries(bzip2 INTERFACE ${BZIP_LIBRARIES})

third_party/lzma/CMakeLists.txt

-include(ExternalProject)
-
-set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install)
-set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives)
-set(patch_dir ${PROJECT_SOURCE_DIR}/third_party/patches)
-
-# To pass custom environment variables to ExternalProject_Add command,
-# we need to do `${CMAKE_COMMAND} -E env ${envs} <COMMANAD>`.
-# https://stackoverflow.com/a/62437353
-# We constrcut the custom environment variables here
-set(envs
-  "PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig"
-  "LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}"
-  "CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden -fPIC $ENV{CFLAGS}"
-)
-
-set(
-  LZMA_LIBRARIES
-  ${INSTALL_DIR}/lib/liblzma.a
-  )
-
-ExternalProject_Add(lzma-
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://tukaani.org/xz/xz-5.2.5.tar.gz
-  URL_HASH SHA256=f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10
-  BUILD_BYPRODUCTS ${LZMA_LIBRARIES}
-  PATCH_COMMAND cp ${patch_dir}/config.guess ${patch_dir}/config.sub ${CMAKE_CURRENT_BINARY_DIR}/src/lzma-/build-aux/
-  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/lzma-/configure --prefix=${INSTALL_DIR} --disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo --disable-lzma-links --disable-scripts --disable-doc --enable-static --disable-shared
-  BUILD_COMMAND  ${CMAKE_COMMAND} -E env ${envs} make VERBOSE=1
-  DOWNLOAD_NO_PROGRESS ON
-  LOG_DOWNLOAD ON
-  LOG_UPDATE ON
-  LOG_CONFIGURE ON
-  LOG_BUILD ON
-  LOG_INSTALL ON
-  LOG_MERGED_STDOUTERR ON
-  LOG_OUTPUT_ON_FAILURE ON
-)
-
-
-add_library(lzma INTERFACE)
-add_dependencies(lzma lzma-)
-target_include_directories(lzma INTERFACE ${INSTALL_DIR}/include)
-target_link_libraries(lzma INTERFACE ${LZMA_LIBRARIES})

third_party/zlib/CMakeLists.txt

-include(ExternalProject)
-
-set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install)
-set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives)
-
-if (MSVC)
-  set(
-    ZLIB_LIBRARIES
-    ${INSTALL_DIR}/lib/zlib.lib
-  )
-
-  ExternalProject_Add(zlib-
-    PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-    DOWNLOAD_DIR ${ARCHIVE_DIR}
-    URL https://github.com/madler/zlib/archive/refs/tags/v1.2.12.tar.gz
-    URL_HASH SHA256=d8688496ea40fb61787500e863cc63c9afcbc524468cedeb478068924eb54932
-    BUILD_BYPRODUCTS ${ZLIB_LIBRARIES}
-    BUILD_IN_SOURCE 1
-    CONFIGURE_COMMAND nmake -f win32/Makefile.msc zlib.lib
-    BUILD_COMMAND cp zlib.lib ${INSTALL_DIR}/lib/
-    INSTALL_COMMAND cp zlib.h zconf.h ${INSTALL_DIR}/include/
-    LOG_DOWNLOAD ON
-    LOG_UPDATE ON
-    LOG_CONFIGURE ON
-    LOG_BUILD ON
-    LOG_INSTALL ON
-    LOG_MERGED_STDOUTERR ON
-    LOG_OUTPUT_ON_FAILURE ON
-  )
-else()
-  # To pass custom environment variables to ExternalProject_Add command,
-  # we need to do `${CMAKE_COMMAND} -E env ${envs} <COMMANAD>`.
-  # https://stackoverflow.com/a/62437353
-  # We constrcut the custom environment variables here
-  set(envs
-    "PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig"
-    "LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}"
-    "CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden -fPIC $ENV{CFLAGS}"
-    "prefix=${INSTALL_DIR}"
-  )
-
-  set(
-    ZLIB_LIBRARIES
-    ${INSTALL_DIR}/lib/libz.a
-  )
-
-  ExternalProject_Add(zlib-
-    PREFIX ${CMAKE_CURRENT_BINARY_DIR}
-    DOWNLOAD_DIR ${ARCHIVE_DIR}
-    URL https://github.com/madler/zlib/archive/refs/tags/v1.2.12.tar.gz
-    URL_HASH SHA256=d8688496ea40fb61787500e863cc63c9afcbc524468cedeb478068924eb54932
-    BUILD_BYPRODUCTS ${ZLIB_LIBRARIES}
-    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/zlib-/configure --static
-    BUILD_COMMAND ${CMAKE_COMMAND} -E env ${envs} make VERBOSE=1
-    LOG_DOWNLOAD ON
-    LOG_UPDATE ON
-    LOG_CONFIGURE ON
-    LOG_BUILD ON
-    LOG_INSTALL ON
-    LOG_MERGED_STDOUTERR ON
-    LOG_OUTPUT_ON_FAILURE ON
-  )
-endif()
-
-add_library(zlib INTERFACE)
-add_dependencies(zlib zlib-)
-target_include_directories(zlib INTERFACE ${INSTALL_DIR}/include)
-target_link_libraries(zlib INTERFACE ${ZLIB_LIBRARIES})