Skip to content

GitLab

Commit 246dd52a authored by moto's avatar moto Committed by Facebook GitHub Bot
Browse files

Add FL Decoder / KenLM integration to build process (#2078) 

Summary:
After all the C++ code from https://github.com/pytorch/audio/issues/2072 are added, this commit will enable decoder/KenLM integration in the build process.

Pull Request resolved: https://github.com/pytorch/audio/pull/2078

Reviewed By: carolineechen

Differential Revision: D33198183

Pulled By: mthrok

fbshipit-source-id: 9d7fa76151d06fbbac3785183c7c2ff9862d3128
parent 396d16cb
Hide whitespace changes
Inline Side-by-side
Showing with 109 additions and 471 deletions
CMakeLists.txt
View file @ 246dd52a
......@@ -59,7 +59,7 @@ endif()
option(BUILD_SOX "Build libsox statically" ON)
option(BUILD_KALDI "Build kaldi statically" ON)
option(BUILD_RNNT "Enable RNN transducer" ON)
option(BUILD_KENLM "Build KenLM statically" ON)
option(BUILD_CTC_DECODER "Build Flashlight CTC decoder" ON)
option(BUILD_TORCHAUDIO_PYTHON_EXTENSION "Build Python extension" OFF)
option(USE_CUDA "Enable CUDA support" OFF)
option(USE_ROCM "Enable ROCM support" OFF)
......
third_party/CMakeLists.txt
View file @ 246dd52a
......@@ -26,7 +26,7 @@ set_property(GLOBAL PROPERTY TORCHAUDIO_THIRD_PARTIES "${TORCHAUDIO_THIRD_PARTIE
################################################################################
# KenLM
################################################################################
if (BUILD_KENLM)
if (BUILD_CTC_DECODER)
add_subdirectory(zlib)
add_subdirectory(bzip2)
add_subdirectory(lzma)
......
third_party/boost/CMakeLists.txt
View file @ 246dd52a
include(ExternalProject)
set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install)
set(INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install/include)
set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives)
ExternalProject_Add(boost_
ExternalProject_Add(boost
PREFIX ${CMAKE_CURRENT_BINARY_DIR}
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz
URL_HASH SHA256=94ced8b72956591c4775ae2207a9763d3600b30d9d7446562c552f0a14a63be7
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND ./bootstrap.sh --with-libraries=thread --prefix=${INSTALL_DIR}
BUILD_COMMAND ./b2 link=static
INSTALL_COMMAND ./b2 link=static install
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory boost ${INCLUDE_DIR}/boost
DOWNLOAD_NO_PROGRESS ON
LOG_DOWNLOAD ON
LOG_UPDATE ON
......@@ -21,8 +22,3 @@ ExternalProject_Add(boost_
LOG_MERGED_STDOUTERR ON
LOG_OUTPUT_ON_FAILURE ON
)
add_library(boost INTERFACE)
add_dependencies(boost boost_)
target_include_directories(boost INTERFACE ${INSTALL_DIR}/include)
target_link_libraries(boost INTERFACE ${INSTALL_DIR}/lib/libboost_thread.a)
third_party/bzip2/CMakeLists.txt
View file @ 246dd52a
......@@ -3,13 +3,20 @@ include(ExternalProject)
set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install)
set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives)
ExternalProject_Add(bzip2_
set(
BZIP_LIBRARIES
${INSTALL_DIR}/lib/libbz2.a
)
ExternalProject_Add(bzip2-
PREFIX ${CMAKE_CURRENT_BINARY_DIR}
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
URL_HASH SHA256=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
BUILD_BYPRODUCTS ${BZIP_LIBRARIES}
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND ""
BUILD_COMMAND make VERBOSE=1 "CFLAGS=-fPIC -fvisibility=hidden -Wall -Winline -O2 -g -D_FILE_OFFSET_BITS=64"
INSTALL_COMMAND make install PREFIX=${INSTALL_DIR}
DOWNLOAD_NO_PROGRESS ON
LOG_DOWNLOAD ON
......@@ -22,6 +29,6 @@ ExternalProject_Add(bzip2_
)
add_library(bzip2 INTERFACE)
add_dependencies(bzip2 bzip2_)
add_dependencies(bzip2 bzip2-)
target_include_directories(bzip2 INTERFACE ${INSTALL_DIR}/include)
target_link_libraries(bzip2 INTERFACE ${INSTALL_DIR}/lib/libbz2.a)
target_link_libraries(bzip2 INTERFACE ${BZIP_LIBRARIES})
third_party/lzma/CMakeLists.txt
View file @ 246dd52a
......@@ -10,15 +10,22 @@ set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives)
set(envs
"PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig"
"LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}"
"CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden $ENV{CFLAGS}"
"CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden -fPIC $ENV{CFLAGS}"
)
ExternalProject_Add(lzma_
set(
LZMA_LIBRARIES
${INSTALL_DIR}/lib/liblzma.a
)
ExternalProject_Add(lzma-
PREFIX ${CMAKE_CURRENT_BINARY_DIR}
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://tukaani.org/xz/xz-5.2.5.tar.gz
URL_HASH SHA256=f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10
CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/lzma_/configure --prefix=${INSTALL_DIR} --disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo --disable-lzma-links --disable-scripts --disable-doc --enable-static --disable-shared
BUILD_BYPRODUCTS ${LZMA_LIBRARIES}
CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/lzma-/configure --prefix=${INSTALL_DIR} --disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo --disable-lzma-links --disable-scripts --disable-doc --enable-static --disable-shared
BUILD_COMMAND ${CMAKE_COMMAND} -E env ${envs} make VERBOSE=1
DOWNLOAD_NO_PROGRESS ON
LOG_DOWNLOAD ON
LOG_UPDATE ON
......@@ -31,6 +38,6 @@ ExternalProject_Add(lzma_
add_library(lzma INTERFACE)
add_dependencies(lzma lzma_)
add_dependencies(lzma lzma-)
target_include_directories(lzma INTERFACE ${INSTALL_DIR}/include)
target_link_libraries(lzma INTERFACE ${INSTALL_DIR}/lib/liblzma.a)
target_link_libraries(lzma INTERFACE ${LZMA_LIBRARIES})
third_party/zlib/CMakeLists.txt
View file @ 246dd52a
......@@ -10,17 +10,23 @@ set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives)
set(envs
"PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig"
"LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}"
"CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden $ENV{CFLAGS}"
"CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden -fPIC $ENV{CFLAGS}"
"prefix=${INSTALL_DIR}"
)
ExternalProject_Add(zlib_
set(
ZLIB_LIBRARIES
${INSTALL_DIR}/lib/libz.a
)
ExternalProject_Add(zlib-
PREFIX ${CMAKE_CURRENT_BINARY_DIR}
DOWNLOAD_DIR ${ARCHIVE_DIR}
URL https://zlib.net/zlib-1.2.11.tar.gz
URL_HASH SHA256=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/zlib_/configure --static
DOWNLOAD_NO_PROGRESS ON
BUILD_BYPRODUCTS ${ZLIB_LIBRARIES}
CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/zlib-/configure --static
BUILD_COMMAND ${CMAKE_COMMAND} -E env ${envs} make VERBOSE=1
LOG_DOWNLOAD ON
LOG_UPDATE ON
LOG_CONFIGURE ON
......@@ -31,6 +37,6 @@ ExternalProject_Add(zlib_
)
add_library(zlib INTERFACE)
add_dependencies(zlib zlib_)
add_dependencies(zlib zlib-)
target_include_directories(zlib INTERFACE ${INSTALL_DIR}/include)
target_link_libraries(zlib INTERFACE ${INSTALL_DIR}/lib/libz.a)
target_link_libraries(zlib INTERFACE ${ZLIB_LIBRARIES})
tools/setup_helpers/extension.py
View file @ 246dd52a
......@@ -37,7 +37,7 @@ def _get_build(var, default=False):
_BUILD_SOX = False if platform.system() == 'Windows' else _get_build("BUILD_SOX", True)
_BUILD_KALDI = False if platform.system() == 'Windows' else _get_build("BUILD_KALDI", True)
_BUILD_RNNT = _get_build("BUILD_RNNT", True)
_BUILD_KENLM = False if platform.system() == 'Windows' else _get_build("BUILD_KENLM", True)
_BUILD_CTC_DECODER = False if platform.system() == 'Windows' else _get_build("BUILD_CTC_DECODER", True)
_USE_ROCM = _get_build("USE_ROCM", torch.cuda.is_available() and torch.version.hip is not None)
_USE_CUDA = _get_build("USE_CUDA", torch.cuda.is_available() and torch.version.hip is None)
_USE_OPENMP = _get_build("USE_OPENMP", True) and \
......@@ -46,10 +46,16 @@ _TORCH_CUDA_ARCH_LIST = os.environ.get('TORCH_CUDA_ARCH_LIST', None)
def get_ext_modules():
return [
modules = [
Extension(name='torchaudio.lib.libtorchaudio', sources=[]),
Extension(name='torchaudio._torchaudio', sources=[]),
]
if _BUILD_CTC_DECODER:
modules.extend([
Extension(name='torchaudio.lib.libtorchaudio_decoder', sources=[]),
Extension(name='torchaudio._torchaudio_decoder', sources=[]),
])
return modules
# Based off of
......@@ -90,7 +96,7 @@ class CMakeBuild(build_ext):
f"-DBUILD_SOX:BOOL={'ON' if _BUILD_SOX else 'OFF'}",
f"-DBUILD_KALDI:BOOL={'ON' if _BUILD_KALDI else 'OFF'}",
f"-DBUILD_RNNT:BOOL={'ON' if _BUILD_RNNT else 'OFF'}",
f"-DBUILD_KENLM:BOOL={'ON' if _BUILD_KENLM else 'OFF'}",
f"-DBUILD_CTC_DECODER:BOOL={'ON' if _BUILD_CTC_DECODER else 'OFF'}",
"-DBUILD_TORCHAUDIO_PYTHON_EXTENSION:BOOL=ON",
f"-DUSE_ROCM:BOOL={'ON' if _USE_ROCM else 'OFF'}",
f"-DUSE_CUDA:BOOL={'ON' if _USE_CUDA else 'OFF'}",
......
torchaudio/csrc/CMakeLists.txt
View file @ 246dd52a
......@@ -126,6 +126,51 @@ define_library(
"${LIBTORCHAUDIO_COMPILE_DEFINITIONS}"
)
################################################################################
# libtorchaudio_decoder.so
################################################################################
if (BUILD_CTC_DECODER)
set(
LIBTORCHAUDIO_DECODER_SOURCES
decoder/src/decoder/Decoder.h
decoder/src/decoder/LexiconDecoder.cpp
decoder/src/decoder/LexiconDecoder.h
decoder/src/decoder/Trie.cpp
decoder/src/decoder/Trie.h
decoder/src/decoder/Utils.cpp
decoder/src/decoder/Utils.h
decoder/src/decoder/lm/KenLM.cpp
decoder/src/decoder/lm/KenLM.h
decoder/src/decoder/lm/LM.h
decoder/src/dictionary/Defines.h
decoder/src/dictionary/Dictionary.cpp
decoder/src/dictionary/Dictionary.h
decoder/src/dictionary/String.cpp
decoder/src/dictionary/String.h
decoder/src/dictionary/System.cpp
decoder/src/dictionary/System.h
decoder/src/dictionary/Utils.cpp
decoder/src/dictionary/Utils.h
)
set(
LIBTORCHAUDIO_DECODER_DEFINITIONS
BUILD_CTC_DECODER
)
set(
LIBTORCHAUDIO_DECODER_DEPS
libtorchaudio
kenlm)
define_library(
libtorchaudio_decoder
"${LIBTORCHAUDIO_DECODER_SOURCES}"
"${PROJECT_SOURCE_DIR}"
"${LIBTORCHAUDIO_DECODER_DEPS}"
"${LIBTORCHAUDIO_COMPILE_DEFINITIONS};${LIBTORCHAUDIO_DECODER_DEFINITIONS}"
)
endif()
# TODO: Add libtorchaudio_decoder
if (APPLE)
set(TORCHAUDIO_LIBRARY libtorchaudio CACHE INTERNAL "")
else()
......@@ -195,4 +240,16 @@ if (BUILD_TORCHAUDIO_PYTHON_EXTENSION)
libtorchaudio
"${LIBTORCHAUDIO_COMPILE_DEFINITIONS}"
)
if(BUILD_CTC_DECODER)
set(
DECODER_EXTENSION_SOURCES
decoder/bindings/pybind.cpp
)
define_extension(
_torchaudio_decoder
"${DECODER_EXTENSION_SOURCES}"
"libtorchaudio;libtorchaudio_decoder"
"${LIBTORCHAUDIO_DECODER_DEFINITIONS}"
)
endif()
endif()
torchaudio/csrc/decoder/bindings/cmake/Buildpybind11.cmake deleted 100755 → 0
View file @ 396d16cb
include(ExternalProject)
set(pybind11_URL https://github.com/pybind/pybind11.git)
set(pybind11_TAG 9a19306fbf30642ca331d0ec88e7da54a96860f9) # release 2.2.4
# Download pybind11
ExternalProject_Add(
pybind11
PREFIX pybind11
GIT_REPOSITORY ${pybind11_URL}
GIT_TAG ${pybind11_TAG}
BUILD_IN_SOURCE 0
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
)
ExternalProject_Get_Property(pybind11 SOURCE_DIR)
set(pybind11_INCLUDE_DIR "${SOURCE_DIR}/include")
\ No newline at end of file
torchaudio/csrc/decoder/bindings/cmake/FindPythonLibsNew.cmake deleted 100644 → 0
View file @ 396d16cb
# - Find python libraries
# This module finds the libraries corresponding to the Python interpreter
# FindPythonInterp provides.
# This code sets the following variables:
#
# PYTHONLIBS_FOUND - have the Python libs been found
# PYTHON_PREFIX - path to the Python installation
# PYTHON_LIBRARIES - path to the python library
# PYTHON_INCLUDE_DIRS - path to where Python.h is found
# PYTHON_MODULE_EXTENSION - lib extension, e.g. '.so' or '.pyd'
# PYTHON_MODULE_PREFIX - lib name prefix: usually an empty string
# PYTHON_SITE_PACKAGES - path to installation site-packages
# PYTHON_IS_DEBUG - whether the Python interpreter is a debug build
#
# Thanks to talljimbo for the patch adding the 'LDVERSION' config
# variable usage.
#=============================================================================
# Copyright 2001-2009 Kitware, Inc.
# Copyright 2012 Continuum Analytics, Inc.
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the names of Kitware, Inc., the Insight Software Consortium,
# nor the names of their contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#=============================================================================
# Checking for the extension makes sure that `LibsNew` was found and not just `Libs`.
if(PYTHONLIBS_FOUND AND PYTHON_MODULE_EXTENSION)
return()
endif()
# Use the Python interpreter to find the libs.
if(PythonLibsNew_FIND_REQUIRED)
find_package(PythonInterp ${PythonLibsNew_FIND_VERSION} REQUIRED)
else()
find_package(PythonInterp ${PythonLibsNew_FIND_VERSION})
endif()
if(NOT PYTHONINTERP_FOUND)
set(PYTHONLIBS_FOUND FALSE)
return()
endif()
# According to http://stackoverflow.com/questions/646518/python-how-to-detect-debug-interpreter
# testing whether sys has the gettotalrefcount function is a reliable, cross-platform
# way to detect a CPython debug interpreter.
#
# The library suffix is from the config var LDVERSION sometimes, otherwise
# VERSION. VERSION will typically be like "2.7" on unix, and "27" on windows.
execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
"from distutils import sysconfig as s;import sys;import struct;
print('.'.join(str(v) for v in sys.version_info));
print(sys.prefix);
print(s.get_python_inc(plat_specific=True));
print(s.get_python_lib(plat_specific=True));
print(s.get_config_var('SO'));
print(hasattr(sys, 'gettotalrefcount')+0);
print(struct.calcsize('@P'));
print(s.get_config_var('LDVERSION') or s.get_config_var('VERSION'));
print(s.get_config_var('LIBDIR') or '');
print(s.get_config_var('MULTIARCH') or '');
"
RESULT_VARIABLE _PYTHON_SUCCESS
OUTPUT_VARIABLE _PYTHON_VALUES
ERROR_VARIABLE _PYTHON_ERROR_VALUE)
if(NOT _PYTHON_SUCCESS MATCHES 0)
if(PythonLibsNew_FIND_REQUIRED)
message(FATAL_ERROR
"Python config failure:\n${_PYTHON_ERROR_VALUE}")
endif()
set(PYTHONLIBS_FOUND FALSE)
return()
endif()
# Convert the process output into a list
if(WIN32)
string(REGEX REPLACE "\\\\" "/" _PYTHON_VALUES ${_PYTHON_VALUES})
endif()
string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES})
string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES})
list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST)
list(GET _PYTHON_VALUES 1 PYTHON_PREFIX)
list(GET _PYTHON_VALUES 2 PYTHON_INCLUDE_DIR)
list(GET _PYTHON_VALUES 3 PYTHON_SITE_PACKAGES)
list(GET _PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION)
list(GET _PYTHON_VALUES 5 PYTHON_IS_DEBUG)
list(GET _PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P)
list(GET _PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX)
list(GET _PYTHON_VALUES 8 PYTHON_LIBDIR)
list(GET _PYTHON_VALUES 9 PYTHON_MULTIARCH)
# Make sure the Python has the same pointer-size as the chosen compiler
# Skip if CMAKE_SIZEOF_VOID_P is not defined
if(CMAKE_SIZEOF_VOID_P AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}"))
if(PythonLibsNew_FIND_REQUIRED)
math(EXPR _PYTHON_BITS "${PYTHON_SIZEOF_VOID_P} * 8")
math(EXPR _CMAKE_BITS "${CMAKE_SIZEOF_VOID_P} * 8")
message(FATAL_ERROR
"Python config failure: Python is ${_PYTHON_BITS}-bit, "
"chosen compiler is ${_CMAKE_BITS}-bit")
endif()
set(PYTHONLIBS_FOUND FALSE)
return()
endif()
# The built-in FindPython didn't always give the version numbers
string(REGEX REPLACE "\\." ";" _PYTHON_VERSION_LIST ${_PYTHON_VERSION_LIST})
list(GET _PYTHON_VERSION_LIST 0 PYTHON_VERSION_MAJOR)
list(GET _PYTHON_VERSION_LIST 1 PYTHON_VERSION_MINOR)
list(GET _PYTHON_VERSION_LIST 2 PYTHON_VERSION_PATCH)
# Make sure all directory separators are '/'
string(REGEX REPLACE "\\\\" "/" PYTHON_PREFIX ${PYTHON_PREFIX})
string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR ${PYTHON_INCLUDE_DIR})
string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES ${PYTHON_SITE_PACKAGES})
if(CMAKE_HOST_WIN32)
set(PYTHON_LIBRARY
"${PYTHON_PREFIX}/libs/Python${PYTHON_LIBRARY_SUFFIX}.lib")
# when run in a venv, PYTHON_PREFIX points to it. But the libraries remain in the
# original python installation. They may be found relative to PYTHON_INCLUDE_DIR.
if(NOT EXISTS "${PYTHON_LIBRARY}")
get_filename_component(_PYTHON_ROOT ${PYTHON_INCLUDE_DIR} DIRECTORY)
set(PYTHON_LIBRARY
"${_PYTHON_ROOT}/libs/Python${PYTHON_LIBRARY_SUFFIX}.lib")
endif()
# raise an error if the python libs are still not found.
if(NOT EXISTS "${PYTHON_LIBRARY}")
message(FATAL_ERROR "Python libraries not found")
endif()
else()
if(PYTHON_MULTIARCH)
set(_PYTHON_LIBS_SEARCH "${PYTHON_LIBDIR}/${PYTHON_MULTIARCH}" "${PYTHON_LIBDIR}")
else()
set(_PYTHON_LIBS_SEARCH "${PYTHON_LIBDIR}")
endif()
#message(STATUS "Searching for Python libs in ${_PYTHON_LIBS_SEARCH}")
# Probably this needs to be more involved. It would be nice if the config
# information the python interpreter itself gave us were more complete.
find_library(PYTHON_LIBRARY
NAMES "python${PYTHON_LIBRARY_SUFFIX}"
PATHS ${_PYTHON_LIBS_SEARCH}
NO_DEFAULT_PATH)
# If all else fails, just set the name/version and let the linker figure out the path.
if(NOT PYTHON_LIBRARY)
set(PYTHON_LIBRARY python${PYTHON_LIBRARY_SUFFIX})
endif()
endif()
MARK_AS_ADVANCED(
PYTHON_LIBRARY
PYTHON_INCLUDE_DIR
)
# We use PYTHON_INCLUDE_DIR, PYTHON_LIBRARY and PYTHON_DEBUG_LIBRARY for the
# cache entries because they are meant to specify the location of a single
# library. We now set the variables listed by the documentation for this
# module.
SET(PYTHON_INCLUDE_DIRS "${PYTHON_INCLUDE_DIR}")
SET(PYTHON_LIBRARIES "${PYTHON_LIBRARY}")
SET(PYTHON_DEBUG_LIBRARIES "${PYTHON_DEBUG_LIBRARY}")
find_package_message(PYTHON
"Found PythonLibs: ${PYTHON_LIBRARY}"
"${PYTHON_EXECUTABLE}${PYTHON_VERSION}")
set(PYTHONLIBS_FOUND TRUE)
torchaudio/csrc/decoder/bindings/cmake/pybind11Tools.cmake deleted 100755 → 0
View file @ 396d16cb
# tools/pybind11Tools.cmake -- Build system for the pybind11 modules
#
# Copyright (c) 2015 Wenzel Jakob <wenzel@inf.ethz.ch>
#
# All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
cmake_minimum_required(VERSION 2.8.12)
# Add a CMake parameter for choosing a desired Python version
if(NOT PYBIND11_PYTHON_VERSION)
set(PYBIND11_PYTHON_VERSION "" CACHE STRING "Python version to use for compiling modules")
endif()
set(Python_ADDITIONAL_VERSIONS 3.7 3.6 3.5 3.4)
find_package(PythonLibsNew ${PYBIND11_PYTHON_VERSION} REQUIRED)
include(CheckCXXCompilerFlag)
include(CMakeParseArguments)
if(NOT PYBIND11_CPP_STANDARD AND NOT CMAKE_CXX_STANDARD)
if(NOT MSVC)
check_cxx_compiler_flag("-std=c++14" HAS_CPP14_FLAG)
if (HAS_CPP14_FLAG)
set(PYBIND11_CPP_STANDARD -std=c++14)
else()
check_cxx_compiler_flag("-std=c++11" HAS_CPP11_FLAG)
if (HAS_CPP11_FLAG)
set(PYBIND11_CPP_STANDARD -std=c++11)
else()
message(FATAL_ERROR "Unsupported compiler -- pybind11 requires C++11 support!")
endif()
endif()
elseif(MSVC)
set(PYBIND11_CPP_STANDARD /std:c++14)
endif()
set(PYBIND11_CPP_STANDARD ${PYBIND11_CPP_STANDARD} CACHE STRING
"C++ standard flag, e.g. -std=c++11, -std=c++14, /std:c++14. Defaults to C++14 mode." FORCE)
endif()
# Checks whether the given CXX/linker flags can compile and link a cxx file. cxxflags and
# linkerflags are lists of flags to use. The result variable is a unique variable name for each set
# of flags: the compilation result will be cached base on the result variable. If the flags work,
# sets them in cxxflags_out/linkerflags_out internal cache variables (in addition to ${result}).
function(_pybind11_return_if_cxx_and_linker_flags_work result cxxflags linkerflags cxxflags_out linkerflags_out)
set(CMAKE_REQUIRED_LIBRARIES ${linkerflags})
check_cxx_compiler_flag("${cxxflags}" ${result})
if (${result})
set(${cxxflags_out} "${cxxflags}" CACHE INTERNAL "" FORCE)
set(${linkerflags_out} "${linkerflags}" CACHE INTERNAL "" FORCE)
endif()
endfunction()
# Internal: find the appropriate link time optimization flags for this compiler
function(_pybind11_add_lto_flags target_name prefer_thin_lto)
if (NOT DEFINED PYBIND11_LTO_CXX_FLAGS)
set(PYBIND11_LTO_CXX_FLAGS "" CACHE INTERNAL "")
set(PYBIND11_LTO_LINKER_FLAGS "" CACHE INTERNAL "")
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
set(cxx_append "")
set(linker_append "")
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT APPLE)
# Clang Gold plugin does not support -Os; append -O3 to MinSizeRel builds to override it
set(linker_append ";$<$<CONFIG:MinSizeRel>:-O3>")
elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
set(cxx_append ";-fno-fat-lto-objects")
endif()
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND prefer_thin_lto)
_pybind11_return_if_cxx_and_linker_flags_work(HAS_FLTO_THIN
"-flto=thin${cxx_append}" "-flto=thin${linker_append}"
PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS)
endif()
if (NOT HAS_FLTO_THIN)
_pybind11_return_if_cxx_and_linker_flags_work(HAS_FLTO
"-flto${cxx_append}" "-flto${linker_append}"
PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS)
endif()
elseif (CMAKE_CXX_COMPILER_ID MATCHES "Intel")
# Intel equivalent to LTO is called IPO
_pybind11_return_if_cxx_and_linker_flags_work(HAS_INTEL_IPO
"-ipo" "-ipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS)
elseif(MSVC)
# cmake only interprets libraries as linker flags when they start with a - (otherwise it
# converts /LTCG to \LTCG as if it was a Windows path). Luckily MSVC supports passing flags
# with - instead of /, even if it is a bit non-standard:
_pybind11_return_if_cxx_and_linker_flags_work(HAS_MSVC_GL_LTCG
"/GL" "-LTCG" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS)
endif()
if (PYBIND11_LTO_CXX_FLAGS)
message(STATUS "LTO enabled")
else()
message(STATUS "LTO disabled (not supported by the compiler and/or linker)")
endif()
endif()
# Enable LTO flags if found, except for Debug builds
if (PYBIND11_LTO_CXX_FLAGS)
target_compile_options(${target_name} PRIVATE "$<$<NOT:$<CONFIG:Debug>>:${PYBIND11_LTO_CXX_FLAGS}>")
endif()
if (PYBIND11_LTO_LINKER_FLAGS)
target_link_libraries(${target_name} PRIVATE "$<$<NOT:$<CONFIG:Debug>>:${PYBIND11_LTO_LINKER_FLAGS}>")
endif()
endfunction()
# Build a Python extension module:
# pybind11_add_module(<name> [MODULE | SHARED] [EXCLUDE_FROM_ALL]
# [NO_EXTRAS] [SYSTEM] [THIN_LTO] source1 [source2 ...])
#
function(pybind11_add_module target_name)
set(options MODULE SHARED EXCLUDE_FROM_ALL NO_EXTRAS SYSTEM THIN_LTO)
cmake_parse_arguments(ARG "${options}" "" "" ${ARGN})
if(ARG_MODULE AND ARG_SHARED)
message(FATAL_ERROR "Can't be both MODULE and SHARED")
elseif(ARG_SHARED)
set(lib_type SHARED)
else()
set(lib_type MODULE)
endif()
if(ARG_EXCLUDE_FROM_ALL)
set(exclude_from_all EXCLUDE_FROM_ALL)
endif()
add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS})
message("ADDING LIBRARY ${target_name}")
if(ARG_SYSTEM)
set(inc_isystem SYSTEM)
endif()
target_include_directories(${target_name} ${inc_isystem}
PRIVATE ${PYBIND11_INCLUDE_DIR} # from project CMakeLists.txt
PRIVATE ${pybind11_INCLUDE_DIR} # from pybind11Config
PRIVATE ${PYTHON_INCLUDE_DIRS})
# Python debug libraries expose slightly different objects
# https://docs.python.org/3.6/c-api/intro.html#debugging-builds
# https://stackoverflow.com/questions/39161202/how-to-work-around-missing-pymodule-create2-in-amd64-win-python35-d-lib
if(PYTHON_IS_DEBUG)
target_compile_definitions(${target_name} PRIVATE Py_DEBUG)
endif()
# The prefix and extension are provided by FindPythonLibsNew.cmake
set_target_properties(${target_name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}")
set_target_properties(${target_name} PROPERTIES SUFFIX "${PYTHON_MODULE_EXTENSION}")
# -fvisibility=hidden is required to allow multiple modules compiled against
# different pybind versions to work properly, and for some features (e.g.
# py::module_local). We force it on everything inside the `pybind11`
# namespace; also turning it on for a pybind module compilation here avoids
# potential warnings or issues from having mixed hidden/non-hidden types.
set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET "hidden")
set_target_properties(${target_name} PROPERTIES CUDA_VISIBILITY_PRESET "hidden")
if(WIN32 OR CYGWIN)
# Link against the Python shared library on Windows
target_link_libraries(${target_name} PRIVATE ${PYTHON_LIBRARIES})
elseif(APPLE)
# It's quite common to have multiple copies of the same Python version
# installed on one's system. E.g.: one copy from the OS and another copy
# that's statically linked into an application like Blender or Maya.
# If we link our plugin library against the OS Python here and import it
# into Blender or Maya later on, this will cause segfaults when multiple
# conflicting Python instances are active at the same time (even when they
# are of the same version).
# Windows is not affected by this issue since it handles DLL imports
# differently. The solution for Linux and Mac OS is simple: we just don't
# link against the Python library. The resulting shared library will have
# missing symbols, but that's perfectly fine -- they will be resolved at
# import time.
target_link_libraries(${target_name} PRIVATE "-undefined dynamic_lookup")
if(ARG_SHARED)
# Suppress CMake >= 3.0 warning for shared libraries
set_target_properties(${target_name} PROPERTIES MACOSX_RPATH ON)
endif()
endif()
# Make sure C++11/14 are enabled
target_compile_options(${target_name} PUBLIC ${PYBIND11_CPP_STANDARD})
if(ARG_NO_EXTRAS)
return()
endif()
_pybind11_add_lto_flags(${target_name} ${ARG_THIN_LTO})
if (NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug)
# Strip unnecessary sections of the binary on Linux/Mac OS
if(CMAKE_STRIP)
if(APPLE)
add_custom_command(TARGET ${target_name} POST_BUILD
COMMAND ${CMAKE_STRIP} -x $<TARGET_FILE:${target_name}>)
else()
add_custom_command(TARGET ${target_name} POST_BUILD
COMMAND ${CMAKE_STRIP} $<TARGET_FILE:${target_name}>)
endif()
endif()
endif()
if(MSVC)
# /MP enables multithreaded builds (relevant when there are many files), /bigobj is
# needed for bigger binding projects due to the limit to 64k addressable sections
target_compile_options(${target_name} PRIVATE /bigobj)
if(CMAKE_VERSION VERSION_LESS 3.11)
target_compile_options(${target_name} PRIVATE $<$<NOT:$<CONFIG:Debug>>:/MP>)
else()
# Only set these options for C++ files. This is important so that, for
# instance, projects that include other types of source files like CUDA
# .cu files don't get these options propagated to nvcc since that would
# cause the build to fail.
target_compile_options(${target_name} PRIVATE $<$<NOT:$<CONFIG:Debug>>:$<$<COMPILE_LANGUAGE:CXX>:/MP>>)
endif()
endif()
endfunction()
\ No newline at end of file
torchaudio/csrc/decoder/src/decoder/lm/KenLM.cpp
View file @ 246dd52a
......@@ -9,7 +9,7 @@
#include <stdexcept>
#include "kenlm/lm/model.hh"
#include "lm/model.hh"
namespace torchaudio {
namespace lib {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment