Revise the parameterization of third party libraries (#2282)

Summary:
Originally, the global property TORCHAUDIO_THIRD_PARTIES was introduced
to handle the optional third party dependencies that can change based on
the build config.

After revising the CMake, it turned out this is not really necessary,
as our torchaudio/csrc/CMakeLists.txt properly branches out for
conditional dependencies. Rather we should leave the global scope untouched.

Pull Request resolved: https://github.com/pytorch/audio/pull/2282

Reviewed By: hwangjeff

Differential Revision: D35059838

Pulled By: mthrok

fbshipit-source-id: ed3557eaa9a669e4466d64893beab5089eca78b8

third_party/CMakeLists.txt

-set(TORCHAUDIO_THIRD_PARTIES "")
-
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
 
 ################################################################################
@@ -10,7 +8,6 @@ if (BUILD_SOX)
   add_subdirectory(sox)
   target_include_directories(libsox INTERFACE ${SOX_INCLUDE_DIR})
   target_link_libraries(libsox INTERFACE ${SOX_LIBRARIES})
-  list(APPEND TORCHAUDIO_THIRD_PARTIES libsox)
 endif()
 
 ################################################################################
@@ -18,11 +15,8 @@ endif()
 ################################################################################
 if (BUILD_KALDI)
   add_subdirectory(kaldi)
-  list(APPEND TORCHAUDIO_THIRD_PARTIES kaldi)
 endif()
 
-set_property(GLOBAL PROPERTY TORCHAUDIO_THIRD_PARTIES "${TORCHAUDIO_THIRD_PARTIES}")
-
 ################################################################################
 # KenLM
 ################################################################################

torchaudio/csrc/CMakeLists.txt

-get_property(TORCHAUDIO_THIRD_PARTIES GLOBAL PROPERTY TORCHAUDIO_THIRD_PARTIES)
-
 ################################################################################
 # libtorchaudio
 ################################################################################
@@ -18,7 +16,6 @@ set(
 set(
   LIBTORCHAUDIO_LINK_LIBRARIES
   torch
-  ${TORCHAUDIO_THIRD_PARTIES}
   )
 
 set(
@@ -70,11 +67,17 @@ if(USE_CUDA)
 endif()
 
 if(BUILD_KALDI)
+  list(APPEND LIBTORCHAUDIO_LINK_LIBRARIES kaldi)
   list(APPEND LIBTORCHAUDIO_SOURCES kaldi.cpp)
   list(APPEND LIBTORCHAUDIO_COMPILE_DEFINITIONS INCLUDE_KALDI)
 endif()
 
 if(BUILD_SOX)
+  list(
+    APPEND
+    LIBTORCHAUDIO_LINK_LIBRARIES
+    libsox
+    )
   list(
     APPEND
     LIBTORCHAUDIO_SOURCES