Fix CUDA build logic for _torchaudio.so (#1737)

It's wrong to depend on `${TORCH_LIBRARIES}` as it pulls in explicit
`libcuda.so.1` dependency, which violates the assumption that GPU
accelerated libraries should be loadable with no NVIDIA drivers installed

Instead, make it depend on `torch` target, which includes all necessary
Torch C++ API dependences

third_party/kaldi/CMakeLists.txt

@@ -29,4 +29,4 @@ set(KALDI_SOURCES
 
 add_library(kaldi STATIC ${KALDI_SOURCES})
 target_include_directories(kaldi PUBLIC src submodule/src)
-target_link_libraries(kaldi ${TORCH_LIBRARIES})
+target_include_directories(kaldi PRIVATE ${TORCH_INCLUDE_DIRS})

torchaudio/csrc/CMakeLists.txt

@@ -127,6 +127,13 @@ if (BUILD_TORCHAUDIO_PYTHON_EXTENSION)
     ${PROJECT_SOURCE_DIR}
     ${Python_INCLUDE_DIR}
     )
+  if(USE_CUDA)
+    target_include_directories(
+      _torchaudio
+      PRIVATE
+      ${CUDA_TOOLKIT_INCLUDE}
+      )
+  endif()
 
   # See https://github.com/pytorch/pytorch/issues/38122
   find_library(TORCH_PYTHON_LIBRARY torch_python PATHS "${TORCH_INSTALL_PREFIX}/lib")
@@ -138,12 +145,20 @@ if (BUILD_TORCHAUDIO_PYTHON_EXTENSION)
 
   target_link_libraries(
     _torchaudio
-    ${TORCH_LIBRARIES}
+    torch
     ${TORCH_PYTHON_LIBRARY}
     ${TORCHAUDIO_THIRD_PARTIES}
     ${ADDITIONAL_ITEMS}
     )
 
+  if(USE_CUDA)
+    target_link_libraries(
+      _torchaudio
+      ${C10_CUDA_LIBRARY}
+      ${CUDA_CUDART_LIBRARY}
+      )
+  endif()
+
   install(
     TARGETS _torchaudio
     LIBRARY DESTINATION .