Make decoder/ffmpeg libs not depend on libtorchaudio (#2121)

Summary:
`libtorchaudio_decoder`, `torchaudio_decoder` and `libtorchaudio_ffmpeg` need not to be linked against `libtorchaudio`.

![dependency graph](https://user-images.githubusercontent.com/855818/147840753-5354bc43-19a5-4a24-a357-6dd3b6ba658e.png)

Pull Request resolved: https://github.com/pytorch/audio/pull/2121

Reviewed By: carolineechen

Differential Revision: D33389197

Pulled By: mthrok

fbshipit-source-id: ebb051e894e17519a87094b8056d26e7a1cd3281

torchaudio/csrc/CMakeLists.txt

@@ -146,15 +146,11 @@ if (BUILD_CTC_DECODER)
     LIBTORCHAUDIO_DECODER_DEFINITIONS
     BUILD_CTC_DECODER
     )
-  set(
-    LIBTORCHAUDIO_DECODER_DEPS
-    libtorchaudio
-    kenlm)
   define_library(
     libtorchaudio_decoder
     "${LIBTORCHAUDIO_DECODER_SOURCES}"
     "${PROJECT_SOURCE_DIR}"
-    "${LIBTORCHAUDIO_DECODER_DEPS}"
+    "torch;kenlm"
     "${LIBTORCHAUDIO_COMPILE_DEFINITIONS};${LIBTORCHAUDIO_DECODER_DEFINITIONS}"
     )
 endif()
@@ -185,7 +181,7 @@ define_library(
   libtorchaudio_ffmpeg
   "${LIBTORCHAUDIO_FFMPEG_SOURCES}"
   "${LIBTORCHAUDIO_INCLUDE_DIRS}"
-  "libtorchaudio;ffmpeg"
+  "torch;ffmpeg"
   "${LIBTORCHAUDIO_COMPILE_DEFINITIONS}"
   )
 endif()
@@ -261,7 +257,7 @@ if (BUILD_TORCHAUDIO_PYTHON_EXTENSION)
     define_extension(
       _torchaudio_decoder
       "${DECODER_EXTENSION_SOURCES}"
-      "libtorchaudio;libtorchaudio_decoder"
+      "libtorchaudio_decoder"
       "${LIBTORCHAUDIO_DECODER_DEFINITIONS}"
       )
   endif()