Fix MKL issue on Intel mac build (#3307)

Summary:
* Remove MKL and NumPy from Conda build env
* Remove `caffe2::mkl` dependency from `torch_cpu`, which introduced unnecessary and undesided dependency on Intel mac.

TorchAudio does not use BLAS libraries directly, thus all the mentions to MKL should be removed from the codebase.
However, this was causing an issue on Intel mac. It turned out that `torch_cpu` target is pulling `caffe2::mkl` dependency, and the linker on macOS keeps library dependency even if no symbol from that library is used. This stray mkl dependency should be fixed on core side, but also we can modify the target temporarily and remove them.

Also we don't need NumPy on build/run time, so that is removed as well.

Pull Request resolved: https://github.com/pytorch/audio/pull/3307

Reviewed By: atalman

Differential Revision: D45606944

Pulled By: mthrok

fbshipit-source-id: 853411ccbbca31796b808a2b052b4cfa564718cd

cmake/TorchAudioHelper.cmake

 find_package(Torch REQUIRED)
 
+# Remove stray mkl dependency found in Intel mac.
+#
+# For Intel mac, torch_cpu has caffe2::mkl, which adds link flags like
+# -lmkl_intel_ilp64, -lmkl_core and -lmkl_intel_thread.
+# Even though TorchAudio does not call any of MKL functions directly,
+# Apple's linker does not drop them, instead it bakes these dependencies
+# Therefore, we remove it.
+# See https://github.com/pytorch/audio/pull/3307
+get_target_property(dep torch_cpu INTERFACE_LINK_LIBRARIES)
+if ("caffe2::mkl" IN_LIST dep)
+  list(REMOVE_ITEM dep "caffe2::mkl")
+  set_target_properties(torch_cpu PROPERTIES INTERFACE_LINK_LIBRARIES "${dep}")
+endif()
+
 function (torchaudio_library name source include_dirs link_libraries compile_defs)
   add_library(${name} SHARED ${source})
   target_include_directories(${name} PRIVATE "${PROJECT_SOURCE_DIR};${include_dirs}")

packaging/pkg_helpers.bash

@@ -240,10 +240,6 @@ setup_conda_pytorch_constraint() {
   # TODO: Remove me later, see https://github.com/pytorch/pytorch/issues/62424 for more details
   if [[ "$(uname)" == Darwin ]]; then
     arch_name="$(uname -m)"
-    if [[ "${arch_name}" != "arm64" && "${PYTHON_VERSION}" != "3.11" ]]; then
-      # Use less than equal to avoid version conflict in python=3.6 environment
-      export CONDA_EXTRA_BUILD_CONSTRAINT="- mkl<=2021.2.0"
-    fi
   fi
 }
 

packaging/torchaudio/meta.yaml

@@ -19,7 +19,6 @@ requirements:
     - ninja
     - numpy>=1.11 # [py <= 39]
     - numpy>=1.21.2 # [py >= 310]
-    - mkl<=2021.2.0 # [osx and x86_64]
     - pytorch-mutex 1.0 {{ build_variant }}  # [not osx ]
     {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT', 'pytorch') }}
     {{ environ.get('CONDA_EXTRA_BUILD_CONSTRAINT', '') }}

packaging/vs2019/conda_build_config.yaml

-blas_impl:
-  - mkl                        # [x86_64]
 c_compiler:
   - vs2019                     # [win]
 cxx_compiler: