Skip to content

GitLab

Commit 776cf099 authored by Andrey Talman's avatar Andrey Talman Committed by Facebook GitHub Bot
Browse files

Introducing pytorch-cuda metapackage (#2612) 

Summary:
Introducing pytorch-cuda metapackage

Same as: https://github.com/pytorch/vision/pull/6371
Following PR: https://github.com/pytorch/builder/pull/1094
Adds cuda metapackage called pytorch-cuda . This way we can make sure to install correct version of cuda dependencies and don't depend on conda-forge.

Pull Request resolved: https://github.com/pytorch/audio/pull/2612

Reviewed By: hwangjeff, seemethere, nateanl

Differential Revision: D38633332

Pulled By: atalman

fbshipit-source-id: 78a6115bb252ebdb6d66a57d7d2c4a4978ddb501
parent f3bb30b8
Hide whitespace changes
Inline Side-by-side
Showing with 38 additions and 44 deletions
.circleci/config.yml
View file @ 776cf099
...@@ -387,10 +387,6 @@ jobs: ...@@ -387,10 +387,6 @@ jobs:
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0" conda install -yq conda-build "conda-package-handling!=1.5.0"
# cudatoolkit >= 11 isn't available for windows in the nvidia channel
if [[ "${CU_VERSION}" =~ cu11.* ]]; then
export CONDA_CHANNEL_FLAGS="-c conda-forge"
fi
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg" export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
bash packaging/build_conda.sh bash packaging/build_conda.sh
environment: environment:
...@@ -478,9 +474,9 @@ jobs: ...@@ -478,9 +474,9 @@ jobs:
set -x set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
if [[ "$CU_VERSION" == cu116 ]]; then if [[ "$CU_VERSION" == cu116 ]]; then
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia/label/cuda-11.6.2 pytorch cuda=${CU_VERSION:2:2}.${CU_VERSION:4} conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
else else
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4} -c conda-forge conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4}
fi fi
conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
- checkout - checkout
...@@ -566,9 +562,9 @@ jobs: ...@@ -566,9 +562,9 @@ jobs:
# Include numpy and cudatoolkit in the install conda-forge chanell is used for cudatoolkit # Include numpy and cudatoolkit in the install conda-forge chanell is used for cudatoolkit
if [[ "$CU_VERSION" == cu116 ]]; then if [[ "$CU_VERSION" == cu116 ]]; then
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia/label/cuda-11.6.2 pytorch numpy ffmpeg cuda=${CU_VERSION:2:2}.${CU_VERSION:4} conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch numpy ffmpeg pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
else else
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch numpy ffmpeg cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4} -c conda-forge conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch numpy ffmpeg cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4}
fi fi
# Install from torchaudio file # Install from torchaudio file
conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2) conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2)
...@@ -826,9 +822,9 @@ jobs: ...@@ -826,9 +822,9 @@ jobs:
set -x set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
if [[ "$CU_VERSION" == cu116 ]]; then if [[ "$CU_VERSION" == cu116 ]]; then
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia/label/cuda-11.6.2 pytorch cuda=${CU_VERSION:2:2}.${CU_VERSION:4} conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
else else
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4} -c conda-forge conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4}
fi fi
conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
# gxx_linux-64 is for installing pesq library that depends on cython # gxx_linux-64 is for installing pesq library that depends on cython
......
.circleci/config.yml.in
View file @ 776cf099
...@@ -387,10 +387,6 @@ jobs: ...@@ -387,10 +387,6 @@ jobs:
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0" conda install -yq conda-build "conda-package-handling!=1.5.0"
# cudatoolkit >= 11 isn't available for windows in the nvidia channel
if [[ "${CU_VERSION}" =~ cu11.* ]]; then
export CONDA_CHANNEL_FLAGS="-c conda-forge"
fi
export FFMPEG_ROOT="${PWD}/third_party/ffmpeg" export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
bash packaging/build_conda.sh bash packaging/build_conda.sh
environment: environment:
...@@ -478,9 +474,9 @@ jobs: ...@@ -478,9 +474,9 @@ jobs:
set -x set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
if [[ "$CU_VERSION" == cu116 ]]; then if [[ "$CU_VERSION" == cu116 ]]; then
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia/label/cuda-11.6.2 pytorch cuda=${CU_VERSION:2:2}.${CU_VERSION:4} conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
else else
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4} -c conda-forge conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4}
fi fi
conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
- checkout - checkout
...@@ -566,9 +562,9 @@ jobs: ...@@ -566,9 +562,9 @@ jobs:
# Include numpy and cudatoolkit in the install conda-forge chanell is used for cudatoolkit # Include numpy and cudatoolkit in the install conda-forge chanell is used for cudatoolkit
if [[ "$CU_VERSION" == cu116 ]]; then if [[ "$CU_VERSION" == cu116 ]]; then
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia/label/cuda-11.6.2 pytorch numpy ffmpeg cuda=${CU_VERSION:2:2}.${CU_VERSION:4} conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch numpy ffmpeg pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
else else
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch numpy ffmpeg cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4} -c conda-forge conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch numpy ffmpeg cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4}
fi fi
# Install from torchaudio file # Install from torchaudio file
conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2) conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2)
...@@ -826,9 +822,9 @@ jobs: ...@@ -826,9 +822,9 @@ jobs:
set -x set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
if [[ "$CU_VERSION" == cu116 ]]; then if [[ "$CU_VERSION" == cu116 ]]; then
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia/label/cuda-11.6.2 pytorch cuda=${CU_VERSION:2:2}.${CU_VERSION:4} conda install -v -y -c pytorch-${UPLOAD_CHANNEL} -c nvidia pytorch pytorch-cuda=${CU_VERSION:2:2}.${CU_VERSION:4}
else else
conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4} -c conda-forge conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4}
fi fi
conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
# gxx_linux-64 is for installing pesq library that depends on cython # gxx_linux-64 is for installing pesq library that depends on cython
......
.circleci/unittest/linux/scripts/install.sh
View file @ 776cf099
...@@ -34,12 +34,10 @@ else ...@@ -34,12 +34,10 @@ else
version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
export CUDATOOLKIT_CHANNEL="nvidia" export CUDATOOLKIT_CHANNEL="nvidia"
cuda_toolkit_pckg="cudatoolkit" cudatoolkit="cudatoolkit=${version}"
if [[ "$CU_VERSION" == cu116 ]]; then if [[ "$version" == "11.6" || "$version" == "11.7" ]]; then
export CUDATOOLKIT_CHANNEL="nvidia/label/cuda-11.6.2" cudatoolkit="pytorch-cuda=${version}"
cuda_toolkit_pckg="cuda"
fi fi
cudatoolkit="${cuda_toolkit_pckg}=${version}"
fi fi
printf "Installing PyTorch with %s\n" "${cudatoolkit}" printf "Installing PyTorch with %s\n" "${cudatoolkit}"
......
packaging/build_conda.sh
View file @ 776cf099
...@@ -13,21 +13,9 @@ setup_conda_pytorch_constraint ...@@ -13,21 +13,9 @@ setup_conda_pytorch_constraint
setup_conda_cudatoolkit_constraint setup_conda_cudatoolkit_constraint
setup_visual_studio_constraint setup_visual_studio_constraint
# nvidia channel included for cudatoolkit >= 11 however for 11.5 we use conda-forge
# HACK HACK HACK: Remove PYTHON_VERSION check once https://github.com/pytorch/builder/pull/961 is merged
export CUDATOOLKIT_CHANNEL="nvidia" export CUDATOOLKIT_CHANNEL="nvidia"
# NOTE: This is needed because `cudatoolkit=11.5` has a dependency on conda-forge
# See: https://github.com/pytorch/audio/pull/2224#issuecomment-1049185550
export CUDA116_CUDA_DEPENDENCY=""
if [[ ${CU_VERSION} = "cu116" ]]; then
export CUDATOOLKIT_CHANNEL="nvidia/label/cuda-11.6.2"
export CUDA116_CUDA_DEPENDENCY="cuda"
elif [[ ! -z ${CU_VERSION} ]]; then
export CUDA116_CUDA_DEPENDENCY="cudatoolkit"
fi
# NOTE: There are some dependencies that are not available for macOS on Python 3.10 without conda-forge # NOTE: There are some dependencies that are not available for macOS on Python 3.10 without conda-forge
if [[ ${OSTYPE} =~ darwin* ]] && [[ ${PYTHON_VERSION} = "3.10" ]]; then if [[ ${OSTYPE} =~ darwin* ]] && [[ ${PYTHON_VERSION} = "3.10" ]]; then
CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c conda-forge" CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c conda-forge"
fi fi
conda build -c defaults -c $CUDATOOLKIT_CHANNEL ${CONDA_CHANNEL_FLAGS:-} --no-anaconda-upload --python "$PYTHON_VERSION" $CUDA116_CUDA_DEPENDENCY packaging/torchaudio conda build -c defaults -c $CUDATOOLKIT_CHANNEL ${CONDA_CHANNEL_FLAGS:-} --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio
packaging/pkg_helpers.bash
View file @ 776cf099
...@@ -81,8 +81,12 @@ setup_cuda() { ...@@ -81,8 +81,12 @@ setup_cuda() {
;; ;;
esac esac
if [[ -n "$CUDA_HOME" ]]; then if [[ -n "$CUDA_HOME" ]]; then
# Adds nvcc binary to the search path so that CMake's `find_package(CUDA)` will pick the right one if [[ "$OSTYPE" == "msys" ]]; then
export PATH="$CUDA_HOME/bin:$PATH" export PATH="$CUDA_HOME\\bin:$PATH"
else
# Adds nvcc binary to the search path so that CMake's `find_package(CUDA)` will pick the right one
export PATH="$CUDA_HOME/bin:$PATH"
fi
export USE_CUDA=1 export USE_CUDA=1
fi fi
} }
...@@ -209,7 +213,11 @@ setup_conda_pytorch_constraint() { ...@@ -209,7 +213,11 @@ setup_conda_pytorch_constraint() {
CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS}" CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS}"
if [[ -z "$PYTORCH_VERSION" ]]; then if [[ -z "$PYTORCH_VERSION" ]]; then
export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch-nightly" export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch-nightly"
export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python3 -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" if [[ "$OSTYPE" == "msys" ]]; then
export PYTORCH_VERSION="$(conda search --json -c pytorch-nightly pytorch | python -c "import sys, json; data=json.load(sys.stdin); print(data['pytorch'][-1]['version'])")"
else
export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python3 -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")"
fi
else else
export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch -c pytorch-test -c pytorch-nightly" export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch -c pytorch-test -c pytorch-nightly"
fi fi
...@@ -237,8 +245,11 @@ setup_conda_cudatoolkit_constraint() { ...@@ -237,8 +245,11 @@ setup_conda_cudatoolkit_constraint() {
export CONDA_BUILD_VARIANT="cpu" export CONDA_BUILD_VARIANT="cpu"
else else
case "$CU_VERSION" in case "$CU_VERSION" in
cu117)
export CONDA_CUDATOOLKIT_CONSTRAINT="- pytorch-cuda=11.7 # [not osx]"
;;
cu116) cu116)
export CONDA_CUDATOOLKIT_CONSTRAINT="" export CONDA_CUDATOOLKIT_CONSTRAINT="- pytorch-cuda=11.6 # [not osx]"
;; ;;
cu113) cu113)
export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=11.3,<11.4 # [not osx]" export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=11.3,<11.4 # [not osx]"
......
packaging/torchaudio/meta.yaml
View file @ 776cf099
...@@ -45,6 +45,7 @@ build: ...@@ -45,6 +45,7 @@ build:
script_env: script_env:
- BUILD_VERSION - BUILD_VERSION
- USE_CUDA - USE_CUDA
- CUDA_HOME
- TORCH_CUDA_ARCH_LIST - TORCH_CUDA_ARCH_LIST
- USE_FFMPEG - USE_FFMPEG
- USE_OPENMP - USE_OPENMP
......
packaging/vs2019/conda_build_config.yaml
View file @ 776cf099
...@@ -5,8 +5,7 @@ c_compiler: ...@@ -5,8 +5,7 @@ c_compiler:
cxx_compiler: cxx_compiler:
- vs2019 # [win] - vs2019 # [win]
python: python:
- 3.5 - 3.7
- 3.6
# This differs from target_platform in that it determines what subdir the compiler # This differs from target_platform in that it determines what subdir the compiler
# will target, not what subdir the compiler package will be itself. # will target, not what subdir the compiler package will be itself.
# For example, we need a win-64 vs2008_win-32 package, so that we compile win-32 # For example, we need a win-64 vs2008_win-32 package, so that we compile win-32
......
tools/setup_helpers/extension.py
View file @ 776cf099
...@@ -7,6 +7,7 @@ from pathlib import Path ...@@ -7,6 +7,7 @@ from pathlib import Path
import torch import torch
from setuptools import Extension from setuptools import Extension
from setuptools.command.build_ext import build_ext from setuptools.command.build_ext import build_ext
from torch.utils.cpp_extension import CUDA_HOME
__all__ = [ __all__ = [
"get_ext_modules", "get_ext_modules",
...@@ -119,6 +120,10 @@ class CMakeBuild(build_ext): ...@@ -119,6 +120,10 @@ class CMakeBuild(build_ext):
_arches = [arch[:-4] if arch.endswith("+PTX") else f"{arch}-real" for arch in _arches] _arches = [arch[:-4] if arch.endswith("+PTX") else f"{arch}-real" for arch in _arches]
cmake_args += [f"-DCMAKE_CUDA_ARCHITECTURES={';'.join(_arches)}"] cmake_args += [f"-DCMAKE_CUDA_ARCHITECTURES={';'.join(_arches)}"]
if platform.system() != "Windows" and CUDA_HOME is not None:
cmake_args += [f"-DCMAKE_CUDA_COMPILER='{CUDA_HOME}/bin/nvcc'"]
cmake_args += [f"-DCUDA_TOOLKIT_ROOT_DIR='{CUDA_HOME}'"]
# Default to Ninja # Default to Ninja
if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows":
cmake_args += ["-GNinja"] cmake_args += ["-GNinja"]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment