Skip to content

GitLab

"tests/python/pytorch/graphbolt/test_minibatch.py" did not exist on "382a2de73c7d576a4e37d9ab6d6e8e54c12609db"
Unverified Commit 155de2b4 authored by Edward Z. Yang's avatar Edward Z. Yang Committed by GitHub
Browse files

Packaging fixes (#1214) 

Add uploading support, make CUDA builds actually work.
parent 64ccfe34
Show whitespace changes
Inline Side-by-side
Showing with 745 additions and 213 deletions
.circleci/config.yml
View file @ 155de2b4
...@@ -2,8 +2,9 @@ version: 2.1 ...@@ -2,8 +2,9 @@ version: 2.1
# How to test the Linux jobs: # How to test the Linux jobs:
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel # - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
# - Replace binary_linux_wheel with the name of the job you want to test # - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
# Job names are 'name:' key.
binary_common: &binary_common binary_common: &binary_common
parameters: parameters:
...@@ -20,19 +21,23 @@ binary_common: &binary_common ...@@ -20,19 +21,23 @@ binary_common: &binary_common
python_version: python_version:
description: "Python version to build against (e.g., 3.7)" description: "Python version to build against (e.g., 3.7)"
type: string type: string
cuda_version: cu_version:
description: "CUDA version to build against (e.g., cpu or 10.0)" description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
type: string type: string
unicode_abi: unicode_abi:
description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
type: string type: string
default: "" default: ""
wheel_docker_image:
description: "Wheel only: what docker image to use"
type: string
default: "soumith/manylinux-cuda100"
environment: environment:
PYTHON_VERSION: << parameters.python_version >> PYTHON_VERSION: << parameters.python_version >>
BUILD_VERSION: << parameters.build_version >> BUILD_VERSION: << parameters.build_version >>
PYTORCH_VERSION: << parameters.pytorch_version >> PYTORCH_VERSION: << parameters.pytorch_version >>
UNICODE_ABI: << parameters.unicode_abi >> UNICODE_ABI: << parameters.unicode_abi >>
CUDA_VERSION: << parameters.cuda_version >> CU_VERSION: << parameters.cu_version >>
jobs: jobs:
circleci_consistency: circleci_consistency:
...@@ -49,13 +54,17 @@ jobs: ...@@ -49,13 +54,17 @@ jobs:
binary_linux_wheel: binary_linux_wheel:
<<: *binary_common <<: *binary_common
docker: docker:
- image: "soumith/manylinux-cuda100" - image: << parameters.wheel_docker_image >>
resource_class: 2xlarge+ resource_class: 2xlarge+
steps: steps:
- checkout - checkout
- run: packaging/build_wheel.sh - run: packaging/build_wheel.sh
- store_artifacts: - store_artifacts:
path: dist path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_linux_conda: binary_linux_conda:
<<: *binary_common <<: *binary_common
...@@ -67,6 +76,10 @@ jobs: ...@@ -67,6 +76,10 @@ jobs:
- run: packaging/build_conda.sh - run: packaging/build_conda.sh
- store_artifacts: - store_artifacts:
path: /opt/conda/conda-bld/linux-64 path: /opt/conda/conda-bld/linux-64
- persist_to_workspace:
root: /opt/conda/conda-bld/linux-64
paths:
- "*"
binary_macos_wheel: binary_macos_wheel:
<<: *binary_common <<: *binary_common
...@@ -85,6 +98,10 @@ jobs: ...@@ -85,6 +98,10 @@ jobs:
packaging/build_wheel.sh packaging/build_wheel.sh
- store_artifacts: - store_artifacts:
path: dist path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_macos_conda: binary_macos_conda:
<<: *binary_common <<: *binary_common
...@@ -101,188 +118,558 @@ jobs: ...@@ -101,188 +118,558 @@ jobs:
packaging/build_conda.sh packaging/build_conda.sh
- store_artifacts: - store_artifacts:
path: /Users/distiller/miniconda3/conda-bld/osx-64 path: /Users/distiller/miniconda3/conda-bld/osx-64
- persist_to_workspace:
root: /Users/distiller/miniconda3/conda-bld/osx-64
paths:
- "*"
# Requires org-member context
binary_conda_upload:
docker:
- image: continuumio/miniconda
steps:
- attach_workspace:
at: ~/workspace
- run:
command: |
# Prevent credential from leaking
conda install -yq anaconda-client
set +x
anaconda login \
--username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
--password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
set -x
anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force
# Requires org-member context
binary_wheel_upload:
docker:
- image: circleci/python:3.7
steps:
- attach_workspace:
at: ~/workspace
- checkout
- run:
command: |
pip install --user awscli
export PATH="$HOME/.local/bin:$PATH"
# Prevent credential from leaking
set +x
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
#for pkg in ~/workspace/*.whl; do
# aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
#done
workflows: workflows:
build: build:
jobs: jobs:
- circleci_consistency - circleci_consistency
- binary_linux_wheel:
name: binary_linux_wheel_py2.7_cpu
python_version: "2.7"
cu_version: "cpu"
- binary_linux_wheel:
name: binary_linux_wheel_py2.7u_cpu
python_version: "2.7"
cu_version: "cpu"
unicode_abi: "1"
- binary_linux_wheel:
name: binary_linux_wheel_py2.7_cu92
python_version: "2.7"
cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_linux_wheel:
name: binary_linux_wheel_py2.7u_cu92
python_version: "2.7"
cu_version: "cu92"
unicode_abi: "1"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_linux_wheel:
name: binary_linux_wheel_py2.7_cu100
python_version: "2.7"
cu_version: "cu100"
- binary_linux_wheel:
name: binary_linux_wheel_py2.7u_cu100
python_version: "2.7"
cu_version: "cu100"
unicode_abi: "1"
- binary_linux_wheel:
name: binary_linux_wheel_py3.5_cpu
python_version: "3.5"
cu_version: "cpu"
- binary_linux_wheel:
name: binary_linux_wheel_py3.5_cu92
python_version: "3.5"
cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_linux_wheel:
name: binary_linux_wheel_py3.5_cu100
python_version: "3.5"
cu_version: "cu100"
- binary_linux_wheel:
name: binary_linux_wheel_py3.6_cpu
python_version: "3.6"
cu_version: "cpu"
- binary_linux_wheel:
name: binary_linux_wheel_py3.6_cu92
python_version: "3.6"
cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_linux_wheel:
name: binary_linux_wheel_py3.6_cu100
python_version: "3.6"
cu_version: "cu100"
- binary_linux_wheel:
name: binary_linux_wheel_py3.7_cpu
python_version: "3.7"
cu_version: "cpu"
- binary_linux_wheel:
name: binary_linux_wheel_py3.7_cu92
python_version: "3.7"
cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_linux_wheel:
name: binary_linux_wheel_py3.7_cu100
python_version: "3.7"
cu_version: "cu100"
- binary_macos_wheel:
name: binary_macos_wheel_py2.7_cpu
python_version: "2.7"
cu_version: "cpu"
- binary_macos_wheel:
name: binary_macos_wheel_py2.7u_cpu
python_version: "2.7"
cu_version: "cpu"
unicode_abi: "1"
- binary_macos_wheel:
name: binary_macos_wheel_py3.5_cpu
python_version: "3.5"
cu_version: "cpu"
- binary_macos_wheel:
name: binary_macos_wheel_py3.6_cpu
python_version: "3.6"
cu_version: "cpu"
- binary_macos_wheel:
name: binary_macos_wheel_py3.7_cpu
python_version: "3.7"
cu_version: "cpu"
- binary_linux_conda:
name: binary_linux_conda_py2.7_cpu
python_version: "2.7"
cu_version: "cpu"
- binary_linux_conda:
name: binary_linux_conda_py2.7_cu92
python_version: "2.7"
cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_linux_conda:
name: binary_linux_conda_py2.7_cu100
python_version: "2.7"
cu_version: "cu100"
- binary_linux_conda:
name: binary_linux_conda_py3.5_cpu
python_version: "3.5"
cu_version: "cpu"
- binary_linux_conda:
name: binary_linux_conda_py3.5_cu92
python_version: "3.5"
cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_linux_conda:
name: binary_linux_conda_py3.5_cu100
python_version: "3.5"
cu_version: "cu100"
- binary_linux_conda:
name: binary_linux_conda_py3.6_cpu
python_version: "3.6"
cu_version: "cpu"
- binary_linux_conda:
name: binary_linux_conda_py3.6_cu92
python_version: "3.6"
cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_linux_conda:
name: binary_linux_conda_py3.6_cu100
python_version: "3.6"
cu_version: "cu100"
- binary_linux_conda:
name: binary_linux_conda_py3.7_cpu
python_version: "3.7"
cu_version: "cpu"
- binary_linux_conda:
name: binary_linux_conda_py3.7_cu92
python_version: "3.7"
cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_linux_conda:
name: binary_linux_conda_py3.7_cu100
python_version: "3.7"
cu_version: "cu100"
- binary_macos_conda:
name: binary_macos_conda_py2.7_cpu
python_version: "2.7"
cu_version: "cpu"
- binary_macos_conda:
name: binary_macos_conda_py3.5_cpu
python_version: "3.5"
cu_version: "cpu"
- binary_macos_conda:
name: binary_macos_conda_py3.6_cpu
python_version: "3.6"
cu_version: "cpu"
- binary_macos_conda:
name: binary_macos_conda_py3.7_cpu
python_version: "3.7"
cu_version: "cpu"
nightly:
triggers:
- schedule:
cron: "0 9 * * *"
filters:
branches:
only:
- master
jobs:
- circleci_consistency
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py2.7_cpu
name: binary_linux_wheel_py2.7_cudacpu
python_version: "2.7" python_version: "2.7"
cuda_version: "cpu" cu_version: "cpu"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py2.7_cpu_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7_cpu
- binary_linux_wheel: - binary_linux_wheel:
name: binary_linux_wheel_py2.7_cudacpu_unicode name: nightly_binary_linux_wheel_py2.7u_cpu
python_version: "2.7" python_version: "2.7"
cuda_version: "cpu" cu_version: "cpu"
unicode_abi: "1" unicode_abi: "1"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py2.7u_cpu_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7u_cpu
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py2.7_cu92
name: binary_linux_wheel_py2.7_cuda10.0
python_version: "2.7" python_version: "2.7"
cuda_version: "10.0" cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py2.7_cu92_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7_cu92
- binary_linux_wheel: - binary_linux_wheel:
name: binary_linux_wheel_py2.7_cuda10.0_unicode name: nightly_binary_linux_wheel_py2.7u_cu92
python_version: "2.7" python_version: "2.7"
cuda_version: "10.0" cu_version: "cu92"
unicode_abi: "1" unicode_abi: "1"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py2.7u_cu92_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7u_cu92
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py2.7_cu100
name: binary_linux_wheel_py2.7_cuda9.2
python_version: "2.7" python_version: "2.7"
cuda_version: "9.2" cu_version: "cu100"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py2.7_cu100_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7_cu100
- binary_linux_wheel: - binary_linux_wheel:
name: binary_linux_wheel_py2.7_cuda9.2_unicode name: nightly_binary_linux_wheel_py2.7u_cu100
python_version: "2.7" python_version: "2.7"
cuda_version: "9.2" cu_version: "cu100"
unicode_abi: "1" unicode_abi: "1"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py2.7u_cu100_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7u_cu100
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py3.5_cpu
name: binary_linux_wheel_py3.5_cudacpu
python_version: "3.5" python_version: "3.5"
cuda_version: "cpu" cu_version: "cpu"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.5_cpu_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.5_cpu
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py3.5_cu92
name: binary_linux_wheel_py3.5_cuda10.0
python_version: "3.5" python_version: "3.5"
cuda_version: "10.0" cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.5_cu92_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.5_cu92
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py3.5_cu100
name: binary_linux_wheel_py3.5_cuda9.2
python_version: "3.5" python_version: "3.5"
cuda_version: "9.2" cu_version: "cu100"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.5_cu100_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.5_cu100
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py3.6_cpu
name: binary_linux_wheel_py3.6_cudacpu
python_version: "3.6" python_version: "3.6"
cuda_version: "cpu" cu_version: "cpu"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.6_cpu_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.6_cpu
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py3.6_cu92
name: binary_linux_wheel_py3.6_cuda10.0
python_version: "3.6" python_version: "3.6"
cuda_version: "10.0" cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.6_cu92_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.6_cu92
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py3.6_cu100
name: binary_linux_wheel_py3.6_cuda9.2
python_version: "3.6" python_version: "3.6"
cuda_version: "9.2" cu_version: "cu100"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.6_cu100_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.6_cu100
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py3.7_cpu
name: binary_linux_wheel_py3.7_cudacpu
python_version: "3.7" python_version: "3.7"
cuda_version: "cpu" cu_version: "cpu"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.7_cpu_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.7_cpu
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py3.7_cu92
name: binary_linux_wheel_py3.7_cuda10.0
python_version: "3.7" python_version: "3.7"
cuda_version: "10.0" cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.7_cu92_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.7_cu92
- binary_linux_wheel: - binary_linux_wheel:
# TODO: cudacpu is ugly name: nightly_binary_linux_wheel_py3.7_cu100
name: binary_linux_wheel_py3.7_cuda9.2
python_version: "3.7" python_version: "3.7"
cuda_version: "9.2" cu_version: "cu100"
- binary_wheel_upload:
name: nightly_binary_linux_wheel_py3.7_cu100_upload
context: org-member
requires:
- nightly_binary_linux_wheel_py3.7_cu100
- binary_macos_wheel: - binary_macos_wheel:
# TODO: cudacpu is ugly name: nightly_binary_macos_wheel_py2.7_cpu
name: binary_macos_wheel_py2.7_cudacpu
python_version: "2.7" python_version: "2.7"
cuda_version: "cpu" cu_version: "cpu"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py2.7_cpu_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py2.7_cpu
- binary_macos_wheel: - binary_macos_wheel:
name: binary_macos_wheel_py2.7_cudacpu_unicode name: nightly_binary_macos_wheel_py2.7u_cpu
python_version: "2.7" python_version: "2.7"
cuda_version: "cpu" cu_version: "cpu"
unicode_abi: "1" unicode_abi: "1"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py2.7u_cpu_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py2.7u_cpu
- binary_macos_wheel: - binary_macos_wheel:
# TODO: cudacpu is ugly name: nightly_binary_macos_wheel_py3.5_cpu
name: binary_macos_wheel_py3.5_cudacpu
python_version: "3.5" python_version: "3.5"
cuda_version: "cpu" cu_version: "cpu"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py3.5_cpu_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py3.5_cpu
- binary_macos_wheel: - binary_macos_wheel:
# TODO: cudacpu is ugly name: nightly_binary_macos_wheel_py3.6_cpu
name: binary_macos_wheel_py3.6_cudacpu
python_version: "3.6" python_version: "3.6"
cuda_version: "cpu" cu_version: "cpu"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py3.6_cpu_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py3.6_cpu
- binary_macos_wheel: - binary_macos_wheel:
# TODO: cudacpu is ugly name: nightly_binary_macos_wheel_py3.7_cpu
name: binary_macos_wheel_py3.7_cudacpu
python_version: "3.7" python_version: "3.7"
cuda_version: "cpu" cu_version: "cpu"
- binary_wheel_upload:
name: nightly_binary_macos_wheel_py3.7_cpu_upload
context: org-member
requires:
- nightly_binary_macos_wheel_py3.7_cpu
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py2.7_cpu
name: binary_linux_conda_py2.7_cudacpu
python_version: "2.7" python_version: "2.7"
cuda_version: "cpu" cu_version: "cpu"
- binary_conda_upload:
name: nightly_binary_linux_conda_py2.7_cpu_upload
context: org-member
requires:
- nightly_binary_linux_conda_py2.7_cpu
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py2.7_cu92
name: binary_linux_conda_py2.7_cuda10.0
python_version: "2.7" python_version: "2.7"
cuda_version: "10.0" cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_conda_upload:
name: nightly_binary_linux_conda_py2.7_cu92_upload
context: org-member
requires:
- nightly_binary_linux_conda_py2.7_cu92
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py2.7_cu100
name: binary_linux_conda_py2.7_cuda9.2
python_version: "2.7" python_version: "2.7"
cuda_version: "9.2" cu_version: "cu100"
- binary_conda_upload:
name: nightly_binary_linux_conda_py2.7_cu100_upload
context: org-member
requires:
- nightly_binary_linux_conda_py2.7_cu100
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py3.5_cpu
name: binary_linux_conda_py3.5_cudacpu
python_version: "3.5" python_version: "3.5"
cuda_version: "cpu" cu_version: "cpu"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.5_cpu_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.5_cpu
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py3.5_cu92
name: binary_linux_conda_py3.5_cuda10.0
python_version: "3.5" python_version: "3.5"
cuda_version: "10.0" cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.5_cu92_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.5_cu92
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py3.5_cu100
name: binary_linux_conda_py3.5_cuda9.2
python_version: "3.5" python_version: "3.5"
cuda_version: "9.2" cu_version: "cu100"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.5_cu100_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.5_cu100
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py3.6_cpu
name: binary_linux_conda_py3.6_cudacpu
python_version: "3.6" python_version: "3.6"
cuda_version: "cpu" cu_version: "cpu"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.6_cpu_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.6_cpu
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py3.6_cu92
name: binary_linux_conda_py3.6_cuda10.0
python_version: "3.6" python_version: "3.6"
cuda_version: "10.0" cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.6_cu92_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.6_cu92
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py3.6_cu100
name: binary_linux_conda_py3.6_cuda9.2
python_version: "3.6" python_version: "3.6"
cuda_version: "9.2" cu_version: "cu100"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.6_cu100_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.6_cu100
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py3.7_cpu
name: binary_linux_conda_py3.7_cudacpu
python_version: "3.7" python_version: "3.7"
cuda_version: "cpu" cu_version: "cpu"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.7_cpu_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.7_cpu
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py3.7_cu92
name: binary_linux_conda_py3.7_cuda10.0
python_version: "3.7" python_version: "3.7"
cuda_version: "10.0" cu_version: "cu92"
wheel_docker_image: "soumith/manylinux-cuda92"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.7_cu92_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.7_cu92
- binary_linux_conda: - binary_linux_conda:
# TODO: cudacpu is ugly name: nightly_binary_linux_conda_py3.7_cu100
name: binary_linux_conda_py3.7_cuda9.2
python_version: "3.7" python_version: "3.7"
cuda_version: "9.2" cu_version: "cu100"
- binary_conda_upload:
name: nightly_binary_linux_conda_py3.7_cu100_upload
context: org-member
requires:
- nightly_binary_linux_conda_py3.7_cu100
- binary_macos_conda: - binary_macos_conda:
# TODO: cudacpu is ugly name: nightly_binary_macos_conda_py2.7_cpu
name: binary_macos_conda_py2.7_cudacpu
python_version: "2.7" python_version: "2.7"
cuda_version: "cpu" cu_version: "cpu"
- binary_conda_upload:
name: nightly_binary_macos_conda_py2.7_cpu_upload
context: org-member
requires:
- nightly_binary_macos_conda_py2.7_cpu
- binary_macos_conda: - binary_macos_conda:
# TODO: cudacpu is ugly name: nightly_binary_macos_conda_py3.5_cpu
name: binary_macos_conda_py3.5_cudacpu
python_version: "3.5" python_version: "3.5"
cuda_version: "cpu" cu_version: "cpu"
- binary_conda_upload:
name: nightly_binary_macos_conda_py3.5_cpu_upload
context: org-member
requires:
- nightly_binary_macos_conda_py3.5_cpu
- binary_macos_conda: - binary_macos_conda:
# TODO: cudacpu is ugly name: nightly_binary_macos_conda_py3.6_cpu
name: binary_macos_conda_py3.6_cudacpu
python_version: "3.6" python_version: "3.6"
cuda_version: "cpu" cu_version: "cpu"
- binary_conda_upload:
name: nightly_binary_macos_conda_py3.6_cpu_upload
context: org-member
requires:
- nightly_binary_macos_conda_py3.6_cpu
- binary_macos_conda: - binary_macos_conda:
# TODO: cudacpu is ugly name: nightly_binary_macos_conda_py3.7_cpu
name: binary_macos_conda_py3.7_cudacpu
python_version: "3.7" python_version: "3.7"
cuda_version: "cpu" cu_version: "cpu"
\ No newline at end of file - binary_conda_upload:
name: nightly_binary_macos_conda_py3.7_cpu_upload
context: org-member
requires:
- nightly_binary_macos_conda_py3.7_cpu
\ No newline at end of file
.circleci/config.yml.in
View file @ 155de2b4
...@@ -2,8 +2,9 @@ version: 2.1 ...@@ -2,8 +2,9 @@ version: 2.1
# How to test the Linux jobs: # How to test the Linux jobs:
# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel # - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
# - Replace binary_linux_wheel with the name of the job you want to test # - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
# Job names are 'name:' key.
binary_common: &binary_common binary_common: &binary_common
parameters: parameters:
...@@ -20,19 +21,23 @@ binary_common: &binary_common ...@@ -20,19 +21,23 @@ binary_common: &binary_common
python_version: python_version:
description: "Python version to build against (e.g., 3.7)" description: "Python version to build against (e.g., 3.7)"
type: string type: string
cuda_version: cu_version:
description: "CUDA version to build against (e.g., cpu or 10.0)" description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
type: string type: string
unicode_abi: unicode_abi:
description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
type: string type: string
default: "" default: ""
wheel_docker_image:
description: "Wheel only: what docker image to use"
type: string
default: "soumith/manylinux-cuda100"
environment: environment:
PYTHON_VERSION: << parameters.python_version >> PYTHON_VERSION: << parameters.python_version >>
BUILD_VERSION: << parameters.build_version >> BUILD_VERSION: << parameters.build_version >>
PYTORCH_VERSION: << parameters.pytorch_version >> PYTORCH_VERSION: << parameters.pytorch_version >>
UNICODE_ABI: << parameters.unicode_abi >> UNICODE_ABI: << parameters.unicode_abi >>
CUDA_VERSION: << parameters.cuda_version >> CU_VERSION: << parameters.cu_version >>
jobs: jobs:
circleci_consistency: circleci_consistency:
...@@ -49,13 +54,17 @@ jobs: ...@@ -49,13 +54,17 @@ jobs:
binary_linux_wheel: binary_linux_wheel:
<<: *binary_common <<: *binary_common
docker: docker:
- image: "soumith/manylinux-cuda100" - image: << parameters.wheel_docker_image >>
resource_class: 2xlarge+ resource_class: 2xlarge+
steps: steps:
- checkout - checkout
- run: packaging/build_wheel.sh - run: packaging/build_wheel.sh
- store_artifacts: - store_artifacts:
path: dist path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_linux_conda: binary_linux_conda:
<<: *binary_common <<: *binary_common
...@@ -67,6 +76,10 @@ jobs: ...@@ -67,6 +76,10 @@ jobs:
- run: packaging/build_conda.sh - run: packaging/build_conda.sh
- store_artifacts: - store_artifacts:
path: /opt/conda/conda-bld/linux-64 path: /opt/conda/conda-bld/linux-64
- persist_to_workspace:
root: /opt/conda/conda-bld/linux-64
paths:
- "*"
binary_macos_wheel: binary_macos_wheel:
<<: *binary_common <<: *binary_common
...@@ -85,6 +98,10 @@ jobs: ...@@ -85,6 +98,10 @@ jobs:
packaging/build_wheel.sh packaging/build_wheel.sh
- store_artifacts: - store_artifacts:
path: dist path: dist
- persist_to_workspace:
root: dist
paths:
- "*"
binary_macos_conda: binary_macos_conda:
<<: *binary_common <<: *binary_common
...@@ -101,30 +118,101 @@ jobs: ...@@ -101,30 +118,101 @@ jobs:
packaging/build_conda.sh packaging/build_conda.sh
- store_artifacts: - store_artifacts:
path: /Users/distiller/miniconda3/conda-bld/osx-64 path: /Users/distiller/miniconda3/conda-bld/osx-64
- persist_to_workspace:
root: /Users/distiller/miniconda3/conda-bld/osx-64
paths:
- "*"
workflows: # Requires org-member context
build: binary_conda_upload:
jobs: docker:
- circleci_consistency - image: continuumio/miniconda
{%- for btype in ["wheel", "conda"] -%} steps:
{%- for os in ["linux", "macos"] -%} - attach_workspace:
{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] -%} at: ~/workspace
{%- for cuda_version in ["cpu", "10.0", "9.2"] -%} - run:
{%- if os != "macos" or cuda_version == "cpu" %} command: |
- binary_{{os}}_{{btype}}: # Prevent credential from leaking
# TODO: cudacpu is ugly conda install -yq anaconda-client
name: binary_{{os}}_{{btype}}_py{{python_version}}_cuda{{cuda_version}} set +x
python_version: "{{python_version}}" anaconda login \
cuda_version: "{{cuda_version}}" --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
{%- if btype == "wheel" and python_version == "2.7" %} --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
set -x
anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force
# Requires org-member context
binary_wheel_upload:
docker:
- image: circleci/python:3.7
steps:
- attach_workspace:
at: ~/workspace
- checkout
- run:
command: |
pip install --user awscli
export PATH="$HOME/.local/bin:$PATH"
# Prevent credential from leaking
set +x
export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
#for pkg in ~/workspace/*.whl; do
# aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
#done
{%- macro workflow(btype, os, python_version, cu_version, unicode, prefix='', upload=False) %}
- binary_{{os}}_{{btype}}: - binary_{{os}}_{{btype}}:
name: binary_{{os}}_{{btype}}_py{{python_version}}_cuda{{cuda_version}}_unicode name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}
python_version: "{{python_version}}" python_version: "{{python_version}}"
cuda_version: "{{cuda_version}}" cu_version: "{{cu_version}}"
{%- if unicode %}
unicode_abi: "1" unicode_abi: "1"
{%- endif -%} {%- endif %}
{%- endif -%} {%- if cu_version == "cu92" %}
wheel_docker_image: "soumith/manylinux-cuda92"
{%- endif %}
{%- if upload %}
- binary_{{btype}}_upload:
name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}_upload
context: org-member
requires:
- {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}
{%- endif %}
{%- endmacro %}
{%- macro workflows(prefix='', upload=False) %}
{%- for btype in ["wheel", "conda"] -%}
{%- for os in ["linux", "macos"] -%}
{%- for python_version in ["2.7", "3.5", "3.6", "3.7"] -%}
{%- for cu_version in (["cpu", "cu92", "cu100"] if os == "linux" else ["cpu"]) -%}
{%- for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]) -%}
{{ workflow(btype, os, python_version, cu_version, unicode, prefix=prefix, upload=upload) }}
{%- endfor -%} {%- endfor -%}
{%- endfor -%} {%- endfor -%}
{%- endfor -%} {%- endfor -%}
{%- endfor -%} {%- endfor -%}
{%- endfor %}
{%- endmacro %}
workflows:
build:
{%- if True %}
jobs:
- circleci_consistency
{{ workflows() }}
nightly:
triggers:
- schedule:
cron: "0 9 * * *"
filters:
branches:
only:
- master
{%- endif %}
jobs:
- circleci_consistency
{{ workflows(prefix="nightly_", upload=True) }}
packaging/build_conda.sh
View file @ 155de2b4
...@@ -4,9 +4,7 @@ set -ex ...@@ -4,9 +4,7 @@ set -ex
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
. "$script_dir/pkg_helpers.bash" . "$script_dir/pkg_helpers.bash"
setup_build_version 0.4.0 setup_env 0.4.0
setup_cuda_suffix
setup_macos
export SOURCE_ROOT_DIR="$PWD" export SOURCE_ROOT_DIR="$PWD"
setup_conda_pytorch_constraint setup_conda_pytorch_constraint
setup_conda_cudatoolkit_constraint setup_conda_cudatoolkit_constraint
......
packaging/build_wheel.sh
View file @ 155de2b4
...@@ -4,10 +4,8 @@ set -ex ...@@ -4,10 +4,8 @@ set -ex
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
. "$script_dir/pkg_helpers.bash" . "$script_dir/pkg_helpers.bash"
setup_python setup_env 0.4.0
setup_cuda_suffix setup_wheel_python
setup_build_version 0.4.0
setup_macos
pip_install numpy pyyaml future ninja pip_install numpy pyyaml future ninja
setup_pip_pytorch_version setup_pip_pytorch_version
python setup.py clean python setup.py clean
......
packaging/pkg_helpers.bash
View file @ 155de2b4
# A set of useful bash functions for common functionality we need to do in # A set of useful bash functions for common functionality we need to do in
# many build scripts # many build scripts
# Respecting PYTHON_VERSION and UNICODE_ABI, add (or install) the correct
# version of Python to perform a build. Relevant to wheel builds. # Setup CUDA environment variables, based on CU_VERSION
setup_python() { #
if [[ "$(uname)" == Darwin ]]; then # Inputs:
eval "$(conda shell.bash hook)" # CU_VERSION (cpu, cu92, cu100)
conda env remove -n "env$PYTHON_VERSION" || true # NO_CUDA_PACKAGE (bool)
conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" #
conda activate "env$PYTHON_VERSION" # Outputs:
else # VERSION_SUFFIX (e.g., "")
case "$PYTHON_VERSION" in # PYTORCH_VERSION_SUFFIX (e.g., +cpu)
2.7) # WHEEL_DIR (e.g., cu100/)
if [[ -n "$UNICODE_ABI" ]]; then # CUDA_HOME (e.g., /usr/local/cuda-9.2, respected by torch.utils.cpp_extension)
python_abi=cp27-cp27mu # FORCE_CUDA (respected by torchvision setup.py)
else # NVCC_FLAGS (respected by torchvision setup.py)
python_abi=cp27-cp27m #
fi # Precondition: CUDA versions are installed in their conventional locations in
;; # /usr/local/cuda-*
3.5) python_abi=cp35-cp35m ;; #
3.6) python_abi=cp36-cp36m ;; # NOTE: Why VERSION_SUFFIX versus PYTORCH_VERSION_SUFFIX? If you're building
3.7) python_abi=cp37-cp37m ;; # a package with CUDA on a platform we support CUDA on, VERSION_SUFFIX ==
*) # PYTORCH_VERSION_SUFFIX and everyone is happy. However, if you are building a
echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" # package with only CPU bits (e.g., torchaudio), then VERSION_SUFFIX is always
# empty, but PYTORCH_VERSION_SUFFIX is +cpu (because that's how you get a CPU
# version of a Python package. But that doesn't apply if you're on OS X,
# since the default CU_VERSION on OS X is cpu.
setup_cuda() {
if [[ "$(uname)" == Darwin ]] || [[ -n "$NO_CUDA_PACKAGE" ]]; then
if [[ "$CU_VERSION" != "cpu" ]]; then
echo "CU_VERSION on OS X / package with no CUDA must be cpu"
exit 1 exit 1
;;
esac
export PATH="/opt/python/$python_abi/bin:$PATH"
fi fi
}
# Fill CUDA_SUFFIX and CU_VERSION with CUDA_VERSION. CUDA_SUFFIX is
# left blank for the default CUDA version (that's a blank suffix)
setup_cuda_suffix() {
if [[ "$(uname)" == Darwin ]]; then if [[ "$(uname)" == Darwin ]]; then
if [[ "$CUDA_VERSION" != "cpu" ]]; then export PYTORCH_VERSION_SUFFIX=""
echo "CUDA_VERSION on OS X must be cpu" else
exit 1 export PYTORCH_VERSION_SUFFIX="+cpu"
fi fi
export CPU_SUFFIX="" export VERSION_SUFFIX=""
export CU_VERSION="cpu" # NB: When there is no CUDA package available, we put these
# packages in the top-level directory, so they are eligible
# for selection even if you are otherwise trying to install
# a cu100 stack. This differs from when there ARE CUDA packages
# available; then we don't want the cpu package; we want
# to give you as much goodies as possible.
export WHEEL_DIR=""
else else
case "$CUDA_VERSION" in case "$CU_VERSION" in
10.0) cu100)
export CUDA_SUFFIX="" export PYTORCH_VERSION_SUFFIX=""
export CU_VERSION="cu100" export CUDA_HOME=/usr/local/cuda-10.0/
export FORCE_CUDA=1
# Hard-coding gencode flags is temporary situation until
# https://github.com/pytorch/pytorch/pull/23408 lands
export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50"
;; ;;
9.2) cu92)
export CUDA_SUFFIX="+cu92" export CUDA_HOME=/usr/local/cuda-9.2/
export CU_VERSION="cu92" export PYTORCH_VERSION_SUFFIX="+cu92"
export FORCE_CUDA=1
export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50"
;; ;;
cpu) cpu)
export CUDA_SUFFIX="+cpu" export PYTORCH_VERSION_SUFFIX="+cpu"
export CU_VERSION="cpu"
;; ;;
*) *)
echo "Unrecognized CUDA_VERSION=$CUDA_VERSION" echo "Unrecognized CU_VERSION=$CU_VERSION"
esac esac
export CPU_SUFFIX="+cpu" export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX"
export WHEEL_DIR="$CU_VERSION/"
fi fi
} }
# If a package is cpu-only, we never provide a cuda suffix # Populate build version if necessary, and add version suffix
setup_cpuonly_cuda_suffix() { #
export CUDA_SUFFIX="" # Inputs:
export CPU_SUFFIX="" # BUILD_VERSION (e.g., 0.2.0 or empty)
} #
# Outputs:
# Fill BUILD_VERSION and BUILD_NUMBER if it doesn't exist already with a nightly string # BUILD_VERSION (e.g., 0.2.0.dev20190807+cpu)
# Usage: setup_build_version 0.2 #
# Fill BUILD_VERSION if it doesn't exist already with a nightly string
# Usage: setup_build_version 0.2.0
setup_build_version() { setup_build_version() {
if [[ -z "$BUILD_VERSION" ]]; then if [[ -z "$BUILD_VERSION" ]]; then
export BUILD_VERSION="$1.dev$(date "+%Y%m%d")" export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX"
else
export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX"
fi fi
} }
...@@ -82,11 +97,55 @@ setup_macos() { ...@@ -82,11 +97,55 @@ setup_macos() {
fi fi
} }
# Top-level entry point for things every package will need to do
#
# Usage: setup_env 0.2.0
setup_env() {
setup_cuda
setup_build_version "$1"
setup_macos
}
# Function to retry functions that sometimes timeout or have flaky failures # Function to retry functions that sometimes timeout or have flaky failures
retry () { retry () {
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
} }
# Inputs:
# PYTHON_VERSION (2.7, 3.5, 3.6, 3.7)
# UNICODE_ABI (bool)
#
# Outputs:
# PATH modified to put correct Python version in PATH
#
# Precondition: If Linux, you are in a soumith/manylinux-cuda* Docker image
setup_wheel_python() {
if [[ "$(uname)" == Darwin ]]; then
eval "$(conda shell.bash hook)"
conda env remove -n "env$PYTHON_VERSION" || true
conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
conda activate "env$PYTHON_VERSION"
else
case "$PYTHON_VERSION" in
2.7)
if [[ -n "$UNICODE_ABI" ]]; then
python_abi=cp27-cp27mu
else
python_abi=cp27-cp27m
fi
;;
3.5) python_abi=cp35-cp35m ;;
3.6) python_abi=cp36-cp36m ;;
3.7) python_abi=cp37-cp37m ;;
*)
echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION"
exit 1
;;
esac
export PATH="/opt/python/$python_abi/bin:$PATH"
fi
}
# Install with pip a bit more robustly than the default # Install with pip a bit more robustly than the default
pip_install() { pip_install() {
retry pip install --progress-bar off "$@" retry pip install --progress-bar off "$@"
...@@ -98,7 +157,7 @@ setup_pip_pytorch_version() { ...@@ -98,7 +157,7 @@ setup_pip_pytorch_version() {
if [[ -z "$PYTORCH_VERSION" ]]; then if [[ -z "$PYTORCH_VERSION" ]]; then
# Install latest prerelease version of torch, per our nightlies, consistent # Install latest prerelease version of torch, per our nightlies, consistent
# with the requested cuda version # with the requested cuda version
pip_install --pre torch -f "https://download.pytorch.org/whl/nightly/${CU_VERSION}/torch_nightly.html" pip_install --pre torch -f "https://download.pytorch.org/whl/nightly/${WHEEL_DIR}torch_nightly.html"
if [[ "$CUDA_VERSION" == "cpu" ]]; then if [[ "$CUDA_VERSION" == "cpu" ]]; then
# CUDA and CPU are ABI compatible on the CPU-only parts, so strip # CUDA and CPU are ABI compatible on the CPU-only parts, so strip
# in this case # in this case
...@@ -107,9 +166,9 @@ setup_pip_pytorch_version() { ...@@ -107,9 +166,9 @@ setup_pip_pytorch_version() {
export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//')" export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//')"
fi fi
else else
# TODO: Maybe add staging too
pip_install "torch==$PYTORCH_VERSION$CUDA_SUFFIX" \ pip_install "torch==$PYTORCH_VERSION$CUDA_SUFFIX" \
-f https://download.pytorch.org/whl/torch_stable.html -f https://download.pytorch.org/whl/torch_stable.html \
-f https://download.pytorch.org/whl/nightly/torch_nightly.html
fi fi
} }
...@@ -117,21 +176,19 @@ setup_pip_pytorch_version() { ...@@ -117,21 +176,19 @@ setup_pip_pytorch_version() {
# CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions # CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions
# #
# You MUST have populated CUDA_SUFFIX before hand. # You MUST have populated CUDA_SUFFIX before hand.
#
# TODO: This is currently hard-coded for CPU-only case
setup_conda_pytorch_constraint() { setup_conda_pytorch_constraint() {
if [[ -z "$PYTORCH_VERSION" ]]; then if [[ -z "$PYTORCH_VERSION" ]]; then
export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" export CONDA_CHANNEL_FLAGS="-c pytorch-nightly"
export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")"
else else
export CONDA_CHANNEL_FLAGS="-c pytorch" export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly"
fi fi
if [[ "$CUDA_VERSION" == cpu ]]; then if [[ "$CU_VERSION" == cpu ]]; then
export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${CPU_SUFFIX}" export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}"
export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION"
else else
export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}"
export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${CUDA_SUFFIX}" export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}"
fi fi
} }
...@@ -141,17 +198,21 @@ setup_conda_cudatoolkit_constraint() { ...@@ -141,17 +198,21 @@ setup_conda_cudatoolkit_constraint() {
if [[ "$(uname)" == Darwin ]]; then if [[ "$(uname)" == Darwin ]]; then
export CONDA_CUDATOOLKIT_CONSTRAINT="" export CONDA_CUDATOOLKIT_CONSTRAINT=""
else else
case "$CUDA_VERSION" in case "$CU_VERSION" in
10.0) cu100)
export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]" export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]"
;; ;;
9.2) cu92)
export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]" export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]"
;; ;;
cpu) cpu)
export CONDA_CUDATOOLKIT_CONSTRAINT="" export CONDA_CUDATOOLKIT_CONSTRAINT=""
export CONDA_CPUONLY_FEATURE="- cpuonly" export CONDA_CPUONLY_FEATURE="- cpuonly"
;; ;;
*)
echo "Unrecognized CU_VERSION=$CU_VERSION"
exit 1
;;
esac esac
fi fi
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment