Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "c808d163f6c65ca851db59e9966807a9220fc493"
Unverified Commit b7a0415e authored by peterjc123's avatar peterjc123 Committed by GitHub
Browse files

Try building Windows nightlies on CircleCI (#2058) 

* Try building Windows nightlies on CircleCI

* Update config.yml

* Merge jobs

* Use cmd

* Debug

* Fix PATH

* Enable build matrix

* Try integrate with regenerate.py

* Fix upload path for Windows wheels

* Bring fewer tests to CI

* Run other tests only on master

* Refactor code

* Use nightly branch

* Fix lint
parent 9c387588
Hide whitespace changes
Inline Side-by-side
Showing with 918 additions and 15 deletions
.circleci/config.yml
View file @ b7a0415e
...@@ -64,6 +64,31 @@ binary_common: &binary_common ...@@ -64,6 +64,31 @@ binary_common: &binary_common
UNICODE_ABI: << parameters.unicode_abi >> UNICODE_ABI: << parameters.unicode_abi >>
CU_VERSION: << parameters.cu_version >> CU_VERSION: << parameters.cu_version >>
binary_windows: &binary_windows
parameters:
# Edit these defaults to do a release`
build_version:
description: "version number of release binary; by default, build a nightly"
type: string
default: ""
pytorch_version:
description: "PyTorch version to build against; by default, use a nightly"
type: string
default: ""
# Don't edit these
python_version:
description: "Python version to build against (e.g., 3.7)"
type: string
cu_version:
description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
type: string
environment:
DESIRED_PYTHON: << parameters.python_version >>
PYTORCH_VERSION: << parameters.pytorch_version >>
CUDA_VERSION: << parameters.cu_version >>
USE_SCCACHE: "1"
VC_YEAR: "2017"
jobs: jobs:
circleci_consistency: circleci_consistency:
docker: docker:
...@@ -230,6 +255,46 @@ jobs: ...@@ -230,6 +255,46 @@ jobs:
bash packaging/build_conda.sh bash packaging/build_conda.sh
shell: powershell.exe shell: powershell.exe
binary_win_conda_release:
<<: *binary_windows
executor:
name: win/default
shell: cmd.exe
steps:
- checkout_merge
- run:
name: Build conda packages
command: |
call packaging/windows/internal/build_conda.bat
- store_artifacts:
path: packaging/windows/output
- persist_to_workspace:
root: packaging/windows/output
paths:
- "*"
- store_test_results:
path: build_results/
binary_win_wheel_release:
<<: *binary_windows
executor:
name: win/default
shell: cmd.exe
steps:
- checkout_merge
- run:
name: Build wheel packages
command: |
call packaging/windows/internal/build_wheels.bat
- store_artifacts:
path: packaging/windows/output
- persist_to_workspace:
root: packaging/windows/output
paths:
- "*"
- store_test_results:
path: build_results/
binary_macos_wheel: binary_macos_wheel:
<<: *binary_common <<: *binary_common
macos: macos:
...@@ -418,6 +483,112 @@ workflows: ...@@ -418,6 +483,112 @@ workflows:
name: binary_macos_wheel_py3.8_cpu name: binary_macos_wheel_py3.8_cpu
python_version: '3.8' python_version: '3.8'
wheel_docker_image: pytorch/manylinux-cuda102 wheel_docker_image: pytorch/manylinux-cuda102
- binary_win_wheel_release:
cu_version: cpu
filters:
branches:
only: master
name: binary_win_wheel_py3.5_cpu
python_version: '3.5'
- binary_win_wheel_release:
cu_version: '92'
filters:
branches:
only: master
name: binary_win_wheel_py3.5_cu92
python_version: '3.5'
- binary_win_wheel_release:
cu_version: '101'
filters:
branches:
only: master
name: binary_win_wheel_py3.5_cu101
python_version: '3.5'
- binary_win_wheel_release:
cu_version: '102'
filters:
branches:
only: master
name: binary_win_wheel_py3.5_cu102
python_version: '3.5'
- binary_win_wheel_release:
cu_version: cpu
filters:
branches:
only: master
name: binary_win_wheel_py3.6_cpu
python_version: '3.6'
- binary_win_wheel_release:
cu_version: '92'
filters:
branches:
only: master
name: binary_win_wheel_py3.6_cu92
python_version: '3.6'
- binary_win_wheel_release:
cu_version: '101'
filters:
branches:
only: master
name: binary_win_wheel_py3.6_cu101
python_version: '3.6'
- binary_win_wheel_release:
cu_version: '102'
filters:
branches:
only: master
name: binary_win_wheel_py3.6_cu102
python_version: '3.6'
- binary_win_wheel_release:
cu_version: cpu
filters:
branches:
only: master
name: binary_win_wheel_py3.7_cpu
python_version: '3.7'
- binary_win_wheel_release:
cu_version: '92'
filters:
branches:
only: master
name: binary_win_wheel_py3.7_cu92
python_version: '3.7'
- binary_win_wheel_release:
cu_version: '101'
filters:
branches:
only: master
name: binary_win_wheel_py3.7_cu101
python_version: '3.7'
- binary_win_wheel_release:
cu_version: '102'
filters:
branches:
only: master
name: binary_win_wheel_py3.7_cu102
python_version: '3.7'
- binary_win_wheel_release:
cu_version: cpu
name: binary_win_wheel_py3.8_cpu
python_version: '3.8'
- binary_win_wheel_release:
cu_version: '92'
filters:
branches:
only: master
name: binary_win_wheel_py3.8_cu92
python_version: '3.8'
- binary_win_wheel_release:
cu_version: '101'
filters:
branches:
only: master
name: binary_win_wheel_py3.8_cu101
python_version: '3.8'
- binary_win_wheel_release:
cu_version: '102'
name: binary_win_wheel_py3.8_cu102
python_version: '3.8'
- binary_linux_conda: - binary_linux_conda:
cu_version: cpu cu_version: cpu
name: binary_linux_conda_py3.5_cpu name: binary_linux_conda_py3.5_cpu
...@@ -518,6 +689,112 @@ workflows: ...@@ -518,6 +689,112 @@ workflows:
name: binary_macos_conda_py3.8_cpu name: binary_macos_conda_py3.8_cpu
python_version: '3.8' python_version: '3.8'
wheel_docker_image: pytorch/manylinux-cuda102 wheel_docker_image: pytorch/manylinux-cuda102
- binary_win_conda_release:
cu_version: cpu
filters:
branches:
only: master
name: binary_win_conda_py3.5_cpu
python_version: '3.5'
- binary_win_conda_release:
cu_version: '92'
filters:
branches:
only: master
name: binary_win_conda_py3.5_cu92
python_version: '3.5'
- binary_win_conda_release:
cu_version: '101'
filters:
branches:
only: master
name: binary_win_conda_py3.5_cu101
python_version: '3.5'
- binary_win_conda_release:
cu_version: '102'
filters:
branches:
only: master
name: binary_win_conda_py3.5_cu102
python_version: '3.5'
- binary_win_conda_release:
cu_version: cpu
filters:
branches:
only: master
name: binary_win_conda_py3.6_cpu
python_version: '3.6'
- binary_win_conda_release:
cu_version: '92'
filters:
branches:
only: master
name: binary_win_conda_py3.6_cu92
python_version: '3.6'
- binary_win_conda_release:
cu_version: '101'
filters:
branches:
only: master
name: binary_win_conda_py3.6_cu101
python_version: '3.6'
- binary_win_conda_release:
cu_version: '102'
filters:
branches:
only: master
name: binary_win_conda_py3.6_cu102
python_version: '3.6'
- binary_win_conda_release:
cu_version: cpu
filters:
branches:
only: master
name: binary_win_conda_py3.7_cpu
python_version: '3.7'
- binary_win_conda_release:
cu_version: '92'
filters:
branches:
only: master
name: binary_win_conda_py3.7_cu92
python_version: '3.7'
- binary_win_conda_release:
cu_version: '101'
filters:
branches:
only: master
name: binary_win_conda_py3.7_cu101
python_version: '3.7'
- binary_win_conda_release:
cu_version: '102'
filters:
branches:
only: master
name: binary_win_conda_py3.7_cu102
python_version: '3.7'
- binary_win_conda_release:
cu_version: cpu
name: binary_win_conda_py3.8_cpu
python_version: '3.8'
- binary_win_conda_release:
cu_version: '92'
filters:
branches:
only: master
name: binary_win_conda_py3.8_cu92
python_version: '3.8'
- binary_win_conda_release:
cu_version: '101'
filters:
branches:
only: master
name: binary_win_conda_py3.8_cu101
python_version: '3.8'
- binary_win_conda_release:
cu_version: '102'
name: binary_win_conda_py3.8_cu102
python_version: '3.8'
- binary_linux_conda_cuda: - binary_linux_conda_cuda:
name: torchvision_linux_py3.8_cu102_cuda name: torchvision_linux_py3.8_cu102_cuda
python_version: "3.8" python_version: "3.8"
...@@ -878,6 +1155,262 @@ workflows: ...@@ -878,6 +1155,262 @@ workflows:
requires: requires:
- nightly_binary_macos_wheel_py3.8_cpu - nightly_binary_macos_wheel_py3.8_cpu
subfolder: '' subfolder: ''
- binary_win_wheel_release:
cu_version: cpu
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.5_cpu
python_version: '3.5'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.5_cpu_upload
requires:
- nightly_binary_win_wheel_py3.5_cpu
subfolder: cpu/
- binary_win_wheel_release:
cu_version: '92'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.5_cu92
python_version: '3.5'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.5_cu92_upload
requires:
- nightly_binary_win_wheel_py3.5_cu92
subfolder: cu92/
- binary_win_wheel_release:
cu_version: '101'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.5_cu101
python_version: '3.5'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.5_cu101_upload
requires:
- nightly_binary_win_wheel_py3.5_cu101
subfolder: cu101/
- binary_win_wheel_release:
cu_version: '102'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.5_cu102
python_version: '3.5'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.5_cu102_upload
requires:
- nightly_binary_win_wheel_py3.5_cu102
subfolder: cu102/
- binary_win_wheel_release:
cu_version: cpu
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.6_cpu
python_version: '3.6'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.6_cpu_upload
requires:
- nightly_binary_win_wheel_py3.6_cpu
subfolder: cpu/
- binary_win_wheel_release:
cu_version: '92'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.6_cu92
python_version: '3.6'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.6_cu92_upload
requires:
- nightly_binary_win_wheel_py3.6_cu92
subfolder: cu92/
- binary_win_wheel_release:
cu_version: '101'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.6_cu101
python_version: '3.6'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.6_cu101_upload
requires:
- nightly_binary_win_wheel_py3.6_cu101
subfolder: cu101/
- binary_win_wheel_release:
cu_version: '102'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.6_cu102
python_version: '3.6'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.6_cu102_upload
requires:
- nightly_binary_win_wheel_py3.6_cu102
subfolder: cu102/
- binary_win_wheel_release:
cu_version: cpu
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.7_cpu
python_version: '3.7'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.7_cpu_upload
requires:
- nightly_binary_win_wheel_py3.7_cpu
subfolder: cpu/
- binary_win_wheel_release:
cu_version: '92'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.7_cu92
python_version: '3.7'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.7_cu92_upload
requires:
- nightly_binary_win_wheel_py3.7_cu92
subfolder: cu92/
- binary_win_wheel_release:
cu_version: '101'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.7_cu101
python_version: '3.7'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.7_cu101_upload
requires:
- nightly_binary_win_wheel_py3.7_cu101
subfolder: cu101/
- binary_win_wheel_release:
cu_version: '102'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.7_cu102
python_version: '3.7'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.7_cu102_upload
requires:
- nightly_binary_win_wheel_py3.7_cu102
subfolder: cu102/
- binary_win_wheel_release:
cu_version: cpu
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.8_cpu
python_version: '3.8'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.8_cpu_upload
requires:
- nightly_binary_win_wheel_py3.8_cpu
subfolder: cpu/
- binary_win_wheel_release:
cu_version: '92'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.8_cu92
python_version: '3.8'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.8_cu92_upload
requires:
- nightly_binary_win_wheel_py3.8_cu92
subfolder: cu92/
- binary_win_wheel_release:
cu_version: '101'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.8_cu101
python_version: '3.8'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.8_cu101_upload
requires:
- nightly_binary_win_wheel_py3.8_cu101
subfolder: cu101/
- binary_win_wheel_release:
cu_version: '102'
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.8_cu102
python_version: '3.8'
- binary_wheel_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_wheel_py3.8_cu102_upload
requires:
- nightly_binary_win_wheel_py3.8_cu102
subfolder: cu102/
- binary_linux_conda: - binary_linux_conda:
cu_version: cpu cu_version: cpu
filters: filters:
...@@ -1197,4 +1730,244 @@ workflows: ...@@ -1197,4 +1730,244 @@ workflows:
only: nightly only: nightly
name: nightly_binary_macos_conda_py3.8_cpu_upload name: nightly_binary_macos_conda_py3.8_cpu_upload
requires: requires:
- nightly_binary_macos_conda_py3.8_cpu - nightly_binary_macos_conda_py3.8_cpu
\ No newline at end of file - binary_win_conda_release:
cu_version: cpu
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.5_cpu
python_version: '3.5'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.5_cpu_upload
requires:
- nightly_binary_win_conda_py3.5_cpu
- binary_win_conda_release:
cu_version: '92'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.5_cu92
python_version: '3.5'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.5_cu92_upload
requires:
- nightly_binary_win_conda_py3.5_cu92
- binary_win_conda_release:
cu_version: '101'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.5_cu101
python_version: '3.5'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.5_cu101_upload
requires:
- nightly_binary_win_conda_py3.5_cu101
- binary_win_conda_release:
cu_version: '102'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.5_cu102
python_version: '3.5'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.5_cu102_upload
requires:
- nightly_binary_win_conda_py3.5_cu102
- binary_win_conda_release:
cu_version: cpu
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.6_cpu
python_version: '3.6'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.6_cpu_upload
requires:
- nightly_binary_win_conda_py3.6_cpu
- binary_win_conda_release:
cu_version: '92'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.6_cu92
python_version: '3.6'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.6_cu92_upload
requires:
- nightly_binary_win_conda_py3.6_cu92
- binary_win_conda_release:
cu_version: '101'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.6_cu101
python_version: '3.6'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.6_cu101_upload
requires:
- nightly_binary_win_conda_py3.6_cu101
- binary_win_conda_release:
cu_version: '102'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.6_cu102
python_version: '3.6'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.6_cu102_upload
requires:
- nightly_binary_win_conda_py3.6_cu102
- binary_win_conda_release:
cu_version: cpu
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.7_cpu
python_version: '3.7'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.7_cpu_upload
requires:
- nightly_binary_win_conda_py3.7_cpu
- binary_win_conda_release:
cu_version: '92'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.7_cu92
python_version: '3.7'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.7_cu92_upload
requires:
- nightly_binary_win_conda_py3.7_cu92
- binary_win_conda_release:
cu_version: '101'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.7_cu101
python_version: '3.7'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.7_cu101_upload
requires:
- nightly_binary_win_conda_py3.7_cu101
- binary_win_conda_release:
cu_version: '102'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.7_cu102
python_version: '3.7'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.7_cu102_upload
requires:
- nightly_binary_win_conda_py3.7_cu102
- binary_win_conda_release:
cu_version: cpu
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.8_cpu
python_version: '3.8'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.8_cpu_upload
requires:
- nightly_binary_win_conda_py3.8_cpu
- binary_win_conda_release:
cu_version: '92'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.8_cu92
python_version: '3.8'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.8_cu92_upload
requires:
- nightly_binary_win_conda_py3.8_cu92
- binary_win_conda_release:
cu_version: '101'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.8_cu101
python_version: '3.8'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.8_cu101_upload
requires:
- nightly_binary_win_conda_py3.8_cu101
- binary_win_conda_release:
cu_version: '102'
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.8_cu102
python_version: '3.8'
- binary_conda_upload:
context: org-member
filters:
branches:
only: nightly
name: nightly_binary_win_conda_py3.8_cu102_upload
requires:
- nightly_binary_win_conda_py3.8_cu102
\ No newline at end of file
.circleci/config.yml.in
View file @ b7a0415e
...@@ -64,6 +64,31 @@ binary_common: &binary_common ...@@ -64,6 +64,31 @@ binary_common: &binary_common
UNICODE_ABI: << parameters.unicode_abi >> UNICODE_ABI: << parameters.unicode_abi >>
CU_VERSION: << parameters.cu_version >> CU_VERSION: << parameters.cu_version >>
binary_windows: &binary_windows
parameters:
# Edit these defaults to do a release`
build_version:
description: "version number of release binary; by default, build a nightly"
type: string
default: ""
pytorch_version:
description: "PyTorch version to build against; by default, use a nightly"
type: string
default: ""
# Don't edit these
python_version:
description: "Python version to build against (e.g., 3.7)"
type: string
cu_version:
description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
type: string
environment:
DESIRED_PYTHON: << parameters.python_version >>
PYTORCH_VERSION: << parameters.pytorch_version >>
CUDA_VERSION: << parameters.cu_version >>
USE_SCCACHE: "1"
VC_YEAR: "2017"
jobs: jobs:
circleci_consistency: circleci_consistency:
docker: docker:
...@@ -230,6 +255,46 @@ jobs: ...@@ -230,6 +255,46 @@ jobs:
bash packaging/build_conda.sh bash packaging/build_conda.sh
shell: powershell.exe shell: powershell.exe
binary_win_conda_release:
<<: *binary_windows
executor:
name: win/default
shell: cmd.exe
steps:
- checkout_merge
- run:
name: Build conda packages
command: |
call packaging/windows/internal/build_conda.bat
- store_artifacts:
path: packaging/windows/output
- persist_to_workspace:
root: packaging/windows/output
paths:
- "*"
- store_test_results:
path: build_results/
binary_win_wheel_release:
<<: *binary_windows
executor:
name: win/default
shell: cmd.exe
steps:
- checkout_merge
- run:
name: Build wheel packages
command: |
call packaging/windows/internal/build_wheels.bat
- store_artifacts:
path: packaging/windows/output
- persist_to_workspace:
root: packaging/windows/output
paths:
- "*"
- store_test_results:
path: build_results/
binary_macos_wheel: binary_macos_wheel:
<<: *binary_common <<: *binary_common
macos: macos:
...@@ -319,7 +384,7 @@ workflows: ...@@ -319,7 +384,7 @@ workflows:
{%- if True %} {%- if True %}
jobs: jobs:
- circleci_consistency - circleci_consistency
{{ workflows() }} {{ workflows(windows_latest_only=True) }}
- binary_linux_conda_cuda: - binary_linux_conda_cuda:
name: torchvision_linux_py3.8_cu102_cuda name: torchvision_linux_py3.8_cu102_cuda
python_version: "3.8" python_version: "3.8"
......
.circleci/regenerate.py
View file @ b7a0415e
...@@ -19,16 +19,23 @@ import yaml ...@@ -19,16 +19,23 @@ import yaml
import os.path import os.path
def workflows(prefix='', filter_branch=None, upload=False, indentation=6): def workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
w = [] w = []
for btype in ["wheel", "conda"]: for btype in ["wheel", "conda"]:
for os_type in ["linux", "macos"]: for os_type in ["linux", "macos", "win"]:
for python_version in ["3.5", "3.6", "3.7", "3.8"]: python_versions = ["3.5", "3.6", "3.7", "3.8"]
for cu_version in (["cpu", "cu92", "cu101", "cu102"] if os_type == "linux" else ["cpu"]): cu_versions = (["cpu", "cu92", "cu101", "cu102"] if os_type == "linux" or os_type == "win" else ["cpu"])
for python_version in python_versions:
for cu_version in cu_versions:
for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]): for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]):
fb = filter_branch
if windows_latest_only and os_type == "win" and filter_branch is None and \
(python_version != python_versions[-1] or
(cu_version not in [cu_versions[0], cu_versions[-1]])):
fb = "master"
w += workflow_pair( w += workflow_pair(
btype, os_type, python_version, cu_version, btype, os_type, python_version, cu_version,
unicode, prefix, upload, filter_branch=filter_branch) unicode, prefix, upload, filter_branch=fb)
return indent(indentation, w) return indent(indentation, w)
...@@ -69,18 +76,20 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version, ...@@ -69,18 +76,20 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version,
d = { d = {
"name": base_workflow_name, "name": base_workflow_name,
"python_version": python_version, "python_version": python_version,
"cu_version": cu_version, "cu_version": cu_version.replace("cu", "") if os_type == "win" else cu_version,
} }
if unicode: if os_type != "win" and unicode:
d["unicode_abi"] = '1' d["unicode_abi"] = '1'
d["wheel_docker_image"] = get_manylinux_image(cu_version) if os_type != "win":
d["wheel_docker_image"] = get_manylinux_image(cu_version)
if filter_branch is not None: if filter_branch is not None:
d["filters"] = {"branches": {"only": filter_branch}} d["filters"] = {"branches": {"only": filter_branch}}
return {f"binary_{os_type}_{btype}": d} w = f"binary_{os_type}_{btype}_release" if os_type == "win" else f"binary_{os_type}_{btype}"
return {w: d}
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None): def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
......
packaging/conda/build_vision.sh
View file @ b7a0415e
...@@ -5,6 +5,10 @@ fi ...@@ -5,6 +5,10 @@ fi
set -ex set -ex
if [[ "$CIRCLECI" == 'true' ]]; then
export PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:.:$PATH"
fi
# Function to retry functions that sometimes timeout or have flaky failures # Function to retry functions that sometimes timeout or have flaky failures
retry () { retry () {
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
......
packaging/windows/internal/build_conda.bat 0 → 100644
View file @ b7a0415e
if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.11
if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs_install.ps1
if errorlevel 1 exit /b 1
call packaging/windows/internal/cuda_install.bat
if errorlevel 1 exit /b 1
call packaging/windows/internal/nightly_defaults.bat Conda
if errorlevel 1 exit /b 1
set PYTORCH_FINAL_PACKAGE_DIR=%CD%\packaging\windows\output
if not exist "%PYTORCH_FINAL_PACKAGE_DIR%" mkdir %PYTORCH_FINAL_PACKAGE_DIR%
bash ./packaging/conda/build_vision.sh %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%
if errorlevel 1 exit /b 1
packaging/windows/internal/build_wheels.bat 0 → 100644
View file @ b7a0415e
if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.11
if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs_install.ps1
if errorlevel 1 exit /b 1
call packaging/windows/internal/cuda_install.bat
if errorlevel 1 exit /b 1
call packaging/windows/internal/nightly_defaults.bat Conda
if errorlevel 1 exit /b 1
call packaging/windows/build_vision.bat %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%
if errorlevel 1 exit /b 1
packaging/windows/internal/nightly_defaults.bat
View file @ b7a0415e
...@@ -102,7 +102,7 @@ if "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch ...@@ -102,7 +102,7 @@ if "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch
:: my_branch_name) or can be a git commit (git checkout 4b2674n...). Default :: my_branch_name) or can be a git commit (git checkout 4b2674n...). Default
:: is 'latest', which is a special term that signals to pull the last commit :: is 'latest', which is a special term that signals to pull the last commit
:: before 0:00 midnight on the NIGHTLIES_DATE :: before 0:00 midnight on the NIGHTLIES_DATE
if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=latest if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=nightly
:: Clone the requested pytorch checkout :: Clone the requested pytorch checkout
if exist "%NIGHTLIES_PYTORCH_ROOT%" ( goto clone_end ) else ( goto clone_start ) if exist "%NIGHTLIES_PYTORCH_ROOT%" ( goto clone_end ) else ( goto clone_start )
......
packaging/windows/internal/setup.bat
View file @ b7a0415e
...@@ -30,7 +30,7 @@ if "%CXX%"=="sccache cl" ( ...@@ -30,7 +30,7 @@ if "%CXX%"=="sccache cl" (
:pytorch :pytorch
:: This stores in e.g. D:/_work/1/s/windows/output/cpu :: This stores in e.g. D:/_work/1/s/windows/output/cpu
pip wheel -e . --no-deps --wheel-dir ../output/%CUDA_PREFIX% pip wheel -e . --no-deps --wheel-dir ../output
:build_end :build_end
IF ERRORLEVEL 1 exit /b 1 IF ERRORLEVEL 1 exit /b 1
......
packaging/windows/internal/test.bat
View file @ b7a0415e
...@@ -11,7 +11,7 @@ if "%BUILD_VISION%" == "" ( ...@@ -11,7 +11,7 @@ if "%BUILD_VISION%" == "" (
pip install future pytest "pillow>=4.1.1" mock pip install future pytest "pillow>=4.1.1" mock
) )
for /F "delims=" %%i in ('where /R %SRC_DIR%\output\%CUDA_PREFIX% *%MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i" for /F "delims=" %%i in ('where /R %SRC_DIR%\output *%MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i"
if ERRORLEVEL 1 exit /b 1 if ERRORLEVEL 1 exit /b 1
......
packaging/windows/internal/vs_install.ps1 0 → 100644
View file @ b7a0415e
$VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe"
$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
"--add Microsoft.VisualStudio.Component.VC.Tools.14.11",
"--add Microsoft.Component.MSBuild",
"--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
"--add Microsoft.VisualStudio.Component.TextTemplating",
"--add Microsoft.VisualStudio.Component.VC.CoreIde",
"--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
"--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
"--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81")
curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
if ($LASTEXITCODE -ne 0) {
echo "Download of the VS 2017 installer failed"
exit 1
}
$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
Remove-Item -Path vs_installer.exe -Force
$exitCode = $process.ExitCode
if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]."
exit 1
}
packaging/windows/templates/upload_to_s3.yml
View file @ b7a0415e
...@@ -8,7 +8,7 @@ steps: ...@@ -8,7 +8,7 @@ steps:
inputs: inputs:
awsCredentials: 'Pytorch S3 bucket' awsCredentials: 'Pytorch S3 bucket'
bucketName: 'pytorch' bucketName: 'pytorch'
sourceFolder: 'packaging/windows/output/${{ parameters.cudaVer }}' sourceFolder: 'packaging/windows/output'
globExpressions: '*.whl' globExpressions: '*.whl'
targetFolder: 'whl/nightly/${{ parameters.cuVer }}/' targetFolder: 'whl/nightly/${{ parameters.cuVer }}/'
filesAcl: 'public-read' filesAcl: 'public-read'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment