Try building Windows nightlies on CircleCI (#2058)

* Try building Windows nightlies on CircleCI

* Update config.yml

* Merge jobs

* Use cmd

* Debug

* Fix PATH

* Enable build matrix

* Try integrate with regenerate.py

* Fix upload path for Windows wheels

* Bring fewer tests to CI

* Run other tests only on master

* Refactor code

* Use nightly branch

* Fix lint

.circleci/config.yml.in

@@ -64,6 +64,31 @@ binary_common: &binary_common
     UNICODE_ABI: << parameters.unicode_abi >>
     CU_VERSION: << parameters.cu_version >>
 
+binary_windows: &binary_windows
+  parameters:
+    # Edit these defaults to do a release`
+    build_version:
+      description: "version number of release binary; by default, build a nightly"
+      type: string
+      default: ""
+    pytorch_version:
+      description: "PyTorch version to build against; by default, use a nightly"
+      type: string
+      default: ""
+    # Don't edit these
+    python_version:
+      description: "Python version to build against (e.g., 3.7)"
+      type: string
+    cu_version:
+      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
+      type: string
+  environment:
+    DESIRED_PYTHON: << parameters.python_version >>
+    PYTORCH_VERSION: << parameters.pytorch_version >>
+    CUDA_VERSION: << parameters.cu_version >>
+    USE_SCCACHE: "1"
+    VC_YEAR: "2017"
+
 jobs:
   circleci_consistency:
     docker:
@@ -230,6 +255,46 @@ jobs:
             bash packaging/build_conda.sh
           shell: powershell.exe
 
+  binary_win_conda_release:
+    <<: *binary_windows
+    executor:
+      name: win/default
+      shell: cmd.exe
+    steps:
+      - checkout_merge
+      - run:
+          name: Build conda packages
+          command: |
+            call packaging/windows/internal/build_conda.bat
+      - store_artifacts:
+          path: packaging/windows/output
+      - persist_to_workspace:
+          root: packaging/windows/output
+          paths:
+            - "*"
+      - store_test_results:
+          path: build_results/
+
+  binary_win_wheel_release:
+    <<: *binary_windows
+    executor:
+      name: win/default
+      shell: cmd.exe
+    steps:
+      - checkout_merge
+      - run:
+          name: Build wheel packages
+          command: |
+            call packaging/windows/internal/build_wheels.bat
+      - store_artifacts:
+          path: packaging/windows/output
+      - persist_to_workspace:
+          root: packaging/windows/output
+          paths:
+            - "*"
+      - store_test_results:
+          path: build_results/
+
   binary_macos_wheel:
     <<: *binary_common
     macos:
@@ -319,7 +384,7 @@ workflows:
 {%- if True %}
     jobs:
       - circleci_consistency
-      {{ workflows() }}
+      {{ workflows(windows_latest_only=True) }}
       - binary_linux_conda_cuda:
           name: torchvision_linux_py3.8_cu102_cuda
           python_version: "3.8"

.circleci/regenerate.py

@@ -19,16 +19,23 @@ import yaml
 import os.path
 
 
-def workflows(prefix='', filter_branch=None, upload=False, indentation=6):
+def workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
     w = []
     for btype in ["wheel", "conda"]:
-        for os_type in ["linux", "macos"]:
-            for python_version in ["3.5", "3.6", "3.7", "3.8"]:
-                for cu_version in (["cpu", "cu92", "cu101", "cu102"] if os_type == "linux" else ["cpu"]):
+        for os_type in ["linux", "macos", "win"]:
+            python_versions = ["3.5", "3.6", "3.7", "3.8"]
+            cu_versions = (["cpu", "cu92", "cu101", "cu102"] if os_type == "linux" or os_type == "win" else ["cpu"])
+            for python_version in python_versions:
+                for cu_version in cu_versions:
                     for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]):
+                        fb = filter_branch
+                        if windows_latest_only and os_type == "win" and filter_branch is None and \
+                            (python_version != python_versions[-1] or
+                             (cu_version not in [cu_versions[0], cu_versions[-1]])):
+                            fb = "master"
                         w += workflow_pair(
                             btype, os_type, python_version, cu_version,
-                            unicode, prefix, upload, filter_branch=filter_branch)
+                            unicode, prefix, upload, filter_branch=fb)
 
     return indent(indentation, w)
 
@@ -69,18 +76,20 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version,
     d = {
         "name": base_workflow_name,
         "python_version": python_version,
-        "cu_version": cu_version,
+        "cu_version": cu_version.replace("cu", "") if os_type == "win" else cu_version,
     }
 
-    if unicode:
+    if os_type != "win" and unicode:
         d["unicode_abi"] = '1'
 
-    d["wheel_docker_image"] = get_manylinux_image(cu_version)
+    if os_type != "win":
+        d["wheel_docker_image"] = get_manylinux_image(cu_version)
 
     if filter_branch is not None:
         d["filters"] = {"branches": {"only": filter_branch}}
 
-    return {f"binary_{os_type}_{btype}": d}
+    w = f"binary_{os_type}_{btype}_release" if os_type == "win" else f"binary_{os_type}_{btype}"
+    return {w: d}
 
 
 def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):

packaging/conda/build_vision.sh

@@ -5,6 +5,10 @@ fi
 
 set -ex
 
+if [[ "$CIRCLECI" == 'true' ]]; then
+    export PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:.:$PATH"
+fi
+
 # Function to retry functions that sometimes timeout or have flaky failures
 retry () {
     $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)

packaging/windows/internal/build_conda.bat

+if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.11
+if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs_install.ps1
+if errorlevel 1 exit /b 1
+
+call packaging/windows/internal/cuda_install.bat
+if errorlevel 1 exit /b 1
+
+call packaging/windows/internal/nightly_defaults.bat Conda
+if errorlevel 1 exit /b 1
+
+set PYTORCH_FINAL_PACKAGE_DIR=%CD%\packaging\windows\output
+if not exist "%PYTORCH_FINAL_PACKAGE_DIR%" mkdir %PYTORCH_FINAL_PACKAGE_DIR%
+
+bash ./packaging/conda/build_vision.sh %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%
+if errorlevel 1 exit /b 1

packaging/windows/internal/build_wheels.bat

+if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.11
+if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs_install.ps1
+if errorlevel 1 exit /b 1
+
+call packaging/windows/internal/cuda_install.bat
+if errorlevel 1 exit /b 1
+
+call packaging/windows/internal/nightly_defaults.bat Conda
+if errorlevel 1 exit /b 1
+
+call packaging/windows/build_vision.bat %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%
+if errorlevel 1 exit /b 1

packaging/windows/internal/nightly_defaults.bat

@@ -102,7 +102,7 @@ if "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch
 ::   my_branch_name) or can be a git commit (git checkout 4b2674n...). Default
 ::   is 'latest', which is a special term that signals to pull the last commit
 ::   before 0:00 midnight on the NIGHTLIES_DATE
-if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=latest
+if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=nightly
 
 :: Clone the requested pytorch checkout
 if exist "%NIGHTLIES_PYTORCH_ROOT%" ( goto clone_end ) else ( goto clone_start )

packaging/windows/internal/setup.bat

@@ -30,7 +30,7 @@ if "%CXX%"=="sccache cl" (
 
 :pytorch
 :: This stores in e.g. D:/_work/1/s/windows/output/cpu
-pip wheel -e . --no-deps --wheel-dir ../output/%CUDA_PREFIX%
+pip wheel -e . --no-deps --wheel-dir ../output
 
 :build_end
 IF ERRORLEVEL 1 exit /b 1

packaging/windows/internal/test.bat

@@ -11,7 +11,7 @@ if "%BUILD_VISION%" == "" (
     pip install future pytest "pillow>=4.1.1" mock
 )
 
-for /F "delims=" %%i in ('where /R %SRC_DIR%\output\%CUDA_PREFIX% *%MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i"
+for /F "delims=" %%i in ('where /R %SRC_DIR%\output *%MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i"
 
 if ERRORLEVEL 1 exit /b 1
 

packaging/windows/internal/vs_install.ps1

+$VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe"
+$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
+                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.14.11",
+                                                     "--add Microsoft.Component.MSBuild",
+                                                     "--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
+                                                     "--add Microsoft.VisualStudio.Component.TextTemplating",
+                                                     "--add Microsoft.VisualStudio.Component.VC.CoreIde",
+                                                     "--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
+                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
+                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81")
+
+curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
+if ($LASTEXITCODE -ne 0) {
+    echo "Download of the VS 2017 installer failed"
+    exit 1
+}
+
+$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
+Remove-Item -Path vs_installer.exe -Force
+$exitCode = $process.ExitCode
+if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
+    echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]."
+    exit 1
+}

packaging/windows/templates/upload_to_s3.yml

@@ -8,7 +8,7 @@ steps:
   inputs:
     awsCredentials: 'Pytorch S3 bucket'
     bucketName: 'pytorch'
-    sourceFolder: 'packaging/windows/output/${{ parameters.cudaVer }}'
+    sourceFolder: 'packaging/windows/output'
     globExpressions: '*.whl'
     targetFolder: 'whl/nightly/${{ parameters.cuVer }}/'
     filesAcl: 'public-read'