Windows build scripts (#1241)

* Windows build scripts

Add testing yaml

* Change urls

* Test MS agent

* Switch base image

* Use fb links

packaging/windows/internal/test.bat

@@ -8,15 +8,19 @@ set PYTHON_VERSION=%PYTHON_PREFIX:py=cp%
 if "%BUILD_VISION%" == "" (
     pip install future pytest coverage hypothesis protobuf
 ) ELSE (
-    pip install future pytest "pillow>=4.1.1"
+    pip install future pytest "pillow>=4.1.1" mock
 )
 
-
-for /F "delims=" %%i in ('where /R %SRC_DIR%\output\%CUDA_PREFIX% %MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i"
+for /F "delims=" %%i in ('where /R %SRC_DIR%\output\%CUDA_PREFIX% *%MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i"
 
 if ERRORLEVEL 1 exit /b 1
 
-if NOT "%BUILD_VISION%" == "" goto smoke_test_end
+if NOT "%BUILD_VISION%" == "" (
+    echo Smoke testing imports
+    python -c "import torchvision"
+    if ERRORLEVEL 1 exit /b 1
+    goto smoke_test_end
+)
 
 echo Smoke testing imports
 python -c "import torch"

packaging/windows/internal/upload.bat

+@echo off
+
+IF "%CONDA_UPLOADER_INSTALLATION%" == "" goto precheck_fail
+IF "%PYTORCH_FINAL_PACKAGE_DIR%" == "" goto precheck_fail
+IF "%today%" == "" goto precheck_fail
+IF "%PYTORCH_ANACONDA_USERNAME%" == "" goto precheck_fail
+IF "%PYTORCH_ANACONDA_PASSWORD%" == "" goto precheck_fail
+
+goto precheck_pass
+
+:precheck_fail
+
+echo Please run nightly_defaults.bat first.
+echo And remember to set `PYTORCH_FINAL_PACKAGE_DIR`
+echo Finally, don't forget to set anaconda tokens
+exit /b 1
+
+:precheck_pass
+
+pushd %today%
+
+:: Install anaconda client
+set "CONDA_HOME=%CONDA_UPLOADER_INSTALLATION%"
+set "tmp_conda=%CONDA_HOME%"
+set "miniconda_exe=%CD%\miniconda.exe"
+rmdir /s /q "%CONDA_HOME%"
+del miniconda.exe
+curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%"
+popd
+
+IF ERRORLEVEL 1 (
+    echo Conda download failed
+    exit /b 1
+)
+
+call %~dp0\..\..\conda\install_conda.bat
+
+IF ERRORLEVEL 1 (
+    echo Conda installation failed
+    exit /b 1
+)
+
+set "ORIG_PATH=%PATH%"
+set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%"
+
+REM conda install -y anaconda-client
+pip install git+https://github.com/peterjc123/anaconda-client.git@log_more_meaningfull_errors
+IF ERRORLEVEL 1 (
+    echo Anaconda client installation failed
+    exit /b 1
+)
+
+REM bash -c "yes | anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%""
+anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%"
+IF ERRORLEVEL 1 (
+    echo Anaconda client login failed
+    exit /b 1
+)
+
+set PYTORCH_FINAL_PACKAGE=
+:: Upload all the packages under `PYTORCH_FINAL_PACKAGE_DIR`
+FOR /F "delims=" %%i IN ('where /R %PYTORCH_FINAL_PACKAGE_DIR% *vision*.tar.bz2') DO (
+    set "PYTORCH_FINAL_PACKAGE=%%i"
+)
+
+IF "%PYTORCH_FINAL_PACKAGE%" == "" (
+    echo No package to upload
+    exit /b 0
+)
+
+:upload
+
+if "%RETRY_TIMES%" == "" (
+    set /a RETRY_TIMES=10
+    set /a SLEEP_TIME=2
+) else (
+    set /a RETRY_TIMES=%RETRY_TIMES%-1
+    set /a SLEEP_TIME=%SLEEP_TIME%*2
+)
+
+echo Uploading %PYTORCH_FINAL_PACKAGE% to Anaconda Cloud
+anaconda upload "%PYTORCH_FINAL_PACKAGE%" -u pytorch-nightly --label main --force --no-progress
+
+IF ERRORLEVEL 1 (
+    echo Anaconda upload retry times remaining: %RETRY_TIMES%
+    echo Sleep time: %SLEEP_TIME% seconds
+    IF %RETRY_TIMES% EQU 0 (
+        echo Upload failed
+        exit /b 1
+    )
+    waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul
+    goto upload
+) ELSE (
+    set RETRY_TIMES=
+    set SLEEP_TIME=
+)

packaging/windows/internal/vs_install.bat

 @echo off
 
 set VS_DOWNLOAD_LINK=https://aka.ms/vs/15/release/vs_buildtools.exe
-IF "%VS_LATEST%" == "1" (
-   set VS_INSTALL_ARGS= --nocache --norestart --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools
-   set VSDEVCMD_ARGS=
-) ELSE (
-   set VS_INSTALL_ARGS=--nocache --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools ^
-                                                --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^
-                                                --add Microsoft.Component.MSBuild ^
-                                                --add Microsoft.VisualStudio.Component.Roslyn.Compiler ^
-                                                --add Microsoft.VisualStudio.Component.TextTemplating ^
-                                                --add Microsoft.VisualStudio.Component.VC.CoreIde ^
-                                                --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest ^
-                                                --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core ^
-                                                --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^
-                                                --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^
-                                                --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81
-   set VSDEVCMD_ARGS=-vcvars_ver=14.11
-)
+REM IF "%VS_LATEST%" == "1" (
+REM    set VS_INSTALL_ARGS= --nocache --norestart --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools
+REM    set VSDEVCMD_ARGS=
+REM ) ELSE (
+set VS_INSTALL_ARGS=--nocache --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools ^
+                                             --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^
+                                             --add Microsoft.Component.MSBuild ^
+                                             --add Microsoft.VisualStudio.Component.Roslyn.Compiler ^
+                                             --add Microsoft.VisualStudio.Component.TextTemplating ^
+                                             --add Microsoft.VisualStudio.Component.VC.CoreIde ^
+                                             --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest ^
+                                             --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core ^
+                                             --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^
+                                             --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^
+                                             --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81
+set VSDEVCMD_ARGS=-vcvars_ver=14.11
+REM )
 
 curl -k -L %VS_DOWNLOAD_LINK% --output vs_installer.exe
 if errorlevel 1 exit /b 1

packaging/windows/templates/auth_task.yml

+jobs:
+- job: 'VSTS_Auth_Task'
+  timeoutInMinutes: 5
+  cancelTimeoutInMinutes: 5
+  variables:
+  - group: 'peterjc-vsts-token'
+
+  pool:
+    vmImage: 'win1803'
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - template: vsts_auth.yml
+    parameters:
+      auth: $(vsts_auth)

packaging/windows/templates/build_conda.yml

+parameters:
+  msagent: false
+
+steps:
+- bash: 'find . -name "*.sh" -exec dos2unix {} +'
+  displayName: Replace file endings
+
+- script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%'
+  displayName: 'Create final package directory'
+
+- bash: './packaging/conda/build_vision.sh $CUDA_VERSION $TORCHVISION_BUILD_VERSION $TORCHVISION_BUILD_NUMBER'
+  displayName: Build
+  env:
+    ${{ if eq(parameters.msagent, 'true') }}:
+      MAX_JOBS: 2

packaging/windows/templates/build_task.yml

+parameters:
+  package: ''
+  spec: ''
+  jobDesc: ''
+  packageDesc: ''
+  msagent: true
+  cpuEnabled: true
+  cudaEnabled: true
+  condaEnabled: true
+  wheelsEnabled: true
+  override: false
+
+jobs:
+- job: 'Windows_${{ parameters.spec }}_${{ parameters.package }}_Build'
+  timeoutInMinutes: 60
+  cancelTimeoutInMinutes: 5
+  condition: > 
+    or(and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CPU'),
+           eq('${{ parameters.condaEnabled }}', 'true'), eq('${{ parameters.cpuEnabled }}', 'true')),
+       and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CPU'),
+           eq('${{ parameters.wheelsEnabled }}', 'true'), eq('${{ parameters.cpuEnabled }}', 'true')),
+       and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CUDA'),
+           eq('${{ parameters.condaEnabled }}', 'true'), eq('${{ parameters.cudaEnabled }}', 'true')),
+       and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CUDA'),
+           eq('${{ parameters.wheelsEnabled }}', 'true'), eq('${{ parameters.cudaEnabled }}', 'true')))
+  variables:
+    - ${{ if eq(parameters.override, 'true') }}:
+      - name: TORCHVISION_BUILD_NUMBER
+        value: 1
+      - name: PYTORCH_REPO
+        value: 'pytorch'
+      - name: PYTORCH_BRANCH
+        value: 'v0.4.0'
+    - ${{ if eq(parameters.msagent, 'true') }}:
+      - name: USE_SCCACHE
+        value: 0
+    - ${{ if eq(parameters.msagent, 'false') }}:
+      - name: USE_SCCACHE
+        value: 1
+    - ${{ if eq(parameters.package, 'Conda') }}:
+      - group: peterjc_anaconda_token
+      - name: PYTORCH_FINAL_PACKAGE_DIR
+        value: '$(Build.Repository.LocalPath)\packaging\windows\output'
+      
+  strategy:
+    maxParallel: 10
+    matrix:
+      ${{ if eq(parameters.spec, 'CPU') }}:
+        PY3.5:
+          DESIRED_PYTHON: 3.5
+          CUDA_VERSION: cpu
+        PY3.6:
+          DESIRED_PYTHON: 3.6
+          CUDA_VERSION: cpu
+        PY3.7:
+          DESIRED_PYTHON: 3.7
+          CUDA_VERSION: cpu
+      ${{ if ne(parameters.spec, 'CPU') }}:
+        # PY3.5_92:
+        #   DESIRED_PYTHON: 3.5
+        #   CUDA_VERSION: 92
+        # PY3.6_92:
+        #   DESIRED_PYTHON: 3.6
+        #   CUDA_VERSION: 92
+        # PY3.7_92:
+        #   DESIRED_PYTHON: 3.7
+        #   CUDA_VERSION: 92
+        PY3.5_100:
+          DESIRED_PYTHON: 3.5
+          CUDA_VERSION: 100
+        PY3.6_100:
+          DESIRED_PYTHON: 3.6
+          CUDA_VERSION: 100
+        PY3.7_100:
+          DESIRED_PYTHON: 3.7
+          CUDA_VERSION: 100
+
+  pool:
+    ${{ if eq(parameters.msagent, 'true') }}:
+      vmImage: 'win1803'
+    ${{ if eq(parameters.msagent, 'false') }}:
+      name: 'release'
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - template: setup_env_for_msagent.yml
+    parameters:
+      msagent: ${{ parameters.msagent }}
+
+  # - ${{ if and(eq(parameters.override, 'true'),  eq(parameters.package, 'Wheels')) }}:
+  #   - template: override_pytorch_version.yml
+
+  - template: setup_nightly_variables.yml
+    parameters:
+      package: ${{ parameters.package }}
+
+  - ${{ if eq(parameters.package, 'Wheels') }}:
+    - template: build_wheels.yml
+      parameters:
+        msagent: ${{ parameters.msagent }}
+
+  - ${{ if eq(parameters.package, 'Conda') }}:
+    - template: build_conda.yml
+      parameters:
+        msagent: ${{ parameters.msagent }}
+
+  - ${{ if or(eq(parameters.package, 'Wheels'), eq(parameters.package, 'Conda')) }}:
+    - template: publish_test_results.yml
+      parameters:
+        msagent: ${{ parameters.msagent }}
+
+  # If you want to upload binaries to S3 & Anaconda Cloud, please uncomment this section.
+  - ${{ if and(eq(parameters.package, 'Wheels'), eq(parameters.spec, 'CPU')) }}:
+    - template: upload_to_s3.yml
+      parameters:
+        cuVer: '$(CUDA_VERSION)'
+        cudaVer: '$(CUDA_VERSION)'
+
+  - ${{ if and(eq(parameters.package, 'Wheels'), ne(parameters.spec, 'CPU')) }}:
+    - template: upload_to_s3.yml
+      parameters:
+        cuVer: 'cu$(CUDA_VERSION)'
+        cudaVer: 'cuda$(CUDA_VERSION)'
+
+  - ${{ if eq(parameters.package, 'Conda') }}:
+    - template: upload_to_conda.yml
+      parameters:
+        user: $(peterjc_conda_username)
+        pass: $(peterjc_conda_password)
+
+  # If you want to upload binaries to Azure Git, please uncomment this section.
+  # - ${{ if or(eq(parameters.package, 'Wheels'), eq(parameters.package, 'Conda')) }}:
+  #   - template: publish_test_results.yml
+  #     parameters:
+  #       msagent: ${{ parameters.msagent }}
+  #   - template: publish_packages.yml
+  #     parameters:
+  #       package: ${{ parameters.package }}

packaging/windows/templates/build_wheels.yml

+parameters:
+  msagent: false
+
+steps:
+- script: 'call packaging/windows/build_vision.bat %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%'
+  displayName: Build
+  env:
+    ${{ if eq(parameters.msagent, 'true') }}:
+      MAX_JOBS: 2

packaging/windows/templates/linux_build_task.yml

+parameters:
+  msagent: true
+  enabled: false
+
+jobs:
+- job: 'Linux_CPU_Conda_Build'
+  timeoutInMinutes: 0
+  cancelTimeoutInMinutes: 5
+  condition: ${{ eq(parameters.enabled, 'true') }}
+  variables:
+    CUDA_VERSION: cpu
+    TORCH_CONDA_BUILD_FOLDER: pytorch-nightly
+    PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)/output'
+
+  strategy:
+    maxParallel: 10
+    matrix:
+      PY3.5:
+        DESIRED_PYTHON: 3.5
+
+  pool:
+    vmImage: 'ubuntu-16.04'
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - script: 'sudo apt-get install p7zip-full'
+    displayName: 'Install 7Zip'
+
+  - task: CondaEnvironment@1
+    displayName: 'Install conda-build'
+    inputs:
+      packageSpecs: 'conda-build'
+
+  - template: build_conda.yml
+    parameters:
+      msagent: ${{ parameters.msagent }}

packaging/windows/templates/override_pytorch_version.yml

+steps:
+- script: 'windows/internal/override_pytorch_version.bat'
+  displayName: 'Override PyTorch Build Version for Wheels'
+
+- script: 'echo $(PYTORCH_BUILD_VERSION)'
+  displayName: 'Show PyTorch Build Version'

packaging/windows/templates/publish_packages.yml

+parameters:
+  package: ''
+
+steps:
+- script: 'packaging/windows/internal/publish.bat'
+  displayName: 'Upload packages to Azure DevOps Repo'
+  env:
+    PACKAGEFULLNAME: ${{ parameters.package }}

packaging/windows/templates/publish_test_results.yml

+steps:
+- task: PublishTestResults@2 # No test results to publish
+  inputs:
+    testResultsFiles: 'windows/pytorch/test/**/*.xml'
+    testRunTitle: 'Publish test results'
+  enabled: false

packaging/windows/templates/setup_env_for_msagent.yml

+parameters:
+   msagent: false
+
+steps:
+- ${{ if eq(parameters.msagent, 'true') }}:
+  - task: BatchScript@1
+    displayName: 'Install 7Zip & cURL'
+    inputs:
+      filename: 'packaging/windows/internal/dep_install.bat'
+
+      modifyEnvironment: true
+
+  - task: BatchScript@1
+    displayName: 'Install Visual Studio 2017'
+    inputs:
+      filename: 'packaging/windows/internal/vs_install.bat'
+
+      modifyEnvironment: true
+
+  - task: BatchScript@1
+    displayName: 'Install CUDA'
+    inputs:
+      filename: 'packaging/windows/internal/cuda_install.bat'
+
+      modifyEnvironment: true

packaging/windows/templates/setup_nightly_variables.yml

+parameters:
+  package: ''
+
+steps:
+- task: BatchScript@1
+  displayName: 'Setup nightly variables'
+  inputs:
+    filename: 'packaging/windows/internal/nightly_defaults.bat'
+    arguments: ${{ parameters.package }}
+
+    modifyEnvironment: true

packaging/windows/templates/upload_to_conda.yml

+parameters:
+  user: ''
+  pass: ''
+
+steps:
+- script: 'call packaging/windows/internal/upload.bat'
+  displayName: 'Upload packages to Anaconda Cloud'
+  env:
+    PYTORCH_ANACONDA_USERNAME: ${{ parameters.user }}
+    PYTORCH_ANACONDA_PASSWORD: ${{ parameters.pass }}

packaging/windows/templates/upload_to_s3.yml

+parameters:
+  cuVer: ''
+  cudaVer: ''
+
+steps:
+- task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1
+  displayName: 'Upload ${{ parameters.cuVer }} wheel to S3'
+  inputs:
+    awsCredentials: 'Pytorch S3 bucket'
+    bucketName: 'pytorch'
+    sourceFolder: 'packaging/windows/output/${{ parameters.cudaVer }}'
+    globExpressions: '*.whl'
+    targetFolder: 'whl/nightly/${{ parameters.cuVer }}/'
+    filesAcl: 'public-read'
+    flattenFolders: 'true'

packaging/windows/templates/vsts_auth.yml

+parameters:
+  auth: ''
+
+steps:
+- script: 'call packaging/windows/internal/auth.bat'
+  displayName: 'Sign in to Azure Pipelines'
+  env:
+    VSTS_AUTH: ${{ parameters.auth }}