Windows build scripts (#1241)

* Windows build scripts

Add testing yaml

* Change urls

* Test MS agent

* Switch base image

* Use fb links

packaging/conda/build_vision.sh

@@ -10,15 +10,56 @@ retry () {
     $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
 }
 
-if [ "$#" -ne 1 ]; then
-    echo "Illegal number of parameters. Pass cuda version"
-    echo "CUDA version should be M.m with no dot, e.g. '8.0' or 'cpu'"
-    exit 1
+# Parse arguments and determmine version
+###########################################################
+if [[ -n "$DESIRED_CUDA" && -n "$TORCHVISION_BUILD_VERSION" && -n "$TORCHVISION_BUILD_NUMBER" ]]; then
+    desired_cuda="$DESIRED_CUDA"
+    build_version="$PYTORCH_BUILD_VERSION"
+    build_number="$PYTORCH_BUILD_NUMBER"
+else
+    if [ "$#" -ne 3 ]; then
+        echo "Illegal number of parameters. Pass cuda version, pytorch version, build number"
+        echo "CUDA version should be Mm with no dot, e.g. '80'"
+        echo "DESIRED_PYTHON should be M.m, e.g. '2.7'"
+        exit 1
+    fi
+
+    desired_cuda="$1"
+    build_version="$2"
+    build_number="$3"
 fi
-desired_cuda="$1"
+if [[ "$desired_cuda" != cpu ]]; then
+  desired_cuda="$(echo $desired_cuda | tr -d cuda. )"
+fi
+echo "Building cuda version $desired_cuda and torchvision version: $build_version build_number: $build_number"
 
-export TORCHVISION_BUILD_VERSION="0.3.0"
-export TORCHVISION_BUILD_NUMBER=1
+if [[ "$desired_cuda" == 'cpu' ]]; then
+    cpu_only=1
+    cuver="cpu"
+else
+    # Switch desired_cuda to be M.m to be consistent with other scripts in
+    # pytorch/builder
+    export FORCE_CUDA=1
+    cuda_nodot="$desired_cuda"
+
+    if [[ ${#cuda_nodot} -eq 2 ]]; then
+        desired_cuda="${desired_cuda:0:1}.${desired_cuda:1:1}"
+    elif [[ ${#cuda_nodot} -eq 3 ]]; then
+        desired_cuda="${desired_cuda:0:2}.${desired_cuda:2:1}"
+    else
+        echo "unknown cuda version $cuda_nodot"
+        exit 1
+    fi
+
+    cuver="cu$cuda_nodot"
+fi
+
+export TORCHVISION_BUILD_VERSION=$build_version
+export TORCHVISION_BUILD_NUMBER=$build_number
+
+if [[ -z "$DESIRED_PYTHON" ]]; then
+    DESIRED_PYTHON=('3.5' '3.6' '3.7')
+fi
 
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
@@ -26,69 +67,147 @@ if [[ -z "$WIN_PACKAGE_WORK_DIR" ]]; then
     WIN_PACKAGE_WORK_DIR="$(echo $(pwd -W) | tr '/' '\\')\\tmp_conda_$(date +%H%M%S)"
 fi
 
-if [[ "$OSTYPE" == "msys" ]]; then
-    mkdir -p "$WIN_PACKAGE_WORK_DIR" || true
-    vision_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/torchvision-src"
-    git config --system core.longpaths true
-else
-    vision_rootdir="$(pwd)/torchvision-src"
-fi
+mkdir -p "$WIN_PACKAGE_WORK_DIR" || true
+vision_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/torchvision-src"
+git config --system core.longpaths true
 
 if [[ ! -d "$vision_rootdir" ]]; then
     rm -rf "$vision_rootdir"
     git clone "https://github.com/pytorch/vision" "$vision_rootdir"
     pushd "$vision_rootdir"
-    git checkout v$TORCHVISION_BUILD_VERSION
+    git checkout $PYTORCH_BRANCH
     popd
 fi
 
 cd "$SOURCE_DIR"
 
-if [[ "$OSTYPE" == "msys" ]]; then
-    export tmp_conda="${WIN_PACKAGE_WORK_DIR}\\conda"
-    export miniconda_exe="${WIN_PACKAGE_WORK_DIR}\\miniconda.exe"
-    rm -rf "$tmp_conda"
-    rm -f "$miniconda_exe"
-    curl -sSk https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe"
-    "$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe"
-    pushd $tmp_conda
-    export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH"
-    popd
-    # We have to skip 3.17 because of the following bug.
-    # https://github.com/conda/conda-build/issues/3285
-    retry conda install -yq conda-build
-fi
+export tmp_conda="${WIN_PACKAGE_WORK_DIR}\\conda"
+export miniconda_exe="${WIN_PACKAGE_WORK_DIR}\\miniconda.exe"
+rm -rf "$tmp_conda"
+rm -f "$miniconda_exe"
+curl -sSk https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe"
+"$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe"
+pushd $tmp_conda
+export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH"
+popd
+retry conda install -yq conda-build
 
-ANACONDA_USER=pytorch
+ANACONDA_USER=pytorch-nightly
 conda config --set anaconda_upload no
 
 
 export TORCHVISION_PACKAGE_SUFFIX=""
 if [[ "$desired_cuda" == 'cpu' ]]; then
     export CONDA_CUDATOOLKIT_CONSTRAINT=""
+    export CONDA_CPUONLY_FEATURE="- cpuonly # [not osx]"
     export CUDA_VERSION="None"
-    if [[ "$OSTYPE" != "darwin"* ]]; then
-        export TORCHVISION_PACKAGE_SUFFIX="-cpu"
-    fi
 else
+    export CONDA_CPUONLY_FEATURE=""
     . ./switch_cuda_version.sh $desired_cuda
-    if [[ "$desired_cuda" == "10.0" ]]; then
-	export CONDA_CUDATOOLKIT_CONSTRAINT="    - cudatoolkit >=10.0,<10.1 # [not osx]"
+    if [[ "$desired_cuda" == "10.1" ]]; then
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.1,<10.2 # [not osx]"
+    elif [[ "$desired_cuda" == "10.0" ]]; then
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]"
+    elif [[ "$desired_cuda" == "9.2" ]]; then
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]"
     elif [[ "$desired_cuda" == "9.0" ]]; then
-	export CONDA_CUDATOOLKIT_CONSTRAINT="    - cudatoolkit >=9.0,<9.1 # [not osx]"
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.0,<9.1 # [not osx]"
+    elif [[ "$desired_cuda" == "8.0" ]]; then
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=8.0,<8.1 # [not osx]"
     else
-	echo "unhandled desired_cuda: $desired_cuda"
-	exit 1
+        echo "unhandled desired_cuda: $desired_cuda"
+        exit 1
     fi
 fi
 
-if [[ "$OSTYPE" == "msys" ]]; then
-    time conda build -c $ANACONDA_USER --no-anaconda-upload vs2017
+if [[ -z "$PYTORCH_VERSION" ]]; then
+    export CONDA_CHANNEL_FLAGS="-c pytorch-nightly"
+    export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | \
+                                python -c "import os, sys, json, re; cuver = '$cuver'; \
+                                cuver = cuver.replace('cu', 'cuda') if cuver != 'cpu' else cuver; \
+                                print(re.sub(r'\\+.*$', '', \
+                                [x['version'] for x in json.load(sys.stdin)['pytorch'] \
+                                    if (x['platform'] == 'darwin' or cuver in x['fn']) \
+                                    and 'py' + os.environ['DESIRED_PYTHON'] in x['fn']][-1]))")"
+    if [[ -z "$PYTORCH_VERSION" ]]; then
+        echo "PyTorch version auto detection failed"
+        echo "No package found for desired_cuda=$desired_cuda and DESIRED_PYTHON=$DESIRED_PYTHON"
+        exit 1
+    fi
+else
+    export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly"
+fi
+if [[ "$desired_cuda" == 'cpu' ]]; then
+    export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION"
+    export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION"
 else
-    time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchvision
+    export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}"
+    export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}"
 fi
-time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchvision
-time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchvision
-time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchvision
+
+# Loop through all Python versions to build a package for each
+for py_ver in "${DESIRED_PYTHON[@]}"; do
+    build_string="py${py_ver}_${build_string_suffix}"
+    folder_tag="${build_string}_$(date +'%Y%m%d')"
+
+    # Create the conda package into this temporary folder. This is so we can find
+    # the package afterwards, as there's no easy way to extract the final filename
+    # from conda-build
+    output_folder="out_$folder_tag"
+    rm -rf "$output_folder"
+    mkdir "$output_folder"
+
+    # We need to build the compiler activation scripts first on Windows
+    time VSDEVCMD_ARGS=${VSDEVCMD_ARGS[@]} \
+        conda build -c "$ANACONDA_USER" \
+                    --no-anaconda-upload \
+                    --output-folder "$output_folder" \
+                    ../vs2017
+
+    conda config --set anaconda_upload no
+    echo "Calling conda-build at $(date)"
+    if [[ "$desired_cuda" == "9.2" ]]; then
+        time CMAKE_ARGS=${CMAKE_ARGS[@]} \
+            BUILD_VERSION="$TORCHVISION_BUILD_VERSION" \
+            CU_VERSION="$cuver" \
+            SOURCE_ROOT_DIR="$vision_rootdir" \
+            conda build -c "$ANACONDA_USER" \
+                        -c defaults \
+                        -c conda-forge \
+                        -c "numba/label/dev" \
+                        --no-anaconda-upload \
+                        --python "$py_ver" \
+                        --output-folder "$output_folder" \
+                        --no-verify \
+                        --no-test \
+                        ../torchvision
+    else
+        time CMAKE_ARGS=${CMAKE_ARGS[@]} \
+            BUILD_VERSION="$TORCHVISION_BUILD_VERSION" \
+            CU_VERSION="$cuver" \
+            SOURCE_ROOT_DIR="$vision_rootdir" \
+            conda build -c "$ANACONDA_USER" \
+                        -c defaults \
+                        -c conda-forge \
+                        --no-anaconda-upload \
+                        --python "$py_ver" \
+                        --output-folder "$output_folder" \
+                        --no-verify \
+                        --no-test \
+                        ../torchvision
+    fi
+    echo "Finished conda-build at $(date)"
+
+    # Extract the package for testing
+    ls -lah "$output_folder"
+    built_package="$(find $output_folder/ -name '*torchvision*.tar.bz2')"
+
+    # Copy the built package to the host machine for persistence before testing
+    if [[ -n "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then
+        mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true
+        cp "$built_package" "$PYTORCH_FINAL_PACKAGE_DIR/"
+    fi
+done
+
 
 set +e

packaging/torchvision/bld.bat

@@ -16,6 +16,7 @@ set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%desired_cuda%
 set CUDA_BIN_PATH=%CUDA_PATH%\bin
 set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr
 if "%desired_cuda%" == "9.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
+if "%desired_cuda%" == "9.2" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
 if "%desired_cuda%" == "10.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50
 
 :cuda_flags_end

packaging/torchvision/meta.yaml

@@ -14,6 +14,7 @@ requirements:
     - setuptools
     {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }}
     {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }}
+    {{ environ.get('CONDA_CPUONLY_FEATURE') }}
 
   run:
     - python
@@ -31,7 +32,7 @@ build:
     - FORCE_CUDA
     - NVCC_FLAGS
   features:
-    {{ CONDA_CPUONLY_FEATURE }}
+    {{ environ.get('CONDA_CPUONLY_FEATURE') }}
 
 test:
   imports:

packaging/vs2017/activate.bat

+:: Set env vars that tell distutils to use the compiler that we put on path
+SET DISTUTILS_USE_SDK=1
+SET MSSdk=1
+
+SET "VS_VERSION=15.0"
+SET "VS_MAJOR=15"
+SET "VS_YEAR=2017"
+
+set "MSYS2_ARG_CONV_EXCL=/AI;/AL;/OUT;/out"
+set "MSYS2_ENV_CONV_EXCL=CL"
+
+:: For Python 3.5+, ensure that we link with the dynamic runtime.  See
+:: http://stevedower.id.au/blog/building-for-python-3-5-part-two/ for more info
+set "PY_VCRUNTIME_REDIST=%PREFIX%\\bin\\vcruntime140.dll"
+
+for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
+    if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
+        set "VSINSTALLDIR=%%i\"
+        goto :vswhere
+    )
+)
+
+:vswhere
+
+:: Shorten PATH to avoid the `input line too long` error.
+SET MyPath=%PATH%
+
+setlocal EnableDelayedExpansion
+
+SET TempPath="%MyPath:;=";"%"
+SET var=
+FOR %%a IN (%TempPath%) DO (
+    IF EXIST %%~sa (
+        SET "var=!var!;%%~sa"
+    )
+)
+
+set "TempPath=!var:~1!"
+endlocal & set "PATH=%TempPath%"
+
+:: Shorten current directory too
+FOR %%A IN (.) DO CD "%%~sA"
+
+:: other things added by install_activate.bat at package build time

packaging/vs2017/conda_build_config.yaml

+blas_impl:
+  - mkl                        # [x86_64]
+c_compiler:
+  - vs2017                     # [win]
+cxx_compiler:
+  - vs2017                     # [win]
+python:
+  - 3.5
+  - 3.6
+# This differs from target_platform in that it determines what subdir the compiler
+#    will target, not what subdir the compiler package will be itself.
+#    For example, we need a win-64 vs2008_win-32 package, so that we compile win-32
+#    code on win-64 miniconda.
+cross_compiler_target_platform:
+  - win-64                     # [win]
+target_platform:
+  - win-64                     # [win]
+vc:
+  - 14
+zip_keys:
+  -                             # [win]
+    - vc                        # [win]
+    - c_compiler                # [win]
+    - cxx_compiler              # [win]

packaging/vs2017/install_activate.bat

+set YEAR=2017
+set VER=15
+
+mkdir "%PREFIX%\etc\conda\activate.d"
+COPY "%RECIPE_DIR%\activate.bat" "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+
+IF "%cross_compiler_target_platform%" == "win-64" (
+  set "target_platform=amd64"
+  echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR% Win64" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  IF "%VSDEVCMD_ARGS%" == "" (
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  ) ELSE (
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  )
+  echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  ) else (
+  set "target_platform=x86"
+  echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  echo CALL "VC\Auxiliary\Build\vcvars32.bat" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  echo popd
+  )
+

packaging/vs2017/install_runtime.bat

+set VC_PATH=x86
+if "%ARCH%"=="64" (
+   set VC_PATH=x64
+)
+
+set MSC_VER=2017
+
+rem :: This should always be present for VC installed with VS.  Not sure about VC installed with Visual C++ Build Tools 2015
+rem FOR /F "usebackq tokens=3*" %%A IN (`REG QUERY "HKEY_LOCAL_MACHINE\Software\Microsoft\DevDiv\VC\Servicing\14.0\IDE.x64" /v UpdateVersion`) DO (
+rem     set SP=%%A
+rem     )
+
+rem if not "%SP%" == "%PKG_VERSION%" (
+rem    echo "Version detected from registry: %SP%"
+rem    echo    "does not match version of package being built (%PKG_VERSION%)"
+rem    echo "Do you have current updates for VS 2015 installed?"
+rem    exit 1
+rem )
+
+
+REM ========== REQUIRES Win 10 SDK be installed, or files otherwise copied to location below!
+robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%"  "%LIBRARY_BIN%" *.dll /E
+robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%"  "%PREFIX%" *.dll /E
+if %ERRORLEVEL% GEQ 8 exit 1
+
+REM ========== This one comes from visual studio 2017
+set "VC_VER=141"
+
+for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
+    if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
+        set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
+        goto :eof
+    )
+)
+
+@setlocal
+call "%VS15VARSALL%" x64
+
+set "REDIST_ROOT=%VCToolsRedistDir%%VC_PATH%"
+
+robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%LIBRARY_BIN%" *.dll /E
+if %ERRORLEVEL% LSS 8 exit 0
+robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%PREFIX%" *.dll /E
+if %ERRORLEVEL% LSS 8 exit 0
+robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%LIBRARY_BIN%" *.dll /E
+if %ERRORLEVEL% LSS 8 exit 0
+robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%PREFIX%" *.dll /E
+if %ERRORLEVEL% LSS 8 exit 0
+@endlocal

packaging/vs2017/meta.yaml

+{% set vcver="14.1" %}
+{% set vcfeature="14" %}
+{% set vsyear="2017" %}
+{% set fullver="15.4.27004.2010" %}
+
+package:
+  name: vs{{ vsyear }}
+  version: {{ fullver }}
+
+build:
+  skip: True  [not win]
+  script_env:
+    - VSDEVCMD_ARGS # [win]
+
+outputs:
+  - name: vs{{ vsyear }}_{{ cross_compiler_target_platform }}
+    script: install_activate.bat
+    track_features:
+      # VS 2017 is binary-compatible with VS 2015/vc14.  Tools are "v141".
+      strong:
+        - vc{{ vcfeature }}
+    run_exports:
+      - vc {{ vcver }}
+    about:
+      summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler
+      license: BSD 3-clause
+  - name: vs{{ vsyear }}_runtime
+    script: install_runtime.bat
+  - name: vc
+    version: {{ vcver }}
+    track_features:
+      - vc{{ vcfeature }}
+    requirements:
+      run:
+        - {{ pin_subpackage('vs' ~ vsyear ~ '_runtime') }}
+    about:
+      home: https://github.com/conda/conda/wiki/VC-features
+      license: Modified BSD License (3-clause)
+      license_family: BSD
+      summary: A meta-package to track VC features.
+      description: |
+          This metapackage is used to activate vc features without
+          depending on Python.
+      doc_url: https://github.com/conda/conda/wiki/VC-features
+      dev_url: https://github.com/conda/conda/wiki/VC-features

packaging/windows/azure-pipelines-ci.yml

+
+# Turn off auto builds for commits
+trigger: none
+pr: none
+
+jobs:
+- template: templates/build_task.yml
+  parameters:
+    package: 'Wheels'
+    spec: 'CPU'
+    msagent: true

packaging/windows/azure-pipelines.yml

+
+# Turn off auto builds for commits
+trigger: none
+pr: none
+
+jobs:
+- template: templates/auth_task.yml
+
+- template: templates/build_task.yml
+  parameters:
+    package: 'Wheels'
+    spec: 'CPU'
+    msagent: true
+
+- template: templates/build_task.yml
+  parameters:
+    package: 'Conda'
+    spec: 'CPU'
+    msagent: true
+
+- template: templates/build_task.yml
+  parameters:
+    package: 'Wheels'
+    spec: 'CUDA'
+    msagent: true
+
+- template: templates/build_task.yml
+  parameters:
+    package: 'Conda'
+    spec: 'CUDA'
+    msagent: true
+
+- template: templates/linux_build_task.yml
+  parameters:
+    msagent: $(ms.hosted.agent.cpu)

packaging/windows/build_vision.bat

@@ -23,18 +23,21 @@ set CUDA_VERSION=%~1
 set TORCHVISION_BUILD_VERSION=%~2
 set TORCHVISION_BUILD_NUMBER=%~3
 
+set BUILD_VERSION=%TORCHVISION_BUILD_VERSION%
+
 :env_end
 
 if NOT "%CUDA_VERSION%" == "cpu" (
     set CUDA_PREFIX=cuda%CUDA_VERSION%
     set CUVER=cu%CUDA_VERSION%
+    set FORCE_CUDA=1
 ) else (
     set CUDA_PREFIX=cpu
     set CUVER=cpu
 )
 
 set BUILD_VISION=1
-set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index
+REM set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index
 
 IF "%DESIRED_PYTHON%" == "" set DESIRED_PYTHON=3.5;3.6;3.7
 set DESIRED_PYTHON_PREFIX=%DESIRED_PYTHON:.=%
@@ -61,8 +64,35 @@ FOR %%v IN (%DESIRED_PYTHON%) DO (
     set PYTHON_VERSION_STR=%%v
     set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
     conda remove -n py!PYTHON_VERSION_STR! --all -y || rmdir %CONDA_HOME%\envs\py!PYTHON_VERSION_STR! /s
-    conda create -n py!PYTHON_VERSION_STR! -y -q numpy>=1.11 mkl>=2018 python=%%v
+    conda create -n py!PYTHON_VERSION_STR! -y -q -c defaults -c conda-forge numpy>=1.11 mkl>=2018 python=%%v ca-certificates scipy av
+)
+
+:: Uncomment for stable releases
+:: FOR %%v IN (%DESIRED_PYTHON%) DO (
+::     set PYTHON_VERSION_STR=%%v
+::     set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
+::     set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%"
+
+::     if "%CUDA_VERSION%" == "100" (
+::         set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl
+::     ) else (
+::         set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0%%2B%CUVER%-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl
+::     )
+::     echo Installing !TORCH_WHEEL!...
+::     pip install "!TORCH_WHEEL!"
+:: )
+
+:: Uncomment for nightly releases
+FOR %%v IN (%DESIRED_PYTHON%) DO (
+    set PYTHON_VERSION_STR=%%v
+    set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
+    set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%"
+
+    set TORCH_WHEEL=torch --pre -f https://download.pytorch.org/whl/nightly/%CUVER%/torch_nightly.html
+    echo Installing !TORCH_WHEEL!...
+    pip install !TORCH_WHEEL!
 )
+
 endlocal
 
 if "%DEBUG%" == "1" (
@@ -71,11 +101,22 @@ if "%DEBUG%" == "1" (
     set BUILD_TYPE=release
 )
 
+:: Install sccache
+if "%USE_SCCACHE%" == "1" (
+    mkdir %CD%\tmp_bin
+    curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %CD%\tmp_bin\sccache.exe
+    if not "%CUDA_VERSION%" == "" (
+        copy %CD%\tmp_bin\sccache.exe %CD%\tmp_bin\nvcc.exe
+
+        set CUDA_NVCC_EXECUTABLE=%CD%\tmp_bin\nvcc
+        set "PATH=%CD%\tmp_bin;%PATH%"
+    )
+)
+
 for %%v in (%DESIRED_PYTHON_PREFIX%) do (
     :: Activate Python Environment
     set PYTHON_PREFIX=%%v
     set "PATH=%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%"
-    pip install %TORCH_WHEEL%
     @setlocal
     :: Set Flags
     if NOT "%CUDA_VERSION%"=="cpu" (

packaging/windows/cuda92.bat

+@echo off
+
+IF NOT "%BUILD_VISION%" == "" (
+    set MODULE_NAME=vision
+) ELSE (
+    set MODULE_NAME=pytorch
+)
+
+IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
+    call internal\clone.bat
+    cd ..
+    IF ERRORLEVEL 1 goto eof
+) ELSE (
+    call internal\clean.bat
+)
+
+call internal\check_deps.bat
+IF ERRORLEVEL 1 goto eof
+
+REM Check for optional components
+
+set USE_CUDA=
+set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+
+IF "%NVTOOLSEXT_PATH%"=="" (
+    echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
+    exit /b 1
+    goto optcheck
+)
+
+IF "%CUDA_PATH_V9_2%"=="" (
+    echo CUDA 9.2 not found, failing
+    exit /b 1
+) ELSE (
+    IF "%BUILD_VISION%" == "" (
+        set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0
+        set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
+    ) ELSE (
+        set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
+    )
+
+    set "CUDA_PATH=%CUDA_PATH_V9_2%"
+    set "PATH=%CUDA_PATH_V9_2%\bin;%PATH%"
+)
+
+:optcheck
+
+IF "%BUILD_VISION%" == "" (
+    call internal\check_opts.bat
+    IF ERRORLEVEL 1 goto eof
+
+    call internal\copy.bat
+    IF ERRORLEVEL 1 goto eof
+)
+
+call internal\setup.bat
+IF ERRORLEVEL 1 goto eof
+
+:eof

packaging/windows/internal/7z_install.bat

-@echo off
-
-curl -k https://www.7-zip.org/a/7z1805-x64.exe -O
-if errorlevel 1 exit /b 1
-
-start /wait 7z1805-x64.exe /S
-if errorlevel 1 exit /b 1
-
-set "PATH=%ProgramFiles%\7-Zip;%PATH%"

packaging/windows/internal/clone.bat

@@ -44,15 +44,11 @@ set PYTORCH_BRANCH=%last_commit%
 
 :latest_end
 
-IF "%BUILD_VISION%" == "" (
-    IF "%PYTORCH_BRANCH%" == "" (
-        set PYTORCH_BRANCH=v%PYTORCH_BUILD_VERSION%
-    )
-    git checkout %PYTORCH_BRANCH%
-    IF ERRORLEVEL 1 git checkout tags/%PYTORCH_BRANCH%
-) ELSE (
-    git checkout v%TORCHVISION_BUILD_VERSION%
+IF "%PYTORCH_BRANCH%" == "" (
+    set PYTORCH_BRANCH=v%TORCHVISION_BUILD_VERSION%
 )
+git checkout %PYTORCH_BRANCH%
+IF ERRORLEVEL 1 git checkout tags/%PYTORCH_BRANCH%
 
 :submodule
 

packaging/windows/internal/cuda_install.bat

-@echo off
+@echo on
+
+if "%CUDA_VERSION%" == "cpu" (
+    echo Skipping for CPU builds
+    exit /b 0
+)
 
 set SRC_DIR=%~dp0\..
 
@@ -9,87 +14,60 @@ set CUDA_VER_MAJOR=%CUDA_VERSION:~0,-1%
 set CUDA_VER_MINOR=%CUDA_VERSION:~-1,1%
 set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR%
 
-IF %CUDA_VER% LEQ 90 (
-    set "NVCC_PACKAGE=compiler_%CUDA_VERSION_STR%"
-) ELSE (
-    set "NVCC_PACKAGE=nvcc_%CUDA_VERSION_STR%"
-)
-
-IF %CUDA_VER% EQU 80 goto cuda80
-IF %CUDA_VER% EQU 90 goto cuda90
-IF %CUDA_VER% EQU 91 goto cuda91
-IF %CUDA_VER% EQU 92 goto cuda92
-IF %CUDA_VER% EQU 100 goto cuda100
+if %CUDA_VER% EQU 92 goto cuda92
+if %CUDA_VER% EQU 100 goto cuda100
 
 echo CUDA %CUDA_VERSION_STR% is not supported
 exit /b 1
 
-:cuda80
-
-echo CUDA 8.0 is not supported
-exit /b 1
-
-:cuda90
-IF NOT EXIST "%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z" (
-    curl -k -L https://www.dropbox.com/s/z5b7ryz0zrimntl/cuda_9.0.176_windows.7z?dl=1 --output "%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z"
+:cuda92
+if not exist "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe" (
+    curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_9.2.148_win10.exe --output "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe"
     if errorlevel 1 exit /b 1
-    set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z"
-    set "NVCC_PACKAGE=compiler_%CUDA_VERSION_STR%"
+    set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe"
+    set "ARGS=nvcc_9.2 cuobjdump_9.2 nvprune_9.2 cupti_9.2 cublas_9.2 cublas_dev_9.2 cudart_9.2 cufft_9.2 cufft_dev_9.2 curand_9.2 curand_dev_9.2 cusolver_9.2 cusolver_dev_9.2 cusparse_9.2 cusparse_dev_9.2 nvgraph_9.2 nvgraph_dev_9.2 npp_9.2 npp_dev_9.2 nvrtc_9.2 nvrtc_dev_9.2 nvml_dev_9.2"
 )
 
-IF NOT EXIST "%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip" (
-    curl -k -L https://www.dropbox.com/s/6p0xyqh472nu8m1/cudnn-9.0-windows7-x64-v7.zip?dl=1 --output "%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip"
+if not exist "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip" (
+    curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-9.2-windows10-x64-v7.2.1.38.zip --output "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip"
     if errorlevel 1 exit /b 1
-    set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip"
+    set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip"
 )
 
 goto cuda_common
 
-:cuda91
+:cuda100
 
-IF NOT EXIST "%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z" (
-    curl -k -L https://www.dropbox.com/s/7a4sbq0dln6v7t2/cuda_9.1.85_windows.7z?dl=1 --output "%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z"
+if not exist "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe" (
+    curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_10.0.130_411.31_win10.exe --output "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe"
     if errorlevel 1 exit /b 1
-    set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z"
-    set "NVCC_PACKAGE=nvcc_%CUDA_VERSION_STR%"
+    set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe"
+    set "ARGS=nvcc_10.0 cuobjdump_10.0 nvprune_10.0 cupti_10.0 cublas_10.0 cublas_dev_10.0 cudart_10.0 cufft_10.0 cufft_dev_10.0 curand_10.0 curand_dev_10.0 cusolver_10.0 cusolver_dev_10.0 cusparse_10.0 cusparse_dev_10.0 nvgraph_10.0 nvgraph_dev_10.0 npp_10.0 npp_dev_10.0 nvrtc_10.0 nvrtc_dev_10.0 nvml_dev_10.0"
 )
 
-IF NOT EXIST "%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip" (
-    curl -k -L https://www.dropbox.com/s/e0prhgsrbyfi4ov/cudnn-9.1-windows7-x64-v7.zip?dl=1 --output "%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip"
+if not exist "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip" (
+    curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-10.0-windows10-x64-v7.4.1.5.zip --output "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip"
     if errorlevel 1 exit /b 1
-    set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip"
+    set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip"
 )
 
 goto cuda_common
 
-:cuda92
-
-echo CUDA 9.2 is not supported
-exit /b 1
-
-:cuda100
-
-echo CUDA 10.0 is not supported
-exit /b 1
-
 :cuda_common
 
-set "CUDA_PREFIX=cuda%CUDA_VERSION%"
-
-IF NOT EXIST "%SRC_DIR%\temp_build\NvToolsExt.7z" (
+if not exist "%SRC_DIR%\temp_build\NvToolsExt.7z" (
     curl -k -L https://www.dropbox.com/s/9mcolalfdj4n979/NvToolsExt.7z?dl=1 --output "%SRC_DIR%\temp_build\NvToolsExt.7z"
     if errorlevel 1 exit /b 1
 )
 
 echo Installing CUDA toolkit...
-
 7z x %CUDA_SETUP_FILE% -o"%SRC_DIR%\temp_build\cuda"
 pushd "%SRC_DIR%\temp_build\cuda"
-dir
-start /wait setup.exe -s %NVCC_PACKAGE% cublas_%CUDA_VERSION_STR% cublas_dev_%CUDA_VERSION_STR% cudart_%CUDA_VERSION_STR% curand_%CUDA_VERSION_STR% curand_dev_%CUDA_VERSION_STR% cusparse_%CUDA_VERSION_STR% cusparse_dev_%CUDA_VERSION_STR% nvrtc_%CUDA_VERSION_STR% nvrtc_dev_%CUDA_VERSION_STR% cufft_%CUDA_VERSION_STR% cufft_dev_%CUDA_VERSION_STR%
+start /wait setup.exe -s %ARGS%
 popd
+
 echo Installing VS integration...
-xcopy /Y "%SRC_DIR%\temp_build\cuda\_vs\*.*" "c:\Program Files (x86)\MSBuild\Microsoft.Cpp\v4.0\V140\BuildCustomizations"
+xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\IDE\VC\VCTargets\BuildCustomizations"
 
 echo Installing NvToolsExt...
 7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt"
@@ -100,23 +78,22 @@ xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\bin\x64\*.*" "%ProgramFiles%\NVIDIA Co
 xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\include\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\include"
 xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\lib\x64\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\lib\x64"
 
+echo Setting up environment...
+set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\libnvvp;%PATH%"
+set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
+set "CUDA_PATH_V%CUDA_VER_MAJOR%_%CUDA_VER_MINOR%=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
+set "NVTOOLSEXT_PATH=%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64"
+
+if not exist "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin\nvcc.exe" (
+    echo CUDA %CUDA_VERSION_STR% installed failed.
+    exit /b 1
+)
+
 echo Installing cuDNN...
 7z x %CUDNN_SETUP_FILE% -o"%SRC_DIR%\temp_build\cudnn"
 xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\bin\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin"
 xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\lib\x64\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\lib\x64"
 xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\include\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\include"
 
-echo Setting up environment...
-set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\libnvvp;%PATH%"
-set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
-set "CUDA_PATH_V%CUDA_VER_MAJOR%_%CUDA_VER_MINOR%=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
-set "NVTOOLSEXT_PATH=%ProgramFiles%\NVIDIA Corporation\NvToolsExt\"
-
 echo Cleaning temp files
-rd /s /q "%SRC_DIR%\temp_build"
-pushd %TEMP%
-rd /s /q .
-popd
-
-echo Using VS2015 as NVCC compiler
-set "CUDAHOSTCXX=%VS140COMNTOOLS%\..\..\VC\bin\amd64\cl.exe"
+rd /s /q "%SRC_DIR%\temp_build" || ver > nul

packaging/windows/internal/dep_install.bat

+@echo off
+
+REM curl -k https://www.7-zip.org/a/7z1805-x64.exe -O
+REM if errorlevel 1 exit /b 1
+
+REM start /wait 7z1805-x64.exe /S
+REM if errorlevel 1 exit /b 1
+
+REM set "PATH=%ProgramFiles%\7-Zip;%PATH%"
+
+choco feature disable --name showDownloadProgress
+choco feature enable --name allowGlobalConfirmation
+
+choco install curl 7zip

packaging/windows/internal/env_fix.bat

+@echo off
+
+:: Caution: Please don't use this script locally
+:: It may destroy your build environment.
+
+setlocal
+
+IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" (
+    echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
+    exit /b 1
+)
+
+for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
+    if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
+        set "VS15INSTALLDIR=%%i"
+        set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
+        goto vswhere
+    )
+)
+
+:vswhere
+
+IF "%VS15VCVARSALL%"=="" (
+    echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
+    exit /b 1
+)
+
+call "%VS15VCVARSALL%" x86_amd64
+for /f "usebackq tokens=*" %%i in (`where link.exe`) do move "%%i" "%%i.bak"
+
+endlocal

packaging/windows/internal/nightly_defaults.bat

+@echo on
+
+if "%~1"=="" goto arg_error
+if NOT "%~2"=="" goto arg_error
+goto arg_end
+
+:arg_error
+
+echo Illegal number of parameters. Pass packge type `Conda` or `Wheels`.
+exit /b 1
+
+:arg_end
+
+echo "nightly_defaults.bat at %CD% starting at %DATE%"
+
+set SRC_DIR=%~dp0\..
+
+:: NIGHTLIES_FOLDER
+:: N.B. this is also defined in cron_start.sh
+::   An arbitrary root folder to store all nightlies folders, each of which is a
+::   parent level date folder with separate subdirs for logs, wheels, conda
+::   packages, etc. This should be kept the same across all scripts called in a
+::   cron job, so it only has a default value in the top-most script
+::   build_cron.sh to avoid the default values from diverging.
+if "%NIGHTLIES_FOLDER%" == "" set "NIGHTLIES_FOLDER=%SRC_DIR%"
+
+:: NIGHTLIES_DATE
+:: N.B. this is also defined in cron_start.sh
+::   The date in YYYY_mm_dd format that we are building for. If this is not
+::   already set, then this will first try to find the date of the nightlies
+::   folder that this builder repo exists in; e.g. if this script exists in
+::   some_dir/2019_09_04/builder/cron/ then this will be set to 2019_09_04 (must
+::   match YYYY_mm_dd). This is for convenience when debugging/uploading past
+::   dates, so that you don't have to set NIGHTLIES_DATE yourself. If a date
+::   folder cannot be found in that exact location, then this will default to
+::   the current date.
+
+
+if "%NIGHTLIES_DATE%" == "" ( goto date_start ) else ( goto date_end )
+
+:date_start
+
+set "DATE_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyy_MM_dd'"
+set "DATE_COMPACT_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyyMMdd'"
+
+FOR /F "delims=" %%i IN ('powershell -c "%DATE_CMD%"') DO set NIGHTLIES_DATE=%%i
+FOR /F "delims=" %%i IN ('powershell -c "%DATE_COMPACT_CMD%"') DO set NIGHTLIES_DATE_COMPACT=%%i
+
+:date_end
+
+if "%NIGHTLIES_DATE_COMPACT%" == "" set NIGHTLIES_DATE_COMPACT=%NIGHTLIES_DATE:~0,4%%NIGHTLIES_DATE:~5,2%%NIGHTLIES_DATE:~8,2%
+
+:: Used in lots of places as the root dir to store all conda/wheel/manywheel
+:: packages as well as logs for the day
+set today=%NIGHTLIES_FOLDER%\%NIGHTLIES_DATE%
+mkdir "%today%" || ver >nul
+
+
+::#############################################################################
+:: Add new configuration variables below this line. 'today' should always be
+:: defined ASAP to avoid weird errors
+::#############################################################################
+
+
+:: List of people to email when things go wrong. This is passed directly to
+:: `mail -t`
+:: TODO: Not supported yet
+if "%NIGHTLIES_EMAIL_LIST%" == "" set NIGHTLIES_EMAIL_LIST=peterghost86@gmail.com
+
+:: PYTORCH_CREDENTIALS_FILE
+::   A bash file that exports credentials needed to upload to aws and anaconda.
+::   Needed variables are PYTORCH_ANACONDA_USERNAME, PYTORCH_ANACONDA_PASSWORD,
+::   AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY. Or it can just export the AWS
+::   keys and then prepend a logged-in conda installation to the path.
+:: TODO: Not supported yet
+if "%PYTORCH_CREDENTIALS_FILE%" == "" set PYTORCH_CREDENTIALS_FILE=/c/Users/administrator/nightlies/credentials.sh
+
+:: Location of the temporary miniconda that is downloaded to install conda-build
+:: and aws to upload finished packages TODO this is messy to install this in
+:: upload.sh and later use it in upload_logs.sh
+if "%CONDA_UPLOADER_INSTALLATION%" == "" set "CONDA_UPLOADER_INSTALLATION=%today%\miniconda"
+
+:: N.B. BUILDER_REPO and BUILDER_BRANCH are both set in cron_start.sh, as that
+:: is the script that actually clones the builder repo that /this/ script is
+:: running from.
+pushd "%SRC_DIR%\.."
+set NIGHTLIES_BUILDER_ROOT=%CD%
+popd
+
+:: The shared pytorch repo to be used by all builds
+if "%NIGHTLIES_PYTORCH_ROOT%" == "" set "NIGHTLIES_PYTORCH_ROOT=%today%\vision"
+
+:: PYTORCH_REPO
+::   The Github org/user whose fork of Pytorch to check out (git clone
+::   https://github.com/<THIS_PART>/pytorch.git). This will always be cloned
+::   fresh to build with. Default is 'pytorch'
+if "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch
+
+:: PYTORCH_BRANCH
+::   The branch of Pytorch to checkout for building (git checkout <THIS_PART>).
+::   This can either be the name of the branch (e.g. git checkout
+::   my_branch_name) or can be a git commit (git checkout 4b2674n...). Default
+::   is 'latest', which is a special term that signals to pull the last commit
+::   before 0:00 midnight on the NIGHTLIES_DATE
+if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=latest
+
+:: Clone the requested pytorch checkout
+if exist "%NIGHTLIES_PYTORCH_ROOT%" ( goto clone_end ) else ( goto clone_start )
+
+:clone_start
+
+git clone --recursive "https://github.com/%PYTORCH_REPO%/vision.git" "%NIGHTLIES_PYTORCH_ROOT%"
+pushd "%NIGHTLIES_PYTORCH_ROOT%"
+
+if "%PYTORCH_BRANCH%" == "latest" ( goto latest_start ) else ( goto latest_end )
+
+:latest_start
+
+:: Switch to the latest commit by 11:59 yesterday
+echo PYTORCH_BRANCH is set to latest so I will find the last commit
+echo before 0:00 midnight on %NIGHTLIES_DATE%
+set git_date=%NIGHTLIES_DATE:_=-%
+FOR /F "delims=" %%i IN ('git log --before %git_date% -n 1 "--pretty=%%H"') DO set last_commit=%%i
+echo Setting PYTORCH_BRANCH to %last_commit% since that was the last
+echo commit before %NIGHTLIES_DATE%
+set PYTORCH_BRANCH=%last_commit%
+
+:latest_end
+
+git checkout "%PYTORCH_BRANCH%"
+git submodule update
+popd
+
+:clone_end
+
+if "%CUDA_VERSION%" == "cpu" (
+    set _DESIRED_CUDA=cpu
+) else (
+    set _DESIRED_CUDA=cu%CUDA_VERSION%
+)
+
+:: PYTORCH_BUILD_VERSION
+::   The actual version string. Used in conda like
+::       pytorch-nightly==1.0.0.dev20180908
+::   or in manylinux like
+::       torch_nightly-1.0.0.dev20180908-cp27-cp27m-linux_x86_64.whl
+if "%TORCHVISION_BUILD_VERSION%" == "" set TORCHVISION_BUILD_VERSION=0.5.0.dev%NIGHTLIES_DATE_COMPACT%
+
+if "%~1" == "Wheels" (
+    if not "%CUDA_VERSION%" == "100" (
+        set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%+%_DESIRED_CUDA%
+    )
+)
+
+:: PYTORCH_BUILD_NUMBER
+::   This is usually the number 1. If more than one build is uploaded for the
+::   same version/date, then this can be incremented to 2,3 etc in which case
+::   '.post2' will be appended to the version string of the package. This can
+::   be set to '0' only if OVERRIDE_PACKAGE_VERSION is being used to bypass
+::   all the version string logic in downstream scripts. Since we use the
+::   override below, exporting this shouldn't actually matter.
+if "%TORCHVISION_BUILD_NUMBER%" == "" set /a TORCHVISION_BUILD_NUMBER=1
+if %TORCHVISION_BUILD_NUMBER% GTR 1 set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%%TORCHVISION_BUILD_NUMBER%
+
+:: The nightly builds use their own versioning logic, so we override whatever
+:: logic is in setup.py or other scripts
+:: TODO: Not supported yet
+set OVERRIDE_PACKAGE_VERSION=%TORCHVISION_BUILD_VERSION%
+set BUILD_VERSION=%TORCHVISION_BUILD_VERSION%
+
+:: Build folder for conda builds to use
+if "%TORCH_CONDA_BUILD_FOLDER%" == "" set TORCH_CONDA_BUILD_FOLDER=torchvision
+
+:: TORCH_PACKAGE_NAME
+::   The name of the package to upload. This should probably be pytorch or
+::   pytorch-nightly. N.B. that pip will change all '-' to '_' but conda will
+::   not. This is dealt with in downstream scripts.
+:: TODO: Not supported yet
+if "%TORCH_PACKAGE_NAME%" == "" set TORCH_PACKAGE_NAME=torchvision
+
+:: PIP_UPLOAD_FOLDER should end in a slash. This is to handle it being empty
+:: (when uploading to e.g. whl/cpu/) and also to handle nightlies (when
+:: uploading to e.g. /whl/nightly/cpu)
+:: TODO: Not supported yet
+if "%PIP_UPLOAD_FOLDER%" == "" set "PIP_UPLOAD_FOLDER=nightly\"
+
+:: The location of the binary_sizes dir in s3 is hardcoded into
+:: upload_binary_sizes.sh
+
+:: DAYS_TO_KEEP
+::   How many days to keep around for clean.sh. Build folders older than this
+::   will be purged at the end of cron jobs. '1' means to keep only the current
+::   day. Values less than 1 are not allowed. The default is 5.
+:: TODO: Not supported yet
+if "%DAYS_TO_KEEP%" == "" set /a DAYS_TO_KEEP=5
+if %DAYS_TO_KEEP% LSS 1 (
+    echo DAYS_TO_KEEP cannot be less than 1.
+    echo A value of 1 means to only keep the build for today
+    exit /b 1
+)

packaging/windows/internal/publish.bat

+@echo off
+
+set SRC_DIR=%~dp0
+pushd %SRC_DIR%
+
+if NOT "%CUDA_VERSION%" == "cpu" (
+    set PACKAGE_SUFFIX=_cuda%CUDA_VERSION%
+) else (
+    set PACKAGE_SUFFIX=
+)
+
+if "%PACKAGEFULLNAME%" == "Conda" (
+    set PACKAGE=conda
+) else (
+    set PACKAGE=wheels
+)
+
+set PUBLISH_BRANCH=%PACKAGE%_%DESIRED_PYTHON%%PACKAGE_SUFFIX%
+
+git clone %ARTIFACT_REPO_URL% -b %PUBLISH_BRANCH% --single-branch >nul 2>&1
+
+IF ERRORLEVEL 1 (
+    echo Branch %PUBLISH_BRANCH% not exist, falling back to master
+    set NO_BRANCH=1
+    git clone %ARTIFACT_REPO_URL% -b master --single-branch >nul 2>&1
+)
+
+IF ERRORLEVEL 1 (
+    echo Clone failed
+    goto err
+)
+
+cd pytorch_builder
+attrib -s -h -r . /s /d
+
+:: Empty repo
+rd /s /q . || ver >nul
+
+IF NOT EXIST %PACKAGE% mkdir %PACKAGE%
+
+xcopy /S /E /Y ..\..\output\*.* %PACKAGE%\
+
+git config --global user.name "Azure DevOps"
+git config --global user.email peterghost86@gmail.com
+git init
+git checkout --orphan %PUBLISH_BRANCH%
+git remote add origin %ARTIFACT_REPO_URL%
+git add .
+git commit -m "Update artifacts"
+
+:push
+
+if "%RETRY_TIMES%" == "" (
+    set /a RETRY_TIMES=10
+    set /a SLEEP_TIME=2
+) else (
+    set /a RETRY_TIMES=%RETRY_TIMES%-1
+    set /a SLEEP_TIME=%SLEEP_TIME%*2
+)
+
+git push origin %PUBLISH_BRANCH% -f > nul 2>&1
+
+IF ERRORLEVEL 1 (
+    echo Git push retry times remaining: %RETRY_TIMES%
+    echo Sleep time: %SLEEP_TIME% seconds
+    IF %RETRY_TIMES% EQU 0 (
+        echo Push failed
+        goto err
+    )
+    waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul
+    goto push
+) ELSE (
+    set RETRY_TIMES=
+    set SLEEP_TIME=
+)
+
+popd
+
+exit /b 0
+
+:err
+
+popd
+
+exit /b 1

packaging/windows/internal/setup.bat

@@ -28,53 +28,6 @@ if "%CXX%"=="sccache cl" (
     sccache --zero-stats
 )
 
-
-if "%BUILD_PYTHONLESS%" == "" goto pytorch else goto libtorch
-
-:libtorch
-set VARIANT=shared-with-deps
-
-mkdir libtorch
-mkdir libtorch\bin
-mkdir libtorch\cmake
-mkdir libtorch\include
-mkdir libtorch\lib
-mkdir libtorch\share
-mkdir libtorch\test
-
-mkdir build
-pushd build
-python ../tools/build_libtorch.py
-popd
-
-IF ERRORLEVEL 1 exit /b 1
-IF NOT ERRORLEVEL 0 exit /b 1
-
-move /Y torch\bin\*.* libtorch\bin\
-move /Y torch\cmake\*.* libtorch\cmake\
-robocopy /move /e torch\include\ libtorch\include\
-move /Y torch\lib\*.* libtorch\lib\
-robocopy /move /e torch\share\ libtorch\share\
-move /Y torch\test\*.* libtorch\test\
-
-move /Y libtorch\bin\*.dll libtorch\lib\
-
-git rev-parse HEAD > libtorch\build-hash
-
-IF "%DEBUG%" == "" (
-    set LIBTORCH_PREFIX=libtorch-win-%VARIANT%
-) ELSE (
-    set LIBTORCH_PREFIX=libtorch-win-%VARIANT%-debug
-)
-
-7z a -tzip %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip libtorch\*
-
-mkdir ..\output\%CUDA_PREFIX%
-copy /Y %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip ..\output\%CUDA_PREFIX%\
-copy /Y %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip ..\output\%CUDA_PREFIX%\%LIBTORCH_PREFIX%-latest.zip
-
-goto build_end
-
 :pytorch
 :: This stores in e.g. D:/_work/1/s/windows/output/cpu
 pip wheel -e . --no-deps --wheel-dir ../output/%CUDA_PREFIX%