Unverified Commit 490966f2 authored by peterjc123's avatar peterjc123 Committed by GitHub
Browse files

Windows build scripts (#1241)

* Windows build scripts

Add testing yaml

* Change urls

* Test MS agent

* Switch base image

* Use fb links
parent ef6d6c47
......@@ -10,15 +10,56 @@ retry () {
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
}
if [ "$#" -ne 1 ]; then
echo "Illegal number of parameters. Pass cuda version"
echo "CUDA version should be M.m with no dot, e.g. '8.0' or 'cpu'"
exit 1
# Parse arguments and determmine version
###########################################################
if [[ -n "$DESIRED_CUDA" && -n "$TORCHVISION_BUILD_VERSION" && -n "$TORCHVISION_BUILD_NUMBER" ]]; then
desired_cuda="$DESIRED_CUDA"
build_version="$PYTORCH_BUILD_VERSION"
build_number="$PYTORCH_BUILD_NUMBER"
else
if [ "$#" -ne 3 ]; then
echo "Illegal number of parameters. Pass cuda version, pytorch version, build number"
echo "CUDA version should be Mm with no dot, e.g. '80'"
echo "DESIRED_PYTHON should be M.m, e.g. '2.7'"
exit 1
fi
desired_cuda="$1"
build_version="$2"
build_number="$3"
fi
desired_cuda="$1"
if [[ "$desired_cuda" != cpu ]]; then
desired_cuda="$(echo $desired_cuda | tr -d cuda. )"
fi
echo "Building cuda version $desired_cuda and torchvision version: $build_version build_number: $build_number"
export TORCHVISION_BUILD_VERSION="0.3.0"
export TORCHVISION_BUILD_NUMBER=1
if [[ "$desired_cuda" == 'cpu' ]]; then
cpu_only=1
cuver="cpu"
else
# Switch desired_cuda to be M.m to be consistent with other scripts in
# pytorch/builder
export FORCE_CUDA=1
cuda_nodot="$desired_cuda"
if [[ ${#cuda_nodot} -eq 2 ]]; then
desired_cuda="${desired_cuda:0:1}.${desired_cuda:1:1}"
elif [[ ${#cuda_nodot} -eq 3 ]]; then
desired_cuda="${desired_cuda:0:2}.${desired_cuda:2:1}"
else
echo "unknown cuda version $cuda_nodot"
exit 1
fi
cuver="cu$cuda_nodot"
fi
export TORCHVISION_BUILD_VERSION=$build_version
export TORCHVISION_BUILD_NUMBER=$build_number
if [[ -z "$DESIRED_PYTHON" ]]; then
DESIRED_PYTHON=('3.5' '3.6' '3.7')
fi
SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
......@@ -26,69 +67,147 @@ if [[ -z "$WIN_PACKAGE_WORK_DIR" ]]; then
WIN_PACKAGE_WORK_DIR="$(echo $(pwd -W) | tr '/' '\\')\\tmp_conda_$(date +%H%M%S)"
fi
if [[ "$OSTYPE" == "msys" ]]; then
mkdir -p "$WIN_PACKAGE_WORK_DIR" || true
vision_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/torchvision-src"
git config --system core.longpaths true
else
vision_rootdir="$(pwd)/torchvision-src"
fi
mkdir -p "$WIN_PACKAGE_WORK_DIR" || true
vision_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/torchvision-src"
git config --system core.longpaths true
if [[ ! -d "$vision_rootdir" ]]; then
rm -rf "$vision_rootdir"
git clone "https://github.com/pytorch/vision" "$vision_rootdir"
pushd "$vision_rootdir"
git checkout v$TORCHVISION_BUILD_VERSION
git checkout $PYTORCH_BRANCH
popd
fi
cd "$SOURCE_DIR"
if [[ "$OSTYPE" == "msys" ]]; then
export tmp_conda="${WIN_PACKAGE_WORK_DIR}\\conda"
export miniconda_exe="${WIN_PACKAGE_WORK_DIR}\\miniconda.exe"
rm -rf "$tmp_conda"
rm -f "$miniconda_exe"
curl -sSk https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe"
"$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe"
pushd $tmp_conda
export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH"
popd
# We have to skip 3.17 because of the following bug.
# https://github.com/conda/conda-build/issues/3285
retry conda install -yq conda-build
fi
export tmp_conda="${WIN_PACKAGE_WORK_DIR}\\conda"
export miniconda_exe="${WIN_PACKAGE_WORK_DIR}\\miniconda.exe"
rm -rf "$tmp_conda"
rm -f "$miniconda_exe"
curl -sSk https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe"
"$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe"
pushd $tmp_conda
export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH"
popd
retry conda install -yq conda-build
ANACONDA_USER=pytorch
ANACONDA_USER=pytorch-nightly
conda config --set anaconda_upload no
export TORCHVISION_PACKAGE_SUFFIX=""
if [[ "$desired_cuda" == 'cpu' ]]; then
export CONDA_CUDATOOLKIT_CONSTRAINT=""
export CONDA_CPUONLY_FEATURE="- cpuonly # [not osx]"
export CUDA_VERSION="None"
if [[ "$OSTYPE" != "darwin"* ]]; then
export TORCHVISION_PACKAGE_SUFFIX="-cpu"
fi
else
export CONDA_CPUONLY_FEATURE=""
. ./switch_cuda_version.sh $desired_cuda
if [[ "$desired_cuda" == "10.0" ]]; then
export CONDA_CUDATOOLKIT_CONSTRAINT=" - cudatoolkit >=10.0,<10.1 # [not osx]"
if [[ "$desired_cuda" == "10.1" ]]; then
export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.1,<10.2 # [not osx]"
elif [[ "$desired_cuda" == "10.0" ]]; then
export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]"
elif [[ "$desired_cuda" == "9.2" ]]; then
export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]"
elif [[ "$desired_cuda" == "9.0" ]]; then
export CONDA_CUDATOOLKIT_CONSTRAINT=" - cudatoolkit >=9.0,<9.1 # [not osx]"
export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.0,<9.1 # [not osx]"
elif [[ "$desired_cuda" == "8.0" ]]; then
export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=8.0,<8.1 # [not osx]"
else
echo "unhandled desired_cuda: $desired_cuda"
exit 1
echo "unhandled desired_cuda: $desired_cuda"
exit 1
fi
fi
if [[ "$OSTYPE" == "msys" ]]; then
time conda build -c $ANACONDA_USER --no-anaconda-upload vs2017
if [[ -z "$PYTORCH_VERSION" ]]; then
export CONDA_CHANNEL_FLAGS="-c pytorch-nightly"
export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | \
python -c "import os, sys, json, re; cuver = '$cuver'; \
cuver = cuver.replace('cu', 'cuda') if cuver != 'cpu' else cuver; \
print(re.sub(r'\\+.*$', '', \
[x['version'] for x in json.load(sys.stdin)['pytorch'] \
if (x['platform'] == 'darwin' or cuver in x['fn']) \
and 'py' + os.environ['DESIRED_PYTHON'] in x['fn']][-1]))")"
if [[ -z "$PYTORCH_VERSION" ]]; then
echo "PyTorch version auto detection failed"
echo "No package found for desired_cuda=$desired_cuda and DESIRED_PYTHON=$DESIRED_PYTHON"
exit 1
fi
else
export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly"
fi
if [[ "$desired_cuda" == 'cpu' ]]; then
export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION"
export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION"
else
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchvision
export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}"
export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}"
fi
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchvision
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchvision
time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchvision
# Loop through all Python versions to build a package for each
for py_ver in "${DESIRED_PYTHON[@]}"; do
build_string="py${py_ver}_${build_string_suffix}"
folder_tag="${build_string}_$(date +'%Y%m%d')"
# Create the conda package into this temporary folder. This is so we can find
# the package afterwards, as there's no easy way to extract the final filename
# from conda-build
output_folder="out_$folder_tag"
rm -rf "$output_folder"
mkdir "$output_folder"
# We need to build the compiler activation scripts first on Windows
time VSDEVCMD_ARGS=${VSDEVCMD_ARGS[@]} \
conda build -c "$ANACONDA_USER" \
--no-anaconda-upload \
--output-folder "$output_folder" \
../vs2017
conda config --set anaconda_upload no
echo "Calling conda-build at $(date)"
if [[ "$desired_cuda" == "9.2" ]]; then
time CMAKE_ARGS=${CMAKE_ARGS[@]} \
BUILD_VERSION="$TORCHVISION_BUILD_VERSION" \
CU_VERSION="$cuver" \
SOURCE_ROOT_DIR="$vision_rootdir" \
conda build -c "$ANACONDA_USER" \
-c defaults \
-c conda-forge \
-c "numba/label/dev" \
--no-anaconda-upload \
--python "$py_ver" \
--output-folder "$output_folder" \
--no-verify \
--no-test \
../torchvision
else
time CMAKE_ARGS=${CMAKE_ARGS[@]} \
BUILD_VERSION="$TORCHVISION_BUILD_VERSION" \
CU_VERSION="$cuver" \
SOURCE_ROOT_DIR="$vision_rootdir" \
conda build -c "$ANACONDA_USER" \
-c defaults \
-c conda-forge \
--no-anaconda-upload \
--python "$py_ver" \
--output-folder "$output_folder" \
--no-verify \
--no-test \
../torchvision
fi
echo "Finished conda-build at $(date)"
# Extract the package for testing
ls -lah "$output_folder"
built_package="$(find $output_folder/ -name '*torchvision*.tar.bz2')"
# Copy the built package to the host machine for persistence before testing
if [[ -n "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then
mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true
cp "$built_package" "$PYTORCH_FINAL_PACKAGE_DIR/"
fi
done
set +e
......@@ -16,6 +16,7 @@ set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%desired_cuda%
set CUDA_BIN_PATH=%CUDA_PATH%\bin
set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr
if "%desired_cuda%" == "9.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
if "%desired_cuda%" == "9.2" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
if "%desired_cuda%" == "10.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50
:cuda_flags_end
......
......@@ -14,6 +14,7 @@ requirements:
- setuptools
{{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }}
{{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }}
{{ environ.get('CONDA_CPUONLY_FEATURE') }}
run:
- python
......@@ -31,7 +32,7 @@ build:
- FORCE_CUDA
- NVCC_FLAGS
features:
{{ CONDA_CPUONLY_FEATURE }}
{{ environ.get('CONDA_CPUONLY_FEATURE') }}
test:
imports:
......
:: Set env vars that tell distutils to use the compiler that we put on path
SET DISTUTILS_USE_SDK=1
SET MSSdk=1
SET "VS_VERSION=15.0"
SET "VS_MAJOR=15"
SET "VS_YEAR=2017"
set "MSYS2_ARG_CONV_EXCL=/AI;/AL;/OUT;/out"
set "MSYS2_ENV_CONV_EXCL=CL"
:: For Python 3.5+, ensure that we link with the dynamic runtime. See
:: http://stevedower.id.au/blog/building-for-python-3-5-part-two/ for more info
set "PY_VCRUNTIME_REDIST=%PREFIX%\\bin\\vcruntime140.dll"
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
set "VSINSTALLDIR=%%i\"
goto :vswhere
)
)
:vswhere
:: Shorten PATH to avoid the `input line too long` error.
SET MyPath=%PATH%
setlocal EnableDelayedExpansion
SET TempPath="%MyPath:;=";"%"
SET var=
FOR %%a IN (%TempPath%) DO (
IF EXIST %%~sa (
SET "var=!var!;%%~sa"
)
)
set "TempPath=!var:~1!"
endlocal & set "PATH=%TempPath%"
:: Shorten current directory too
FOR %%A IN (.) DO CD "%%~sA"
:: other things added by install_activate.bat at package build time
blas_impl:
- mkl # [x86_64]
c_compiler:
- vs2017 # [win]
cxx_compiler:
- vs2017 # [win]
python:
- 3.5
- 3.6
# This differs from target_platform in that it determines what subdir the compiler
# will target, not what subdir the compiler package will be itself.
# For example, we need a win-64 vs2008_win-32 package, so that we compile win-32
# code on win-64 miniconda.
cross_compiler_target_platform:
- win-64 # [win]
target_platform:
- win-64 # [win]
vc:
- 14
zip_keys:
- # [win]
- vc # [win]
- c_compiler # [win]
- cxx_compiler # [win]
set YEAR=2017
set VER=15
mkdir "%PREFIX%\etc\conda\activate.d"
COPY "%RECIPE_DIR%\activate.bat" "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
IF "%cross_compiler_target_platform%" == "win-64" (
set "target_platform=amd64"
echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR% Win64" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
IF "%VSDEVCMD_ARGS%" == "" (
echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
) ELSE (
echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
)
echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
) else (
set "target_platform=x86"
echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo CALL "VC\Auxiliary\Build\vcvars32.bat" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
echo popd
)
set VC_PATH=x86
if "%ARCH%"=="64" (
set VC_PATH=x64
)
set MSC_VER=2017
rem :: This should always be present for VC installed with VS. Not sure about VC installed with Visual C++ Build Tools 2015
rem FOR /F "usebackq tokens=3*" %%A IN (`REG QUERY "HKEY_LOCAL_MACHINE\Software\Microsoft\DevDiv\VC\Servicing\14.0\IDE.x64" /v UpdateVersion`) DO (
rem set SP=%%A
rem )
rem if not "%SP%" == "%PKG_VERSION%" (
rem echo "Version detected from registry: %SP%"
rem echo "does not match version of package being built (%PKG_VERSION%)"
rem echo "Do you have current updates for VS 2015 installed?"
rem exit 1
rem )
REM ========== REQUIRES Win 10 SDK be installed, or files otherwise copied to location below!
robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%" "%LIBRARY_BIN%" *.dll /E
robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%" "%PREFIX%" *.dll /E
if %ERRORLEVEL% GEQ 8 exit 1
REM ========== This one comes from visual studio 2017
set "VC_VER=141"
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
goto :eof
)
)
@setlocal
call "%VS15VARSALL%" x64
set "REDIST_ROOT=%VCToolsRedistDir%%VC_PATH%"
robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%LIBRARY_BIN%" *.dll /E
if %ERRORLEVEL% LSS 8 exit 0
robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%PREFIX%" *.dll /E
if %ERRORLEVEL% LSS 8 exit 0
robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%LIBRARY_BIN%" *.dll /E
if %ERRORLEVEL% LSS 8 exit 0
robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%PREFIX%" *.dll /E
if %ERRORLEVEL% LSS 8 exit 0
@endlocal
{% set vcver="14.1" %}
{% set vcfeature="14" %}
{% set vsyear="2017" %}
{% set fullver="15.4.27004.2010" %}
package:
name: vs{{ vsyear }}
version: {{ fullver }}
build:
skip: True [not win]
script_env:
- VSDEVCMD_ARGS # [win]
outputs:
- name: vs{{ vsyear }}_{{ cross_compiler_target_platform }}
script: install_activate.bat
track_features:
# VS 2017 is binary-compatible with VS 2015/vc14. Tools are "v141".
strong:
- vc{{ vcfeature }}
run_exports:
- vc {{ vcver }}
about:
summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler
license: BSD 3-clause
- name: vs{{ vsyear }}_runtime
script: install_runtime.bat
- name: vc
version: {{ vcver }}
track_features:
- vc{{ vcfeature }}
requirements:
run:
- {{ pin_subpackage('vs' ~ vsyear ~ '_runtime') }}
about:
home: https://github.com/conda/conda/wiki/VC-features
license: Modified BSD License (3-clause)
license_family: BSD
summary: A meta-package to track VC features.
description: |
This metapackage is used to activate vc features without
depending on Python.
doc_url: https://github.com/conda/conda/wiki/VC-features
dev_url: https://github.com/conda/conda/wiki/VC-features
# Turn off auto builds for commits
trigger: none
pr: none
jobs:
- template: templates/build_task.yml
parameters:
package: 'Wheels'
spec: 'CPU'
msagent: true
# Turn off auto builds for commits
trigger: none
pr: none
jobs:
- template: templates/auth_task.yml
- template: templates/build_task.yml
parameters:
package: 'Wheels'
spec: 'CPU'
msagent: true
- template: templates/build_task.yml
parameters:
package: 'Conda'
spec: 'CPU'
msagent: true
- template: templates/build_task.yml
parameters:
package: 'Wheels'
spec: 'CUDA'
msagent: true
- template: templates/build_task.yml
parameters:
package: 'Conda'
spec: 'CUDA'
msagent: true
- template: templates/linux_build_task.yml
parameters:
msagent: $(ms.hosted.agent.cpu)
......@@ -23,18 +23,21 @@ set CUDA_VERSION=%~1
set TORCHVISION_BUILD_VERSION=%~2
set TORCHVISION_BUILD_NUMBER=%~3
set BUILD_VERSION=%TORCHVISION_BUILD_VERSION%
:env_end
if NOT "%CUDA_VERSION%" == "cpu" (
set CUDA_PREFIX=cuda%CUDA_VERSION%
set CUVER=cu%CUDA_VERSION%
set FORCE_CUDA=1
) else (
set CUDA_PREFIX=cpu
set CUVER=cpu
)
set BUILD_VISION=1
set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index
REM set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index
IF "%DESIRED_PYTHON%" == "" set DESIRED_PYTHON=3.5;3.6;3.7
set DESIRED_PYTHON_PREFIX=%DESIRED_PYTHON:.=%
......@@ -61,8 +64,35 @@ FOR %%v IN (%DESIRED_PYTHON%) DO (
set PYTHON_VERSION_STR=%%v
set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
conda remove -n py!PYTHON_VERSION_STR! --all -y || rmdir %CONDA_HOME%\envs\py!PYTHON_VERSION_STR! /s
conda create -n py!PYTHON_VERSION_STR! -y -q numpy>=1.11 mkl>=2018 python=%%v
conda create -n py!PYTHON_VERSION_STR! -y -q -c defaults -c conda-forge numpy>=1.11 mkl>=2018 python=%%v ca-certificates scipy av
)
:: Uncomment for stable releases
:: FOR %%v IN (%DESIRED_PYTHON%) DO (
:: set PYTHON_VERSION_STR=%%v
:: set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
:: set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%"
:: if "%CUDA_VERSION%" == "100" (
:: set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl
:: ) else (
:: set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0%%2B%CUVER%-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl
:: )
:: echo Installing !TORCH_WHEEL!...
:: pip install "!TORCH_WHEEL!"
:: )
:: Uncomment for nightly releases
FOR %%v IN (%DESIRED_PYTHON%) DO (
set PYTHON_VERSION_STR=%%v
set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=!
set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%"
set TORCH_WHEEL=torch --pre -f https://download.pytorch.org/whl/nightly/%CUVER%/torch_nightly.html
echo Installing !TORCH_WHEEL!...
pip install !TORCH_WHEEL!
)
endlocal
if "%DEBUG%" == "1" (
......@@ -71,11 +101,22 @@ if "%DEBUG%" == "1" (
set BUILD_TYPE=release
)
:: Install sccache
if "%USE_SCCACHE%" == "1" (
mkdir %CD%\tmp_bin
curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %CD%\tmp_bin\sccache.exe
if not "%CUDA_VERSION%" == "" (
copy %CD%\tmp_bin\sccache.exe %CD%\tmp_bin\nvcc.exe
set CUDA_NVCC_EXECUTABLE=%CD%\tmp_bin\nvcc
set "PATH=%CD%\tmp_bin;%PATH%"
)
)
for %%v in (%DESIRED_PYTHON_PREFIX%) do (
:: Activate Python Environment
set PYTHON_PREFIX=%%v
set "PATH=%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%"
pip install %TORCH_WHEEL%
@setlocal
:: Set Flags
if NOT "%CUDA_VERSION%"=="cpu" (
......
@echo off
IF NOT "%BUILD_VISION%" == "" (
set MODULE_NAME=vision
) ELSE (
set MODULE_NAME=pytorch
)
IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" (
call internal\clone.bat
cd ..
IF ERRORLEVEL 1 goto eof
) ELSE (
call internal\clean.bat
)
call internal\check_deps.bat
IF ERRORLEVEL 1 goto eof
REM Check for optional components
set USE_CUDA=
set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
IF "%NVTOOLSEXT_PATH%"=="" (
echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing
exit /b 1
goto optcheck
)
IF "%CUDA_PATH_V9_2%"=="" (
echo CUDA 9.2 not found, failing
exit /b 1
) ELSE (
IF "%BUILD_VISION%" == "" (
set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0
set TORCH_NVCC_FLAGS=-Xfatbin -compress-all
) ELSE (
set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50
)
set "CUDA_PATH=%CUDA_PATH_V9_2%"
set "PATH=%CUDA_PATH_V9_2%\bin;%PATH%"
)
:optcheck
IF "%BUILD_VISION%" == "" (
call internal\check_opts.bat
IF ERRORLEVEL 1 goto eof
call internal\copy.bat
IF ERRORLEVEL 1 goto eof
)
call internal\setup.bat
IF ERRORLEVEL 1 goto eof
:eof
@echo off
curl -k https://www.7-zip.org/a/7z1805-x64.exe -O
if errorlevel 1 exit /b 1
start /wait 7z1805-x64.exe /S
if errorlevel 1 exit /b 1
set "PATH=%ProgramFiles%\7-Zip;%PATH%"
......@@ -44,15 +44,11 @@ set PYTORCH_BRANCH=%last_commit%
:latest_end
IF "%BUILD_VISION%" == "" (
IF "%PYTORCH_BRANCH%" == "" (
set PYTORCH_BRANCH=v%PYTORCH_BUILD_VERSION%
)
git checkout %PYTORCH_BRANCH%
IF ERRORLEVEL 1 git checkout tags/%PYTORCH_BRANCH%
) ELSE (
git checkout v%TORCHVISION_BUILD_VERSION%
IF "%PYTORCH_BRANCH%" == "" (
set PYTORCH_BRANCH=v%TORCHVISION_BUILD_VERSION%
)
git checkout %PYTORCH_BRANCH%
IF ERRORLEVEL 1 git checkout tags/%PYTORCH_BRANCH%
:submodule
......
@echo off
@echo on
if "%CUDA_VERSION%" == "cpu" (
echo Skipping for CPU builds
exit /b 0
)
set SRC_DIR=%~dp0\..
......@@ -9,87 +14,60 @@ set CUDA_VER_MAJOR=%CUDA_VERSION:~0,-1%
set CUDA_VER_MINOR=%CUDA_VERSION:~-1,1%
set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR%
IF %CUDA_VER% LEQ 90 (
set "NVCC_PACKAGE=compiler_%CUDA_VERSION_STR%"
) ELSE (
set "NVCC_PACKAGE=nvcc_%CUDA_VERSION_STR%"
)
IF %CUDA_VER% EQU 80 goto cuda80
IF %CUDA_VER% EQU 90 goto cuda90
IF %CUDA_VER% EQU 91 goto cuda91
IF %CUDA_VER% EQU 92 goto cuda92
IF %CUDA_VER% EQU 100 goto cuda100
if %CUDA_VER% EQU 92 goto cuda92
if %CUDA_VER% EQU 100 goto cuda100
echo CUDA %CUDA_VERSION_STR% is not supported
exit /b 1
:cuda80
echo CUDA 8.0 is not supported
exit /b 1
:cuda90
IF NOT EXIST "%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z" (
curl -k -L https://www.dropbox.com/s/z5b7ryz0zrimntl/cuda_9.0.176_windows.7z?dl=1 --output "%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z"
:cuda92
if not exist "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe" (
curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_9.2.148_win10.exe --output "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe"
if errorlevel 1 exit /b 1
set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z"
set "NVCC_PACKAGE=compiler_%CUDA_VERSION_STR%"
set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe"
set "ARGS=nvcc_9.2 cuobjdump_9.2 nvprune_9.2 cupti_9.2 cublas_9.2 cublas_dev_9.2 cudart_9.2 cufft_9.2 cufft_dev_9.2 curand_9.2 curand_dev_9.2 cusolver_9.2 cusolver_dev_9.2 cusparse_9.2 cusparse_dev_9.2 nvgraph_9.2 nvgraph_dev_9.2 npp_9.2 npp_dev_9.2 nvrtc_9.2 nvrtc_dev_9.2 nvml_dev_9.2"
)
IF NOT EXIST "%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip" (
curl -k -L https://www.dropbox.com/s/6p0xyqh472nu8m1/cudnn-9.0-windows7-x64-v7.zip?dl=1 --output "%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip"
if not exist "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip" (
curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-9.2-windows10-x64-v7.2.1.38.zip --output "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip"
if errorlevel 1 exit /b 1
set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip"
set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip"
)
goto cuda_common
:cuda91
:cuda100
IF NOT EXIST "%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z" (
curl -k -L https://www.dropbox.com/s/7a4sbq0dln6v7t2/cuda_9.1.85_windows.7z?dl=1 --output "%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z"
if not exist "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe" (
curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_10.0.130_411.31_win10.exe --output "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe"
if errorlevel 1 exit /b 1
set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z"
set "NVCC_PACKAGE=nvcc_%CUDA_VERSION_STR%"
set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe"
set "ARGS=nvcc_10.0 cuobjdump_10.0 nvprune_10.0 cupti_10.0 cublas_10.0 cublas_dev_10.0 cudart_10.0 cufft_10.0 cufft_dev_10.0 curand_10.0 curand_dev_10.0 cusolver_10.0 cusolver_dev_10.0 cusparse_10.0 cusparse_dev_10.0 nvgraph_10.0 nvgraph_dev_10.0 npp_10.0 npp_dev_10.0 nvrtc_10.0 nvrtc_dev_10.0 nvml_dev_10.0"
)
IF NOT EXIST "%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip" (
curl -k -L https://www.dropbox.com/s/e0prhgsrbyfi4ov/cudnn-9.1-windows7-x64-v7.zip?dl=1 --output "%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip"
if not exist "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip" (
curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-10.0-windows10-x64-v7.4.1.5.zip --output "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip"
if errorlevel 1 exit /b 1
set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip"
set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip"
)
goto cuda_common
:cuda92
echo CUDA 9.2 is not supported
exit /b 1
:cuda100
echo CUDA 10.0 is not supported
exit /b 1
:cuda_common
set "CUDA_PREFIX=cuda%CUDA_VERSION%"
IF NOT EXIST "%SRC_DIR%\temp_build\NvToolsExt.7z" (
if not exist "%SRC_DIR%\temp_build\NvToolsExt.7z" (
curl -k -L https://www.dropbox.com/s/9mcolalfdj4n979/NvToolsExt.7z?dl=1 --output "%SRC_DIR%\temp_build\NvToolsExt.7z"
if errorlevel 1 exit /b 1
)
echo Installing CUDA toolkit...
7z x %CUDA_SETUP_FILE% -o"%SRC_DIR%\temp_build\cuda"
pushd "%SRC_DIR%\temp_build\cuda"
dir
start /wait setup.exe -s %NVCC_PACKAGE% cublas_%CUDA_VERSION_STR% cublas_dev_%CUDA_VERSION_STR% cudart_%CUDA_VERSION_STR% curand_%CUDA_VERSION_STR% curand_dev_%CUDA_VERSION_STR% cusparse_%CUDA_VERSION_STR% cusparse_dev_%CUDA_VERSION_STR% nvrtc_%CUDA_VERSION_STR% nvrtc_dev_%CUDA_VERSION_STR% cufft_%CUDA_VERSION_STR% cufft_dev_%CUDA_VERSION_STR%
start /wait setup.exe -s %ARGS%
popd
echo Installing VS integration...
xcopy /Y "%SRC_DIR%\temp_build\cuda\_vs\*.*" "c:\Program Files (x86)\MSBuild\Microsoft.Cpp\v4.0\V140\BuildCustomizations"
xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\IDE\VC\VCTargets\BuildCustomizations"
echo Installing NvToolsExt...
7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt"
......@@ -100,23 +78,22 @@ xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\bin\x64\*.*" "%ProgramFiles%\NVIDIA Co
xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\include\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\include"
xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\lib\x64\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\lib\x64"
echo Setting up environment...
set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\libnvvp;%PATH%"
set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
set "CUDA_PATH_V%CUDA_VER_MAJOR%_%CUDA_VER_MINOR%=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
set "NVTOOLSEXT_PATH=%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64"
if not exist "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin\nvcc.exe" (
echo CUDA %CUDA_VERSION_STR% installed failed.
exit /b 1
)
echo Installing cuDNN...
7z x %CUDNN_SETUP_FILE% -o"%SRC_DIR%\temp_build\cudnn"
xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\bin\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin"
xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\lib\x64\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\lib\x64"
xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\include\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\include"
echo Setting up environment...
set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\libnvvp;%PATH%"
set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
set "CUDA_PATH_V%CUDA_VER_MAJOR%_%CUDA_VER_MINOR%=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%"
set "NVTOOLSEXT_PATH=%ProgramFiles%\NVIDIA Corporation\NvToolsExt\"
echo Cleaning temp files
rd /s /q "%SRC_DIR%\temp_build"
pushd %TEMP%
rd /s /q .
popd
echo Using VS2015 as NVCC compiler
set "CUDAHOSTCXX=%VS140COMNTOOLS%\..\..\VC\bin\amd64\cl.exe"
rd /s /q "%SRC_DIR%\temp_build" || ver > nul
@echo off
REM curl -k https://www.7-zip.org/a/7z1805-x64.exe -O
REM if errorlevel 1 exit /b 1
REM start /wait 7z1805-x64.exe /S
REM if errorlevel 1 exit /b 1
REM set "PATH=%ProgramFiles%\7-Zip;%PATH%"
choco feature disable --name showDownloadProgress
choco feature enable --name allowGlobalConfirmation
choco install curl 7zip
@echo off
:: Caution: Please don't use this script locally
:: It may destroy your build environment.
setlocal
IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" (
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
exit /b 1
)
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do (
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
set "VS15INSTALLDIR=%%i"
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
goto vswhere
)
)
:vswhere
IF "%VS15VCVARSALL%"=="" (
echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows
exit /b 1
)
call "%VS15VCVARSALL%" x86_amd64
for /f "usebackq tokens=*" %%i in (`where link.exe`) do move "%%i" "%%i.bak"
endlocal
@echo on
if "%~1"=="" goto arg_error
if NOT "%~2"=="" goto arg_error
goto arg_end
:arg_error
echo Illegal number of parameters. Pass packge type `Conda` or `Wheels`.
exit /b 1
:arg_end
echo "nightly_defaults.bat at %CD% starting at %DATE%"
set SRC_DIR=%~dp0\..
:: NIGHTLIES_FOLDER
:: N.B. this is also defined in cron_start.sh
:: An arbitrary root folder to store all nightlies folders, each of which is a
:: parent level date folder with separate subdirs for logs, wheels, conda
:: packages, etc. This should be kept the same across all scripts called in a
:: cron job, so it only has a default value in the top-most script
:: build_cron.sh to avoid the default values from diverging.
if "%NIGHTLIES_FOLDER%" == "" set "NIGHTLIES_FOLDER=%SRC_DIR%"
:: NIGHTLIES_DATE
:: N.B. this is also defined in cron_start.sh
:: The date in YYYY_mm_dd format that we are building for. If this is not
:: already set, then this will first try to find the date of the nightlies
:: folder that this builder repo exists in; e.g. if this script exists in
:: some_dir/2019_09_04/builder/cron/ then this will be set to 2019_09_04 (must
:: match YYYY_mm_dd). This is for convenience when debugging/uploading past
:: dates, so that you don't have to set NIGHTLIES_DATE yourself. If a date
:: folder cannot be found in that exact location, then this will default to
:: the current date.
if "%NIGHTLIES_DATE%" == "" ( goto date_start ) else ( goto date_end )
:date_start
set "DATE_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyy_MM_dd'"
set "DATE_COMPACT_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyyMMdd'"
FOR /F "delims=" %%i IN ('powershell -c "%DATE_CMD%"') DO set NIGHTLIES_DATE=%%i
FOR /F "delims=" %%i IN ('powershell -c "%DATE_COMPACT_CMD%"') DO set NIGHTLIES_DATE_COMPACT=%%i
:date_end
if "%NIGHTLIES_DATE_COMPACT%" == "" set NIGHTLIES_DATE_COMPACT=%NIGHTLIES_DATE:~0,4%%NIGHTLIES_DATE:~5,2%%NIGHTLIES_DATE:~8,2%
:: Used in lots of places as the root dir to store all conda/wheel/manywheel
:: packages as well as logs for the day
set today=%NIGHTLIES_FOLDER%\%NIGHTLIES_DATE%
mkdir "%today%" || ver >nul
::#############################################################################
:: Add new configuration variables below this line. 'today' should always be
:: defined ASAP to avoid weird errors
::#############################################################################
:: List of people to email when things go wrong. This is passed directly to
:: `mail -t`
:: TODO: Not supported yet
if "%NIGHTLIES_EMAIL_LIST%" == "" set NIGHTLIES_EMAIL_LIST=peterghost86@gmail.com
:: PYTORCH_CREDENTIALS_FILE
:: A bash file that exports credentials needed to upload to aws and anaconda.
:: Needed variables are PYTORCH_ANACONDA_USERNAME, PYTORCH_ANACONDA_PASSWORD,
:: AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY. Or it can just export the AWS
:: keys and then prepend a logged-in conda installation to the path.
:: TODO: Not supported yet
if "%PYTORCH_CREDENTIALS_FILE%" == "" set PYTORCH_CREDENTIALS_FILE=/c/Users/administrator/nightlies/credentials.sh
:: Location of the temporary miniconda that is downloaded to install conda-build
:: and aws to upload finished packages TODO this is messy to install this in
:: upload.sh and later use it in upload_logs.sh
if "%CONDA_UPLOADER_INSTALLATION%" == "" set "CONDA_UPLOADER_INSTALLATION=%today%\miniconda"
:: N.B. BUILDER_REPO and BUILDER_BRANCH are both set in cron_start.sh, as that
:: is the script that actually clones the builder repo that /this/ script is
:: running from.
pushd "%SRC_DIR%\.."
set NIGHTLIES_BUILDER_ROOT=%CD%
popd
:: The shared pytorch repo to be used by all builds
if "%NIGHTLIES_PYTORCH_ROOT%" == "" set "NIGHTLIES_PYTORCH_ROOT=%today%\vision"
:: PYTORCH_REPO
:: The Github org/user whose fork of Pytorch to check out (git clone
:: https://github.com/<THIS_PART>/pytorch.git). This will always be cloned
:: fresh to build with. Default is 'pytorch'
if "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch
:: PYTORCH_BRANCH
:: The branch of Pytorch to checkout for building (git checkout <THIS_PART>).
:: This can either be the name of the branch (e.g. git checkout
:: my_branch_name) or can be a git commit (git checkout 4b2674n...). Default
:: is 'latest', which is a special term that signals to pull the last commit
:: before 0:00 midnight on the NIGHTLIES_DATE
if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=latest
:: Clone the requested pytorch checkout
if exist "%NIGHTLIES_PYTORCH_ROOT%" ( goto clone_end ) else ( goto clone_start )
:clone_start
git clone --recursive "https://github.com/%PYTORCH_REPO%/vision.git" "%NIGHTLIES_PYTORCH_ROOT%"
pushd "%NIGHTLIES_PYTORCH_ROOT%"
if "%PYTORCH_BRANCH%" == "latest" ( goto latest_start ) else ( goto latest_end )
:latest_start
:: Switch to the latest commit by 11:59 yesterday
echo PYTORCH_BRANCH is set to latest so I will find the last commit
echo before 0:00 midnight on %NIGHTLIES_DATE%
set git_date=%NIGHTLIES_DATE:_=-%
FOR /F "delims=" %%i IN ('git log --before %git_date% -n 1 "--pretty=%%H"') DO set last_commit=%%i
echo Setting PYTORCH_BRANCH to %last_commit% since that was the last
echo commit before %NIGHTLIES_DATE%
set PYTORCH_BRANCH=%last_commit%
:latest_end
git checkout "%PYTORCH_BRANCH%"
git submodule update
popd
:clone_end
if "%CUDA_VERSION%" == "cpu" (
set _DESIRED_CUDA=cpu
) else (
set _DESIRED_CUDA=cu%CUDA_VERSION%
)
:: PYTORCH_BUILD_VERSION
:: The actual version string. Used in conda like
:: pytorch-nightly==1.0.0.dev20180908
:: or in manylinux like
:: torch_nightly-1.0.0.dev20180908-cp27-cp27m-linux_x86_64.whl
if "%TORCHVISION_BUILD_VERSION%" == "" set TORCHVISION_BUILD_VERSION=0.5.0.dev%NIGHTLIES_DATE_COMPACT%
if "%~1" == "Wheels" (
if not "%CUDA_VERSION%" == "100" (
set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%+%_DESIRED_CUDA%
)
)
:: PYTORCH_BUILD_NUMBER
:: This is usually the number 1. If more than one build is uploaded for the
:: same version/date, then this can be incremented to 2,3 etc in which case
:: '.post2' will be appended to the version string of the package. This can
:: be set to '0' only if OVERRIDE_PACKAGE_VERSION is being used to bypass
:: all the version string logic in downstream scripts. Since we use the
:: override below, exporting this shouldn't actually matter.
if "%TORCHVISION_BUILD_NUMBER%" == "" set /a TORCHVISION_BUILD_NUMBER=1
if %TORCHVISION_BUILD_NUMBER% GTR 1 set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%%TORCHVISION_BUILD_NUMBER%
:: The nightly builds use their own versioning logic, so we override whatever
:: logic is in setup.py or other scripts
:: TODO: Not supported yet
set OVERRIDE_PACKAGE_VERSION=%TORCHVISION_BUILD_VERSION%
set BUILD_VERSION=%TORCHVISION_BUILD_VERSION%
:: Build folder for conda builds to use
if "%TORCH_CONDA_BUILD_FOLDER%" == "" set TORCH_CONDA_BUILD_FOLDER=torchvision
:: TORCH_PACKAGE_NAME
:: The name of the package to upload. This should probably be pytorch or
:: pytorch-nightly. N.B. that pip will change all '-' to '_' but conda will
:: not. This is dealt with in downstream scripts.
:: TODO: Not supported yet
if "%TORCH_PACKAGE_NAME%" == "" set TORCH_PACKAGE_NAME=torchvision
:: PIP_UPLOAD_FOLDER should end in a slash. This is to handle it being empty
:: (when uploading to e.g. whl/cpu/) and also to handle nightlies (when
:: uploading to e.g. /whl/nightly/cpu)
:: TODO: Not supported yet
if "%PIP_UPLOAD_FOLDER%" == "" set "PIP_UPLOAD_FOLDER=nightly\"
:: The location of the binary_sizes dir in s3 is hardcoded into
:: upload_binary_sizes.sh
:: DAYS_TO_KEEP
:: How many days to keep around for clean.sh. Build folders older than this
:: will be purged at the end of cron jobs. '1' means to keep only the current
:: day. Values less than 1 are not allowed. The default is 5.
:: TODO: Not supported yet
if "%DAYS_TO_KEEP%" == "" set /a DAYS_TO_KEEP=5
if %DAYS_TO_KEEP% LSS 1 (
echo DAYS_TO_KEEP cannot be less than 1.
echo A value of 1 means to only keep the build for today
exit /b 1
)
@echo off
set SRC_DIR=%~dp0
pushd %SRC_DIR%
if NOT "%CUDA_VERSION%" == "cpu" (
set PACKAGE_SUFFIX=_cuda%CUDA_VERSION%
) else (
set PACKAGE_SUFFIX=
)
if "%PACKAGEFULLNAME%" == "Conda" (
set PACKAGE=conda
) else (
set PACKAGE=wheels
)
set PUBLISH_BRANCH=%PACKAGE%_%DESIRED_PYTHON%%PACKAGE_SUFFIX%
git clone %ARTIFACT_REPO_URL% -b %PUBLISH_BRANCH% --single-branch >nul 2>&1
IF ERRORLEVEL 1 (
echo Branch %PUBLISH_BRANCH% not exist, falling back to master
set NO_BRANCH=1
git clone %ARTIFACT_REPO_URL% -b master --single-branch >nul 2>&1
)
IF ERRORLEVEL 1 (
echo Clone failed
goto err
)
cd pytorch_builder
attrib -s -h -r . /s /d
:: Empty repo
rd /s /q . || ver >nul
IF NOT EXIST %PACKAGE% mkdir %PACKAGE%
xcopy /S /E /Y ..\..\output\*.* %PACKAGE%\
git config --global user.name "Azure DevOps"
git config --global user.email peterghost86@gmail.com
git init
git checkout --orphan %PUBLISH_BRANCH%
git remote add origin %ARTIFACT_REPO_URL%
git add .
git commit -m "Update artifacts"
:push
if "%RETRY_TIMES%" == "" (
set /a RETRY_TIMES=10
set /a SLEEP_TIME=2
) else (
set /a RETRY_TIMES=%RETRY_TIMES%-1
set /a SLEEP_TIME=%SLEEP_TIME%*2
)
git push origin %PUBLISH_BRANCH% -f > nul 2>&1
IF ERRORLEVEL 1 (
echo Git push retry times remaining: %RETRY_TIMES%
echo Sleep time: %SLEEP_TIME% seconds
IF %RETRY_TIMES% EQU 0 (
echo Push failed
goto err
)
waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul
goto push
) ELSE (
set RETRY_TIMES=
set SLEEP_TIME=
)
popd
exit /b 0
:err
popd
exit /b 1
......@@ -28,53 +28,6 @@ if "%CXX%"=="sccache cl" (
sccache --zero-stats
)
if "%BUILD_PYTHONLESS%" == "" goto pytorch else goto libtorch
:libtorch
set VARIANT=shared-with-deps
mkdir libtorch
mkdir libtorch\bin
mkdir libtorch\cmake
mkdir libtorch\include
mkdir libtorch\lib
mkdir libtorch\share
mkdir libtorch\test
mkdir build
pushd build
python ../tools/build_libtorch.py
popd
IF ERRORLEVEL 1 exit /b 1
IF NOT ERRORLEVEL 0 exit /b 1
move /Y torch\bin\*.* libtorch\bin\
move /Y torch\cmake\*.* libtorch\cmake\
robocopy /move /e torch\include\ libtorch\include\
move /Y torch\lib\*.* libtorch\lib\
robocopy /move /e torch\share\ libtorch\share\
move /Y torch\test\*.* libtorch\test\
move /Y libtorch\bin\*.dll libtorch\lib\
git rev-parse HEAD > libtorch\build-hash
IF "%DEBUG%" == "" (
set LIBTORCH_PREFIX=libtorch-win-%VARIANT%
) ELSE (
set LIBTORCH_PREFIX=libtorch-win-%VARIANT%-debug
)
7z a -tzip %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip libtorch\*
mkdir ..\output\%CUDA_PREFIX%
copy /Y %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip ..\output\%CUDA_PREFIX%\
copy /Y %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip ..\output\%CUDA_PREFIX%\%LIBTORCH_PREFIX%-latest.zip
goto build_end
:pytorch
:: This stores in e.g. D:/_work/1/s/windows/output/cpu
pip wheel -e . --no-deps --wheel-dir ../output/%CUDA_PREFIX%
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment